Merge branch 'fusion-ldrd-dev' into epic-dev

Merge remote-tracking branch 'origin/master' into fusion-ldrd-dev
Fix typo in cmake file
2022-03-30 12:35:54 -07:00 · 2021-08-02 16:00:08 -07:00 · 2021-07-06 21:18:33 -07:00 · 2021-07-01 16:41:56 -07:00 · 2021-07-01 16:41:54 -07:00 · 2021-07-01 16:40:23 -07:00
105 changed files with 2203 additions and 5352 deletions
@@ -50,7 +50,6 @@ variables:
  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
  MFEM_DATA_REPO: https://github.com/mfem/data.git
  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1

 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
@@ -30,5 +30,5 @@

 opt_mpi_cuda_xl_16_1_1_8:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
  extends: .build_and_test_on_lassen
@@ -10,15 +10,6 @@

 Version 4.3.1 (development)
 ===========================
- Added support for hr-adaptivity using TMOP-based error estimator.
-
- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
-  solver is now fully working on GPUs.
-
- Tetrahedral meshes no longer need to be reordered to support high order
-  Nedelec basis functions. This will allow future support for Nedelec basis
-  functions on wedges and pyramids which are not amenable to reordering. The
-  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.


 Version 4.3, released on July 29, 2021
@@ -1,66 +0,0 @@
-cff-version: 1.2.0
-message: "If you use MFEM, please cite it as follows."
-authors:
- family-names: "MFEM Team"
-title: "MFEM: Modular Finite Element Methods [Software]"
-doi: 10.11578/dc.20171025.1248
-url: "https://mfem.org"
-preferred-citation:
-  type: article
-  authors:
-  - family-names: "Anderson"
-    given-names: "Robert"
-    orcid: "https://orcid.org/0000-0002-3508-9944"
-  - family-names: "Andrej"
-    given-names: "Julian"
-    orcid: "https://orcid.org/0000-0001-7661-4840"
-  - family-names: "Barker"
-    given-names: "Andrew"
-    orcid: "https://orcid.org/0000-0003-3572-911X"
-  - family-names: "Bramwell"
-    given-names: "Jamie"
-  - family-names: "Camier"
-    given-names: "Jean-Sylvain"
-    orcid: "https://orcid.org/0000-0003-2421-1999"
-  - family-names: "Cerveny"
-    given-names: "Jakub"
-    orcid: "https://orcid.org/0000-0003-4231-2531"
-  - family-names: "Dobrev"
-    given-names: "Veselin"
-    orcid: "https://orcid.org/0000-0003-1793-5622"
-  - family-names: "Dudouit"
-    given-names: "Yohann"
-    orcid: "https://orcid.org/0000-0001-5831-561X"
-  - family-names: "Fisher"
-    given-names: "Aaron"
-  - family-names: "Kolev"
-    given-names: "Tzanio"
-    orcid: "https://orcid.org/0000-0002-2810-3090"
-  - family-names: "Pazner"
-    given-names: "Will"
-    orcid: "https://orcid.org/0000-0003-4885-2934"
-  - family-names: "Stowell"
-    given-names: "Mark"
-    orcid: "https://orcid.org/0000-0002-5389-7435"
-  - family-names: "Tomov"
-    given-names: "Vladimir"
-    orcid: "https://orcid.org/0000-0002-1846-6816"
-  - family-names: "Akkerman"
-    given-names: "Ido"
-    orcid: "https://orcid.org/0000-0002-5937-0300"
-  - family-names: "Dahm"
-    given-names: "Johann"
-    orcid: "https://orcid.org/0000-0001-9657-3564"
-  - family-names: "Medina"
-    given-names: "David"
-  - family-names: "Zampini"
-    given-names: "Stefano"
-    orcid: "https://orcid.org/0000-0002-0435-0433"
-  doi: "10.1016/j.camwa.2020.06.009"
-  journal: "Computers \\& Mathematics with Applications"
-  month: 1
-  start: 42 # First page number
-  end: 74 # Last page number
-  title: "MFEM: A Modular Finite Element Methods Library"
-  volume: 81
-  year: 2021
@@ -267,6 +267,15 @@ if (MFEM_USE_SUNDIALS)
  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

+# EPIC
+if (MFEM_USE_EPIC)
+  if (NOT (MFEM_USE_MPI AND MFEM_USE_SUNDIALS AND MFEM_USE_LAPACK) )
+    message(FATAL_ERROR " *** EPIC requires that MPI, SUNDIALS and LAPACK be enabled.")
+  else()
+    find_package(EPIC REQUIRED SUNDIALS NVector_Serial NVector_Parallel BLAS LAPACK)
+  endif()
+endif()
+
 # Mesquite
 if (MFEM_USE_MESQUITE)
  find_package(Mesquite REQUIRED)
@@ -428,7 +437,7 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
+set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS EPIC PETSC
    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
    CUSPARSE MKL_CPARDISO AMGX CALIPER)
@@ -549,7 +549,7 @@ The specific libraries and their options are:
  Options: HYPRE_OPT, HYPRE_LIB.
  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.1   (HYPRE built with CUDA)
+            HYPRE >= 2.22.0   (HYPRE built with CUDA)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -29,6 +29,7 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
+set(MFEM_USE_EPIC @MFEM_USE_EPIC@)
 set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
@@ -165,6 +165,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 #cmakedefine MFEM_USE_SUNDIALS

+// Enable MFEM functionality based on the EPIC libraries.
+#cmakedefine MFEM_USE_EPIC
+
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@

@@ -0,0 +1,21 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - EPIC_FOUND
+#   - EPIC_LIBRARIES
+#   - EPIC_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(EPIC EPIC EPIC_DIR
+       "include" Epic.h "lib" epic1.0.0
+  "Paths to headers required by EPIC." "Libraries required by EPIC.")
+
@@ -759,7 +759,7 @@ function(mfem_export_mk_files)
  set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_EPIC MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
@@ -85,6 +85,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS

+// Enable MFEM functionality based on the EPIC libraries.
+// #define MFEM_USE_EPIC
+
 // Enable MFEM functionality based on the Mesquite library.
 // #define MFEM_USE_MESQUITE

@@ -29,6 +29,7 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
+MFEM_USE_EPIC          = @MFEM_USE_EPIC@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
@@ -30,6 +30,7 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
+option(MFEM_USE_EPIC "Enable EPIC usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
@@ -115,6 +116,9 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")

+set(EPIC_DIR "${MFEM_DIR}/../epic-cpp/instdir" CACHE PATH
+    "Path to the EPIC library.")
+
 set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
    "Path to the Mesquite library.")

@@ -122,6 +122,7 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
+MFEM_USE_EPIC          = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
@@ -231,6 +232,11 @@ endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES

+# EPIC library configuration
+MESQUITE_DIR = @MFEM_DIR@/../epic-cpp/instdir
+MESQUITE_OPT = -I$(EPIC_DIR)/include
+MESQUITE_LIB = -L$(EPIC_DIR)/lib -lepic1.0.0
+
 # MESQUITE library configuration
 MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
 MESQUITE_OPT = -I$(MESQUITE_DIR)/include
@@ -159,6 +159,11 @@ if (MFEM_USE_AMGX)
  add_subdirectory(amgx)
 endif()

+# Include the examples/epic directory if EPIC is enabled.
+if (MFEM_USE_EPIC)
+  add_subdirectory(epic)
+endif()
+
 # Include the examples/ginkgo directory if GINKGO is enabled.
 if (MFEM_USE_GINKGO)
  add_subdirectory(ginkgo)
@@ -206,9 +206,9 @@ int main(int argc, char *argv[])
   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
+   MFEM_PERF_BEGIN("Solve A X=B");
   if (!pa)
   {
-      MFEM_PERF_SCOPE("Solve A X=B (FA)");
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
      GSSmoother M((SparseMatrix&)(*A));
@@ -223,7 +223,6 @@ int main(int argc, char *argv[])
   }
   else // Jacobi preconditioning in partial assembly mode
   {
-      MFEM_PERF_SCOPE("Solve A X=B (PA)");
      if (UsesTensorBasis(fespace))
      {
         OperatorJacobiSmoother M(a, ess_tdof_list);
@@ -234,6 +233,7 @@ int main(int argc, char *argv[])
         CG(*A, B, X, 1, 400, 1e-12, 0.0);
      }
   }
+   MFEM_PERF_END("Solve A X=B");
   // 12. Recover the solution as a finite element grid function.
   a.RecoverFEMSolution(X, b, x);

@@ -231,29 +231,28 @@ int main(int argc, char *argv[])
   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
   //     * With partial assembly, use Jacobi smoothing, for now.
+   MFEM_PERF_BEGIN("Solve A X = B");
+   Solver *prec = NULL;
+   if (pa)
   {
-      MFEM_PERF_SCOPE("Solve A X=B");
-      Solver *prec = NULL;
-      if (pa)
+      if (UsesTensorBasis(fespace))
      {
-         if (UsesTensorBasis(fespace))
-         {
-            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-         }
+         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
      }
-      else
-      {
-         prec = new HypreBoomerAMG;
-      }
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
   }
+   else
+   {
+      prec = new HypreBoomerAMG;
+   }
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   if (prec) { cg.SetPreconditioner(*prec); }
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete prec;
+   MFEM_PERF_END("Solve A X = B");
   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, x);
@@ -0,0 +1,64 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(EPIC_EXAMPLES_SRCS)
+list(APPEND EPIC_EXAMPLES_SRCS
+  ex16.cpp
+  )
+
+if (MFEM_USE_MPI)
+list(APPEND EPIC_EXAMPLES_SRCS
+    ex16p.cpp
+    )
+endif()
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_epic" target, see below.
+add_custom_target(test_epic
+  ${CMAKE_CTEST_COMMAND} -R epic USES_TERMINAL)
+
+# Add one executable per cpp file, adding "epic_" as prefix. Sets
+# "test_epic" as a target that depends on the given examples.
+set(PFX epic_)
+add_mfem_examples(EPIC_EXAMPLES_SRCS ${PFX} "" test_epic)
+
+# Testing.
+# The EPIC tests can be run separately using the target "test_epic"
+# which builds the examples and runs:
+#   ctest -R epic
+
+# Example 16: use the default options
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${EPIC_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
+
@@ -0,0 +1,17 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               http://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the EPIC suite of time integration.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_EPIC = YES".
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.
@@ -0,0 +1,610 @@
+//                                MFEM Example 16
+//                               EPIC Modification
+//
+// Compile with: make ex16
+//
+// Sample runs:  ex16
+//               ex16 -m ../../data/inline-tri.mesh
+//               ex16 -m ../../data/disc-nurbs.mesh -tf 2
+//               ex16 -s 8 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
+//               ex16 -m ../../data/fichera-q2.mesh
+//               ex16 -m ../../data/escher.mesh
+//               ex16 -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
+//               ex16 -m ../../data/amr-quad.mesh -o 4 -r 0
+//               ex16 -m ../../data/amr-hex.mesh -o 2 -r 0
+//
+// Description:  This example solves a time dependent nonlinear heat equation
+//               problem of the form du/dt = C(u), with a non-linear diffusion
+//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
+//
+//               We recommend viewing examples 2, 9 and 10 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ImplicitSolveOperator;
+class JacobianOperator;
+
+/** After spatial discretization, the conduction model can be written as:
+ *
+ *     du/dt = M^{-1}(-K(u) u)
+ *
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
+ *
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
+ */
+class ConductionOperator : public TimeDependentOperator
+{
+protected:
+   FiniteElementSpace &fespace;
+   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
+
+   BilinearForm *M;
+   mutable BilinearForm *K;
+   mutable BilinearForm *dK;
+   mutable BilinearForm *J_K;
+
+   SparseMatrix Mmat;
+   mutable SparseMatrix J_K_mat;
+
+   mutable CGSolver M_solver; // Krylov solver for inverting the mass matrix M
+   DSmoother M_prec;  // Preconditioner for the mass matrix M
+
+   CGSolver Jg_solver; // Krylov solver for inverting the Jacobian in the nonlinear solve
+   DSmoother Jg_prec;  // Preconditioner for the Jacobian Jg
+
+   NewtonSolver newton_solver;
+   mutable JacobianOperator *jac;
+
+   double alpha, kappa;
+
+   mutable Vector z; // auxiliary vector
+
+   mutable int nRhsMult, nSetJac, nJacMult, nImpSolve, nImpIter, nImpMult, nImpSet;
+
+public:
+   Vector u0;
+
+   ConductionOperator(FiniteElementSpace &f, double alpha, double kappa, const Vector &u);
+
+   void UpdateStats();
+   void PrintStats(ostream& out);
+
+   void ExtractJacobians(const Vector& x, std::ostream &out, std::ostream &out2);
+
+   BilinearForm& GetKLambda(const Vector& u) const;
+   BilinearForm& GetdKLambda(const Vector& u) const;
+
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
+   virtual Operator& GetGradient(const Vector &k) const;
+
+   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
+
+   virtual ~ConductionOperator();
+};
+
+class ImplicitSolveOperator : public Operator
+{
+private:
+   double dt;
+   const Vector* x;
+   ConductionOperator* oper;
+
+   const SparseMatrix* M;
+   mutable SparseMatrix* Jg;
+
+   mutable Vector u, z;
+   mutable int nMult, nSet;
+
+public:
+   ImplicitSolveOperator(ConductionOperator* oper, const SparseMatrix* M, double dt, const Vector* x);
+
+   int GetnMult() { return nMult; }
+   int GetnSet() { return nSet; }
+   virtual void Mult(const Vector &k, Vector &gk) const;
+   virtual Operator &GetGradient(const Vector &k) const;
+};
+
+class JacobianOperator : public Operator
+{
+private:
+   Operator* J;
+   Operator* M_solver;
+
+   mutable int nMult;
+   mutable Vector z;
+public:
+   JacobianOperator(Operator* J, Operator* M_solver);
+
+   int GetnMult() { return nMult; }
+
+   void ExtractJacobian(const Vector& x, std::ostream &out);
+   virtual void Mult(const Vector &k, Vector &gk) const;
+};
+
+double InitialTemperature(const Vector &x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ref_levels = 2;
+   int order = 2;
+   int ode_solver_type = 8; // Exponential Euler
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
+   bool visualization = true;
+   bool visit = false;
+   int vis_steps = 5;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver:\n\t"
+                  "1  - Forward Euler,\n\t"
+                  "2  - RK2,\n\t"
+                  "3  - RK3 SSP,\n\t"
+                  "4  - RK4,\n\t"
+                  "5  - Backward Euler,\n\t"
+                  "6  - SDIRK 2,\n\t"
+                  "7  - SDIRK 3,\n\t"
+                  "8  - EPIC (exponential euler)\n\t");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&alpha, "-a", "--alpha",
+                  "Alpha coefficient.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "Kappa coefficient offset.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (ode_solver_type < 1 || ode_solver_type > 9)
+   {
+      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      return 3;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
+   //    command-line parameter.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define the vector finite element space representing the current and the
+   //    initial temperature, u_ref.
+   H1_FECollection fe_coll(order, dim);
+   FiniteElementSpace fespace(mesh, &fe_coll);
+
+   int fe_size = fespace.GetTrueVSize();
+   cout << "Number of temperature unknowns: " << fe_size << endl;
+
+   GridFunction u_gf(&fespace);
+
+   // 5. Set the initial conditions for u. All boundaries are considered
+   //    natural.
+   FunctionCoefficient u_0(InitialTemperature);
+   u_gf.ProjectCoefficient(u_0);
+   Vector u;
+   u_gf.GetTrueDofs(u);
+
+   // 6. Initialize the conduction operator and the visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
+
+   u_gf.SetFromTrueDofs(u);
+   {
+      ofstream omesh("ex16.mesh");
+      omesh.precision(precision);
+      mesh->Print(omesh);
+      ofstream osol("ex16-init.gf");
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   VisItDataCollection visit_dc("Example16", mesh);
+   visit_dc.RegisterField("temperature", &u_gf);
+   if (visit)
+   {
+      visit_dc.SetCycle(0);
+      visit_dc.SetTime(0.0);
+      visit_dc.Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         cout << "Unable to connect to GLVis server at "
+              << vishost << ':' << visport << endl;
+         visualization = false;
+         cout << "GLVis visualization disabled.\n";
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *mesh << u_gf;
+         sout << "pause\n";
+         sout << flush;
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      }
+   }
+
+   // 7. Define the ODE solver used for time integration.
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // MFEM explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
+      // EPIC 
+      case 8: ode_solver = new EPI2();break;
+      case 9: ode_solver = new EPIRK4(); break;
+   }
+
+   // Initialize integrators
+   ode_solver->Init(oper);
+
+   // 8. Perform time-integration (looping over the time iterations, ti, with a
+   //    time-step dt).
+   cout << "Integrating the ODE ..." << endl;
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   /*ofstream out_jac_an("jacobian_an.txt");
+   ofstream out_jac_fd("jacobian_fd.txt");
+   oper.ExtractJacobians(u, out_jac_fd, out_jac_an);*/
+
+   bool last_step = false;
+   int ti;
+   for (ti = 1; !last_step; ti++)
+   {
+      double dt_real = min(dt, t_final - t);
+
+      // Note that since we are using the "one-step" mode of the SUNDIALS
+      // solvers, they will, generally, step over the final time and will not
+      // explicitly perform the interpolation to t_final as they do in the
+      // "normal" step mode.
+      ode_solver->Step(u, t, dt_real);
+
+      oper.UpdateStats();
+
+      last_step = (t >= t_final - 1e-8*dt);
+
+      if (last_step || (ti % vis_steps) == 0) {
+         cout << "step " << ti << ", t = " << t << endl;
+
+         u_gf.SetFromTrueDofs(u);
+         if (visualization) {
+            sout << "solution\n" << *mesh << u_gf << flush;
+         }
+
+         if (visit) {
+            visit_dc.SetCycle(ti);
+            visit_dc.SetTime(t);
+            visit_dc.Save();
+         }
+      }
+   }
+   tic_toc.Stop();
+   double comp_time = tic_toc.RealTime();
+   cout << "Done, " << comp_time << "s." << endl;
+
+   // 9. Save the final solution. This output can be viewed later using GLVis:
+   //    "glvis -m ex16.mesh -g ex16-final.gf".
+   {
+      ofstream osol("ex16-final.gf");
+      osol.precision(precision);
+      u_gf.Save(osol);
+
+      ofstream ostats("ex16-stats.txt");
+      ostats << "time " << comp_time << endl;
+      oper.PrintStats(ostats);
+   }
+
+   // 10. Free the used memory.
+   delete ode_solver;
+   delete mesh;
+
+   return 0;
+}
+
+ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al, double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL), dK(NULL), J_K(NULL), jac(NULL), z(height), u0(height),
+     nRhsMult(0), nSetJac(0), nJacMult(0), nImpSolve(0), nImpIter(0), nImpMult(0), nImpSet(0)
+{
+   const double rel_tol = 1e-8;
+
+   M = new BilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble();
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
+
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(rel_tol);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(50);
+   M_solver.SetPrintLevel(0);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(Mmat);
+
+   Jg_solver.SetRelTol(rel_tol);
+   Jg_solver.SetAbsTol(0.0);
+   Jg_solver.SetMaxIter(50);
+   Jg_solver.SetPrintLevel(0);
+   Jg_solver.SetPreconditioner(Jg_prec);
+
+   newton_solver.SetMaxIter(10);
+   newton_solver.SetRelTol(rel_tol);
+   newton_solver.SetPrintLevel(-1);
+   newton_solver.SetSolver(Jg_solver);
+   newton_solver.SetMaxIter(100);
+   newton_solver.iterative_mode = false;
+
+   alpha = al;
+   kappa = kap;
+}
+
+void ConductionOperator::UpdateStats()
+{
+   if (jac)
+   {
+      nJacMult += jac->GetnMult();
+   }
+}
+
+void ConductionOperator::PrintStats(ostream &out)
+{
+   out  << "nRhsMult " << nRhsMult << endl
+        << "nSetJac " << nSetJac << endl
+        << "nJacMult " << nJacMult  << endl
+        << "nImplicitSolve " << nImpSolve  << endl
+        << "nImplicitIter " << nImpIter << endl
+        << "nImplicitMult " << nImpMult << endl
+        << "nImplicitSet " << nImpSet << endl;
+}
+
+BilinearForm& ConductionOperator::GetKLambda(const Vector &u) const
+{
+   GridFunction conductivity_gf(&fespace);
+   conductivity_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < conductivity_gf.Size(); i++)
+   {
+      conductivity_gf(i) = kappa + alpha*conductivity_gf(i);
+   }
+
+   GridFunctionCoefficient conductivity_coeff(&conductivity_gf);
+
+   delete K;
+   K = new BilinearForm(&fespace);
+   K->AddDomainIntegrator(new DiffusionIntegrator(conductivity_coeff));
+   K->Assemble();
+
+   return *K;
+}
+
+BilinearForm& ConductionOperator::GetdKLambda(const Vector &u) const
+{
+   GridFunction conductivity_gf(&fespace);
+   conductivity_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < conductivity_gf.Size(); i++)
+   {
+      conductivity_gf(i) = kappa + alpha*conductivity_gf(i);
+   }
+
+   // Define diffusion form with conductivity = kappa(u0)
+   GridFunctionCoefficient conductivity_coeff(&conductivity_gf);
+
+   // Define advection form with velocity = grad kappa(u0)
+   GridFunction neg_cond_gf(conductivity_gf);
+   neg_cond_gf.Neg();
+   GradientGridFunctionCoefficient velocity_coeff(&neg_cond_gf);
+
+   delete dK;
+   dK = new BilinearForm(&fespace);
+
+   dK->AddDomainIntegrator(new DiffusionIntegrator(conductivity_coeff));
+   dK->AddDomainIntegrator(new MixedScalarWeakDivergenceIntegrator(velocity_coeff));
+   dK->Assemble();
+
+   return *dK;
+}
+
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   GetKLambda(u);
+   K->Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+   nRhsMult++;
+}
+
+void ConductionOperator::ImplicitSolve(const double dt, const Vector &x, Vector &k)
+{
+   ImplicitSolveOperator imp_oper(this, &this->Mmat, dt, &x);
+   newton_solver.SetOperator(imp_oper);
+
+   Vector zero; // empty vector is interpreted as zero r.h.s. by NewtonSolver
+   newton_solver.Mult(zero, k);
+   MFEM_VERIFY(newton_solver.GetConverged(), "Newton solver did not converge.");
+
+   nImpSolve++;
+   nImpMult += imp_oper.GetnMult();
+   nImpSet  += imp_oper.GetnSet();
+   nImpIter += newton_solver.GetNumIterations();
+}
+
+Operator &ConductionOperator::GetGradient(const Vector &u) const
+{
+   delete jac;
+   GetdKLambda(u);
+   jac = new JacobianOperator(dK, &M_solver);
+
+   nSetJac++;
+
+   return *jac;
+}
+
+ConductionOperator::~ConductionOperator()
+{
+   delete M;
+   delete K;
+   delete dK;
+   delete J_K;
+   delete jac;
+}
+
+ImplicitSolveOperator::ImplicitSolveOperator(ConductionOperator *oper_, const SparseMatrix* M_, double dt_, const Vector* x_):
+   Operator(oper_->Height()), oper(oper_), M(M_), dt(dt_), x(x_), u(height), z(height), Jg(NULL), nMult(0), nSet(0)
+{ }
+
+
+void ImplicitSolveOperator::Mult(const Vector& y, Vector& gy) const
+{
+   // Compute gy = g(y) = My + dt K(lambda(u)) u
+   // with u = x + dt y
+   add(*x, dt, y, u);
+   BilinearForm& K = oper->GetKLambda(u);
+   K.Mult(u, gy);
+
+   M->AddMult(y, gy);
+
+   nMult++;
+}
+
+Operator& ImplicitSolveOperator::GetGradient(const Vector &k) const
+{
+   add(*x, dt, k, u);
+
+   BilinearForm& dK = oper->GetdKLambda(u);
+   Array<int> ess_tdof_list;
+   SparseMatrix dK_mat;
+   dK.FormSystemMatrix(ess_tdof_list, dK_mat);
+
+   delete Jg;
+   Jg = Add(1.0, *M, dt, dK_mat);
+
+   nSet++;
+   return *Jg;
+}
+
+JacobianOperator::JacobianOperator(Operator* J_, Operator* M_solver_):
+   Operator(M_solver_->Height()), J(J_), M_solver(M_solver_), z(height), nMult(0)
+{ }
+
+void JacobianOperator::Mult(const Vector &v, Vector &Jv) const
+{
+   Vector temp(v);
+   J->Mult(v, z);
+   z.Neg(); // z = -z
+   M_solver->Mult(z, Jv);
+   nMult++;
+}
+
+
+void ConductionOperator::ExtractJacobians(const Vector& x, std::ostream &out, std::ostream &out2)
+{
+   int n = x.Size();
+
+   Vector e(n);
+   e = 0.0;
+
+   double eps = 1e-8;
+   Vector fx(n), fx_eps(n), x_eps(n);
+   Mult(x, fx);
+
+   DenseMatrix J(n);
+
+   for (int i = 0; i < n; i++)
+   {
+      e[i] = 1.0;
+      add(x, eps, e, x_eps);
+      Mult(x_eps, fx_eps);
+      fx_eps -= fx;
+      fx_eps /= eps;
+      J.SetCol(i, fx_eps);
+      e[i] = 0.0;
+   }
+
+   J.PrintMatlab(out);
+   GetGradient(x);
+   jac->ExtractJacobian(x, out2);
+}
+
+void JacobianOperator::ExtractJacobian(const Vector& x, std::ostream &out)
+{
+   int n = z.Size();
+
+   Vector e(n);
+   e= 0.0;
+
+   Vector J_i(n);
+   DenseMatrix J(n);
+
+   for (int i = 0; i < n; i++)
+   {
+      e[i] = 1.0;
+      Mult(e, J_i);
+      J.SetCol(i, J_i);
+      e[i] = 0.0;
+   }
+
+   J.PrintMatlab(out);
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5) { return 2.0; }
+   else                  { return 1.0; }
+}
@@ -0,0 +1,494 @@
+//                         MFEM Example 16 - Parallel Version
+//                             SUNDIALS Modification
+//
+// Compile with: make ex16p
+//
+// Sample runs:
+//     mpirun -np 4 ex16p
+//     mpirun -np 4 ex16p -m ../../data/inline-tri.mesh
+//     mpirun -np 4 ex16p -m ../../data/disc-nurbs.mesh -tf 2
+//     mpirun -np 4 ex16p -s 12 -a 0.0 -k 1.0
+//     mpirun -np 4 ex16p -s 8 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
+//     mpirun -np 8 ex16p -s 9 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
+//     mpirun -np 4 ex16p -s 10 -dt 2.0e-4 -tf 4.0e-2
+//     mpirun -np 16 ex16p -m ../../data/fichera-q2.mesh
+//     mpirun -np 16 ex16p -m ../../data/escher-p2.mesh
+//     mpirun -np 8 ex16p -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
+//     mpirun -np 4 ex16p -m ../../data/amr-quad.mesh -o 4 -rs 0 -rp 0
+//     mpirun -np 4 ex16p -m ../../data/amr-hex.mesh -o 2 -rs 0 -rp 0
+//
+// Description:  This example solves a time dependent nonlinear heat equation
+//               problem of the form du/dt = C(u), with a non-linear diffusion
+//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
+//
+//               We recommend viewing examples 2, 9 and 10 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/** After spatial discretization, the conduction model can be written as:
+ *
+ *     du/dt = M^{-1}(-Ku)
+ *
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
+ *
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
+ */
+class ConductionOperator : public TimeDependentOperator
+{
+protected:
+   ParFiniteElementSpace &fespace;
+   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
+
+   ParBilinearForm *M;
+   ParBilinearForm *K;
+
+   HypreParMatrix Mmat;
+   HypreParMatrix Kmat;
+   HypreParMatrix *T; // T = M + dt K
+   double current_dt;
+
+   CGSolver M_solver;    // Krylov solver for inverting the mass matrix M
+   HypreSmoother M_prec; // Preconditioner for the mass matrix M
+
+   CGSolver T_solver;    // Implicit solver for T = M + dt K
+   HypreSmoother T_prec; // Preconditioner for the implicit solver
+
+   double alpha, kappa;
+
+   mutable Vector z; // auxiliary vector
+
+public:
+   ConductionOperator(ParFiniteElementSpace &f, double alpha, double kappa,
+                      const Vector &u);
+
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
+
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
+
+   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+                                int jok, int *jcur, double gamma);
+
+   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
+
+   virtual ~ConductionOperator();
+};
+
+double InitialTemperature(const Vector &x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 2;
+   int par_ref_levels = 1;
+   int order = 2;
+   int ode_solver_type = 8; // Exponential Euler 
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
+   bool visualization = true;
+   bool visit = false;
+   int vis_steps = 5;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver:\n\t"
+                  "1  - Forward Euler,\n\t"
+                  "2  - RK2,\n\t"
+                  "3  - RK3 SSP,\n\t"
+                  "4  - RK4,\n\t"
+                  "5  - Backward Euler,\n\t"
+                  "6  - SDIRK 2,\n\t"
+                  "7  - SDIRK 3,\n\t"
+                  "8  - Exponential Euler,\n\t");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&alpha, "-a", "--alpha",
+                  "Alpha coefficient.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "Kappa coefficient offset.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      MPI_Finalize();
+      return 1;
+   }
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // check for vaild ODE solver option
+   if (ode_solver_type < 1 || ode_solver_type > 8)
+   {
+      if (myid == 0)
+      {
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      }
+      MPI_Finalize();
+      return 1;
+   }
+
+   // 3. Read the serial mesh from the given mesh file on all processors. We can
+   //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
+   //    with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the mesh in serial to increase the resolution. In this example
+   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
+   //    a command-line parameter.
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define the vector finite element space representing the current and the
+   //    initial temperature, u_ref.
+   H1_FECollection fe_coll(order, dim);
+   ParFiniteElementSpace fespace(pmesh, &fe_coll);
+
+   int fe_size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of temperature unknowns: " << fe_size << endl;
+   }
+
+   ParGridFunction u_gf(&fespace);
+
+   // 7. Set the initial conditions for u. All boundaries are considered
+   //    natural.
+   FunctionCoefficient u_0(InitialTemperature);
+   u_gf.ProjectCoefficient(u_0);
+   Vector u;
+   u_gf.GetTrueDofs(u);
+
+   // 8. Initialize the conduction operator and the VisIt visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
+
+   u_gf.SetFromTrueDofs(u);
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "ex16-mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "ex16-init." << setfill('0') << setw(6) << myid;
+      ofstream omesh(mesh_name.str().c_str());
+      omesh.precision(precision);
+      pmesh->Print(omesh);
+      ofstream osol(sol_name.str().c_str());
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   VisItDataCollection visit_dc("Example16-Parallel", pmesh);
+   visit_dc.RegisterField("temperature", &u_gf);
+   if (visit)
+   {
+      visit_dc.SetCycle(0);
+      visit_dc.SetTime(0.0);
+      visit_dc.Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      sout << "parallel " << num_procs << " " << myid << endl;
+      int good = sout.good(), all_good;
+      MPI_Allreduce(&good, &all_good, 1, MPI_INT, MPI_MIN, pmesh->GetComm());
+      if (!all_good)
+      {
+         sout.close();
+         visualization = false;
+         if (myid == 0)
+         {
+            cout << "Unable to connect to GLVis server at "
+                 << vishost << ':' << visport << endl;
+            cout << "GLVis visualization disabled.\n";
+         }
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *pmesh << u_gf;
+         sout << "pause\n";
+         sout << flush;
+         if (myid == 0)
+         {
+            cout << "GLVis visualization paused."
+                 << " Press space (in the GLVis window) to resume it.\n";
+         }
+      }
+   }
+
+   // 9. Define the ODE solver used for time integration.
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   EPICSolver *epic_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // MFEM explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
+      // EPIC
+      case 8:
+         epic_solver = new EPICSolver();
+	 epic_solver->Init(oper);
+	 ode_solver = epic_solver;
+	 break;
+   }
+
+   // Initialize MFEM integrators
+   ode_solver->Init(oper);
+
+   // 10. Perform time-integration (looping over the time iterations, ti, with a
+   //     time-step dt).
+   if (myid == 0)
+   {
+      cout << "Integrating the ODE ..." << endl;
+   }
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   bool last_step = false;
+   for (int ti = 1; !last_step; ti++)
+   {
+      double dt_real = min(dt, t_final - t);
+
+      // Note that since we are using the "one-step" mode of the SUNDIALS
+      // solvers, they will, generally, step over the final time and will not
+      // explicitly perform the interpolation to t_final as they do in the
+      // "normal" step mode.
+
+      ode_solver->Step(u, t, dt_real);
+
+      last_step = (t >= t_final - 1e-8*dt);
+
+      if (last_step || (ti % vis_steps) == 0)
+      {
+         if (myid == 0)
+         {
+            cout << "step " << ti << ", t = " << t << endl;
+         }
+
+         u_gf.SetFromTrueDofs(u);
+         if (visualization)
+         {
+            sout << "parallel " << num_procs << " " << myid << "\n";
+            sout << "solution\n" << *pmesh << u_gf << flush;
+         }
+
+         if (visit)
+         {
+            visit_dc.SetCycle(ti);
+            visit_dc.SetTime(t);
+            visit_dc.Save();
+         }
+      }
+      oper.SetParameters(u);
+   }
+   tic_toc.Stop();
+   if (myid == 0)
+   {
+      cout << "Done, " << tic_toc.RealTime() << "s." << endl;
+   }
+
+   // 11. Save the final solution in parallel. This output can be viewed later
+   //     using GLVis: "glvis -np <np> -m ex16-mesh -g ex16-final".
+   {
+      ostringstream sol_name;
+      sol_name << "ex16-final." << setfill('0') << setw(6) << myid;
+      ofstream osol(sol_name.str().c_str());
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   // 12. Free the used memory.
+   delete ode_solver;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
+     T(NULL),
+     M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
+{
+   const double rel_tol = 1e-8;
+
+   M = new ParBilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble(0); // keep sparsity pattern of M and K the same
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
+
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(rel_tol);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(100);
+   M_solver.SetPrintLevel(0);
+   M_prec.SetType(HypreSmoother::Jacobi);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(Mmat);
+
+   alpha = al;
+   kappa = kap;
+
+   T_solver.iterative_mode = false;
+   T_solver.SetRelTol(rel_tol);
+   T_solver.SetAbsTol(0.0);
+   T_solver.SetMaxIter(100);
+   T_solver.SetPrintLevel(0);
+   T_solver.SetPreconditioner(T_prec);
+
+   SetParameters(u);
+}
+
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Kmat.Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, dt, Kmat);
+   T_solver.SetOperator(*T);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
+}
+
+int ConductionOperator::SUNImplicitSetup(const Vector &x,
+                                         const Vector &fx, int jok, int *jcur,
+                                         double gamma)
+{
+   // Setup the ODE Jacobian T = M + gamma K.
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, gamma, Kmat);
+   T_solver.SetOperator(*T);
+   *jcur = 1;
+   return (0);
+}
+
+int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
+{
+   // Solve the system A x = z => (M - gamma K) x = M b.
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, x);
+   return (0);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
+{
+   ParGridFunction u_alpha_gf(&fespace);
+   u_alpha_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < u_alpha_gf.Size(); i++)
+   {
+      u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
+   }
+
+   delete K;
+   K = new ParBilinearForm(&fespace);
+
+   GridFunctionCoefficient u_coeff(&u_alpha_gf);
+
+   K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
+   K->Assemble(0); // keep sparsity pattern of M and K the same
+   K->FormSystemMatrix(ess_tdof_list, Kmat);
+}
+
+ConductionOperator::~ConductionOperator()
+{
+   delete T;
+   delete M;
+   delete K;
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5)
+   {
+      return 2.0;
+   }
+   else
+   {
+      return 1.0;
+   }
+}
@@ -0,0 +1,76 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/epic/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = ex16
+PAR_EXAMPLES = ex16p
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_EPIC),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with EPIC)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+SERIAL_NAME := Serial EPIC example
+PARALLEL_NAME := Parallel EPIC example
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
+%-test-seq: %
+	@$(call mfem-test,$<,, $(SERIAL_NAME))
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f deformed.* velocity.* elastic_energy.*
+	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.* Example16*
@@ -118,6 +118,7 @@ int main(int argc, char *argv[])
   {
      pmesh->UniformRefinement();
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -24,10 +24,7 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. In this example,
-//               the diffusion operator is linearized by evaluating with the
-//               lagged solution from the previous timestep, so there is only
-//               a linear solve.
+//               high-order implicit (SDIRK) time integration.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -329,8 +326,8 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-Ku
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -341,7 +338,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   // for du_dt
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -24,11 +24,8 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. In this example,
-//               the diffusion operator is linearized by evaluating with the
-//               lagged solution from the previous timestep, so there is only
-//               a linear solve. Optional saving with ADIOS2
-//               (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration. Optional saving
+//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -423,8 +420,8 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-Ku
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -435,7 +432,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   // for du_dt
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -113,6 +113,7 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
+   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use Nedelec or
   //    Raviart-Thomas finite elements of the specified order.
@@ -141,6 +141,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use Nedelec or Raviart-Thomas finite elements of the specified order.
@@ -277,8 +277,10 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 6. Set element attributes in order to distinguish elements in the
-   //    PML region
+   // 6. Reorient mesh in case of a tet mesh
+   mesh->ReorientTetMesh();
+
+   // Set element attributes in order to distinguish elements in the PML region
   pml->SetAttributes(mesh);

   // 7. Define a finite element space on the mesh. Here we use the Nedelec
@@ -316,6 +316,9 @@ int main(int argc, char *argv[])
      }
   }

+   // 7a. Reorient mesh in case of a tet mesh
+   pmesh->ReorientTetMesh();
+
   // 8. Set element attributes in order to distinguish elements in the PML
   pml->SetAttributes(pmesh);

@@ -113,6 +113,7 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
+   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
@@ -139,7 +139,9 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -149,6 +151,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -135,7 +135,9 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -145,6 +147,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -106,6 +106,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 6. Define the trial, interfacial (trace) and test DPG spaces:
   //    - The trial space, x0_space, contains the non-interfacial unknowns and
@@ -45,6 +45,9 @@ endif
 ifeq ($(MFEM_USE_HIOP),YES)
   SUBDIRS += hiop
 endif
+ifeq ($(MFEM_USE_EPIC),YES)
+   SUBDIRS += epic
+endif
 ifeq ($(MFEM_USE_PETSC),YES)
   SUBDIRS += petsc
 endif
@@ -121,7 +121,9 @@ int main(int argc, char *argv[])

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -131,6 +133,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -122,7 +122,9 @@ int main(int argc, char *argv[])

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -132,6 +134,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -39,7 +39,6 @@ set(SRCS
  complex_fem.cpp
  convergence.cpp
  datacollection.cpp
-  doftrans.cpp
  eltrans.cpp
  estimators.cpp
  fe.cpp
@@ -106,7 +105,6 @@ set(SRCS
  tmop/tmop_pa_w3.cpp
  tmop/tmop_pa_w3_c0.cpp
  tmop_tools.cpp
-  tmop_amr.cpp
  gslib.cpp
  transfer.cpp
  lor.cpp
@@ -120,7 +118,6 @@ set(HDRS
  complex_fem.hpp
  convergence.hpp
  datacollection.hpp
-  doftrans.hpp
  eltrans.hpp
  estimators.hpp
  fe.hpp
@@ -167,7 +164,6 @@ set(HDRS
  tmop.hpp
  tmop/tmop_pa.hpp
  tmop_tools.hpp
-  tmop_amr.hpp
  gslib.hpp
  transfer.hpp
  lor.hpp
@@ -391,7 +391,6 @@ void BilinearForm::Assemble(int skip_zeros)
   }

   ElementTransformation *eltrans;
-   DofTransformation * doftrans;
   Mesh *mesh = fes -> GetMesh();
   DenseMatrix elmat, *elmat_p;

@@ -425,7 +424,7 @@ void BilinearForm::Assemble(int skip_zeros)
      for (int i = 0; i < fes -> GetNE(); i++)
      {
         int elem_attr = fes->GetMesh()->GetAttribute(i);
-         doftrans = fes->GetElementVDofs(i, vdofs);
+         fes->GetElementVDofs(i, vdofs);
         if (element_matrices)
         {
            elmat_p = &(*element_matrices)(i);
@@ -459,11 +458,6 @@ void BilinearForm::Assemble(int skip_zeros)
            {
               elmat_p = &elmat;
            }
-            if (doftrans)
-            {
-               doftrans->TransformDual(elmat);
-            }
-            elmat_p = &elmat;
         }
         if (static_cond)
         {
@@ -509,7 +503,7 @@ void BilinearForm::Assemble(int skip_zeros)
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

         const FiniteElement &be = *fes->GetBE(i);
-         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
+         fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
         int k = 0;
         for (; k < boundary_integs.Size(); k++)
@@ -529,22 +523,17 @@ void BilinearForm::Assemble(int skip_zeros)
            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
            elmat += elemmat;
         }
-         if (doftrans)
-         {
-            doftrans->TransformDual(elmat);
-         }
-         elmat_p = &elmat;
         if (!static_cond)
         {
-            mat->AddSubMatrix(vdofs, vdofs, *elmat_p, skip_zeros);
+            mat->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
            if (hybridization)
            {
-               hybridization->AssembleBdrMatrix(i, *elmat_p);
+               hybridization->AssembleBdrMatrix(i, elmat);
            }
         }
         else
         {
-            static_cond->AssembleBdrMatrix(i, *elmat_p);
+            static_cond->AssembleBdrMatrix(i, elmat);
         }
      }
   }
@@ -1329,10 +1318,9 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      return;
   }

+   Array<int> tr_vdofs, te_vdofs;
   ElementTransformation *eltrans;
-   DofTransformation * dom_dof_trans;
-   DofTransformation * ran_dof_trans;
-   DenseMatrix elmat;
+   DenseMatrix elemmat;

   Mesh *mesh = test_fes -> GetMesh();

@@ -1345,24 +1333,16 @@ void MixedBilinearForm::Assemble (int skip_zeros)
   {
      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
-         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
-         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
+         trial_fes -> GetElementVDofs (i, tr_vdofs);
+         test_fes  -> GetElementVDofs (i, te_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
-
-         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-         elmat = 0.0;
         for (int k = 0; k < domain_integs.Size(); k++)
         {
            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
                                                        *test_fes  -> GetFE(i),
                                                        *eltrans, elemmat);
-            elmat += elemmat;
+            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
-         }
-         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

@@ -1394,12 +1374,9 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

-         dom_dof_trans = trial_fes -> GetBdrElementVDofs (i, trial_vdofs);
-         ran_dof_trans = test_fes  -> GetBdrElementVDofs (i, test_vdofs);
+         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
+         test_fes  -> GetBdrElementVDofs (i, te_vdofs);
         eltrans = test_fes -> GetBdrElementTransformation (i);
-
-         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-         elmat = 0.0;
         for (int k = 0; k < boundary_integs.Size(); k++)
         {
            if (boundary_integs_marker[k] &&
@@ -1408,34 +1385,29 @@ void MixedBilinearForm::Assemble (int skip_zeros)
            boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
                                                        *test_fes  -> GetBE(i),
                                                        *eltrans, elemmat);
-            elmat += elemmat;
+            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
-         }
-         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

   if (trace_face_integs.Size())
   {
      FaceElementTransformations *ftr;
-      Array<int> test_vdofs2;
+      Array<int> te_vdofs2;
      const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;

      int nfaces = mesh->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
         ftr = mesh->GetFaceElementTransformations(i);
-         trial_fes->GetFaceVDofs(i, trial_vdofs);
-         test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
+         trial_fes->GetFaceVDofs(i, tr_vdofs);
+         test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
         trial_face_fe = trial_fes->GetFaceElement(i);
         test_fe1 = test_fes->GetFE(ftr->Elem1No);
         if (ftr->Elem2No >= 0)
         {
-            test_fes->GetElementVDofs(ftr->Elem2No, test_vdofs2);
-            test_vdofs.Append(test_vdofs2);
+            test_fes->GetElementVDofs(ftr->Elem2No, te_vdofs2);
+            te_vdofs.Append(te_vdofs2);
            test_fe2 = test_fes->GetFE(ftr->Elem2No);
         }
         else
@@ -1449,7 +1421,7 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         {
            trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
                                                     *test_fe2, *ftr, elemmat);
-            mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
+            mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
      }
   }
@@ -1489,8 +1461,8 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         ftr = mesh->GetBdrFaceTransformations(i);
         if (ftr)
         {
-            trial_fes->GetFaceVDofs(ftr->ElementNo, trial_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
+            trial_fes->GetFaceVDofs(ftr->ElementNo, tr_vdofs);
+            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
            trial_face_fe = trial_fes->GetFaceElement(ftr->ElementNo);
            test_fe1 = test_fes->GetFE(ftr->Elem1No);
            // The test_fe2 object is really a dummy and not used on the
@@ -1507,7 +1479,7 @@ void MixedBilinearForm::Assemble (int skip_zeros)
                                                                 *test_fe1,
                                                                 *test_fe2,
                                                                 *ftr, elemmat);
-               mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
+               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
            }
         }
      }
@@ -1869,8 +1841,6 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)

   Array<int> dom_vdofs, ran_vdofs;
   ElementTransformation *T;
-   DofTransformation * dom_dof_trans;
-   DofTransformation * ran_dof_trans;
   const FiniteElement *dom_fe, *ran_fe;
   DenseMatrix totelmat, elmat;

@@ -1883,8 +1853,8 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
   {
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
-         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
-         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
+         trial_fes->GetElementVDofs(i, dom_vdofs);
+         test_fes->GetElementVDofs(i, ran_vdofs);
         T = test_fes->GetElementTransformation(i);
         dom_fe = trial_fes->GetFE(i);
         ran_fe = test_fes->GetFE(i);
@@ -1897,10 +1867,6 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
                                                     elmat);
            totelmat += elmat;
         }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
-         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
      }
   }
@@ -1,358 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-
-namespace mfem
-{
-
-void DofTransformation::TransformPrimal(Vector &v) const
-{
-   TransformPrimal(v.GetData());
-}
-
-void DofTransformation::TransformPrimalCols(DenseMatrix &V) const
-{
-   for (int c=0; c<V.Width(); c++)
-   {
-      TransformPrimal(V.GetColumn(c));
-   }
-}
-
-void DofTransformation::TransformDual(Vector &v) const
-{
-   TransformDual(v.GetData());
-}
-
-void DofTransformation::TransformDual(DenseMatrix &V) const
-{
-   TransformDualCols(V);
-   TransformDualRows(V);
-}
-
-void DofTransformation::TransformDualRows(DenseMatrix &V) const
-{
-   Vector row;
-   for (int r=0; r<V.Height(); r++)
-   {
-      V.GetRow(r, row);
-      TransformDual(row);
-      V.SetRow(r, row);
-   }
-}
-
-void DofTransformation::TransformDualCols(DenseMatrix &V) const
-{
-   for (int c=0; c<V.Width(); c++)
-   {
-      TransformDual(V.GetColumn(c));
-   }
-}
-
-void DofTransformation::InvTransformPrimal(Vector &v) const
-{
-   InvTransformPrimal(v.GetData());
-}
-
-void TransformPrimal(const DofTransformation *ran_dof_trans,
-                     const DofTransformation *dom_dof_trans,
-                     DenseMatrix &elmat)
-{
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformPrimalCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
-   {
-      ran_dof_trans->TransformPrimalCols(elmat);
-   }
-   else if (dom_dof_trans)
-   {
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
-}
-
-void TransformDual(const DofTransformation *ran_dof_trans,
-                   const DofTransformation *dom_dof_trans,
-                   DenseMatrix &elmat)
-{
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformDualCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
-   {
-      ran_dof_trans->TransformDualCols(elmat);
-   }
-   else if (dom_dof_trans)
-   {
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
-}
-
-void VDofTransformation::TransformPrimal(double *v) const
-{
-   int size = doftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         doftrans_->TransformPrimal(&v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         doftrans_->TransformPrimal(vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void VDofTransformation::InvTransformPrimal(double *v) const
-{
-   int size = doftrans_->Height();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         doftrans_->InvTransformPrimal(&v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         doftrans_->InvTransformPrimal(vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void VDofTransformation::TransformDual(double *v) const
-{
-   int size = doftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         doftrans_->TransformDual(&v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         doftrans_->TransformDual(vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-const double ND_DofTransformation::T_data[24] =
-{
-   1.0,  0.0,  0.0,  1.0,
-   -1.0, -1.0,  0.0,  1.0,
-   0.0,  1.0, -1.0, -1.0,
-   1.0,  0.0, -1.0, -1.0,
-   -1.0, -1.0,  1.0,  0.0,
-   0.0,  1.0,  1.0,  0.0
-};
-
-const DenseTensor ND_DofTransformation
-::T(const_cast<double*>(ND_DofTransformation::T_data), 2, 2, 6);
-
-const double ND_DofTransformation::TInv_data[24] =
-{
-   1.0,  0.0,  0.0,  1.0,
-   -1.0, -1.0,  0.0,  1.0,
-   -1.0, -1.0,  1.0,  0.0,
-   1.0,  0.0, -1.0, -1.0,
-   0.0,  1.0, -1.0, -1.0,
-   0.0,  1.0,  1.0,  0.0
-};
-
-const DenseTensor ND_DofTransformation
-::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
-
-ND_DofTransformation::ND_DofTransformation(int size, int p)
-   : DofTransformation(size),
-     order(p)
-{
-}
-
-ND_TriDofTransformation::ND_TriDofTransformation(int p)
-   : ND_DofTransformation(p*(p + 2), p)
-{
-}
-
-void ND_TriDofTransformation::TransformPrimal(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<1; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TriDofTransformation::InvTransformPrimal(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<1; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TriDofTransformation::TransformDual(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<1; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-ND_TetDofTransformation::ND_TetDofTransformation(int p)
-   : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
-{
-}
-
-void ND_TetDofTransformation::TransformPrimal(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::InvTransformPrimal(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::TransformDual(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-} // namespace mfem
@@ -1,277 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_DOFTRANSFORM
-#define MFEM_DOFTRANSFORM
-
-#include "../config/config.hpp"
-#include "../linalg/linalg.hpp"
-#include "intrules.hpp"
-#include "fe.hpp"
-
-namespace mfem
-{
-
-/** The DofTransformation class is an abstract base class for a family of
-    transformations that map local degrees of freedom (DoFs), contained within
-    individual elements, to global degrees of freedom, stored within
-    GridFunction objects. These transformations are necessary to ensure that
-    basis functions in neighboring elements align correctly. Closely related but
-    complementary transformations are required for the entries stored in
-    LinearForm and BilinearForm objects. The DofTransformation class is designed
-    to apply the action of both of these types of DoF transformations.
-
-    Let the "primal transformation" be given by the operator T. This means that
-    given a local element vector v the data that must be placed into a
-    GridFunction object is v_t = T * v.
-
-    We also need the inverse of the primal transformation T^{-1} so that we can
-    recover the local element vector from data read out of a GridFunction
-    e.g. v = T^{-1} * v_t.
-
-    We need to preserve the action of our linear forms applied to primal
-    vectors. In other words, if f is the local vector computed by a linear
-    form then f * v = f_t * v_t (where "*" represents an inner product of
-    vectors). This requires that f_t = T^{-T} * f i.e. the "dual transform" is
-    given by the transpose of the inverse of the primal transformation.
-
-    For bilinear forms we require that v^T * A * v = v_t^T * A_t * v_t. This
-    implies that A_t = T^{-T} * A * T^{-1}. This can be accomplished by
-    performing dual transformations of the rows and columns of the matrix A.
-
-    For discrete linear operators the range must be modified with the primal
-    transformation rather than the dual transformation because the result is a
-    primal vector rather than a dual vector. This leads to the transformation
-    D_t = T * D * T^{-1}. This can be accomplished by using a primal
-    transformation on the columns of D and a dual transformation on its rows.
-*/
-class DofTransformation
-{
-protected:
-   int size_;
-
-   Array<int> Fo;
-
-   DofTransformation(int size)
-      : size_(size) {}
-
-public:
-
-   inline int Size() const { return size_; }
-   inline int Height() const { return size_; }
-   inline int NumRows() const { return size_; }
-   inline int Width() const { return size_; }
-   inline int NumCols() const { return size_; }
-
-   /** @brief Configure the transformation using face orientations for the
-       current element. */
-   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
-   inline void SetFaceOrientations(const Array<int> & face_orientation)
-   { Fo = face_orientation; }
-
-   inline const Array<int> & GetFaceOrientations() const { return Fo; }
-
-   /** Transform local DoFs to align with the global DoFs. For example, this
-       transformation can be used to map the local vector computed by
-       FiniteElement::Project() to the transformed vector stored within a
-       GridFunction object. */
-   virtual void TransformPrimal(double *v) const = 0;
-   virtual void TransformPrimal(Vector &v) const;
-
-   /// Transform groups of DoFs stored as dense matrices
-   virtual void TransformPrimalCols(DenseMatrix &V) const;
-
-   /** Inverse transform local DoFs. Used to transform DoFs from a global vector
-       back to their element-local form. For example, this must be used to
-       transform the vector obtained using GridFunction::GetSubVector before it
-       can be used to compute a local interpolation.
-   */
-   virtual void InvTransformPrimal(double *v) const = 0;
-   virtual void InvTransformPrimal(Vector &v) const;
-
-   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
-       into a LinearForm object. */
-   virtual void TransformDual(double *v) const = 0;
-   virtual void TransformDual(Vector &v) const;
-
-   /** Transform a matrix of dual DoFs entries as computed by a
-       BilinearFormIntegrator before summing into a BilinearForm object. */
-   virtual void TransformDual(DenseMatrix &V) const;
-
-   /// Transform groups of dual DoFs stored as dense matrices
-   virtual void TransformDualRows(DenseMatrix &V) const;
-   virtual void TransformDualCols(DenseMatrix &V) const;
-
-   virtual ~DofTransformation() {}
-};
-
-/** Transform a matrix of DoFs entries from different finite element spaces as
-    computed by a DiscreteInterpolator before copying into a
-    DiscreteLinearOperator.
-*/
-void TransformPrimal(const DofTransformation *ran_dof_trans,
-                     const DofTransformation *dom_dof_trans,
-                     DenseMatrix &elmat);
-
-/** Transform a matrix of dual DoFs entries from different finite element spaces
-    as computed by a BilinearFormIntegrator before summing into a
-    MixedBilinearForm object.
-*/
-void TransformDual(const DofTransformation *ran_dof_trans,
-                   const DofTransformation *dom_dof_trans,
-                   DenseMatrix &elmat);
-
-/** The VDofTransformation class implements a nested transformation where an
-    arbitrary DofTransformation is replicated with a vdim >= 1.
-*/
-class VDofTransformation : public DofTransformation
-{
-private:
-   int vdim_;
-   int ordering_;
-   DofTransformation * doftrans_;
-
-public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   VDofTransformation(int vdim = 1, int ordering = 0)
-      : DofTransformation(0),
-        vdim_(vdim), ordering_(ordering),
-        doftrans_(NULL) {}
-
-   /// Constructor with a known DofTransformation
-   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
-                      int ordering = 0)
-      : DofTransformation(vdim * doftrans.Size()),
-        vdim_(vdim), ordering_(ordering),
-        doftrans_(&doftrans) {}
-
-   /// Set or change the vdim parameter
-   inline void SetVDim(int vdim)
-   {
-      vdim_ = vdim;
-      if (doftrans_)
-      {
-         size_ = vdim_ * doftrans_->Size();
-      }
-   }
-
-   /// Return the current vdim value
-   inline int GetVDim() const { return vdim_; }
-
-   /// Set or change the nested DofTransformation object
-   inline void SetDofTransformation(DofTransformation & doftrans)
-   {
-      size_ = vdim_ * doftrans.Size();
-      doftrans_ = &doftrans;
-   }
-
-   /// Return the nested DofTransformation object
-   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
-
-   inline void SetFaceOrientation(const Array<int> & face_orientation)
-   { Fo = face_orientation; doftrans_->SetFaceOrientations(face_orientation); }
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-   void InvTransformPrimal(double *v) const;
-   void TransformDual(double *v) const;
-};
-
-/** Abstract base class for high-order Nedelec spaces on elements with
-    triangular faces.
-
-    The Nedelec DoFs on the interior of triangular faces come in pairs which
-    share an interpolation point but have different vector directions. These
-    directions depend on the orientation of the face and can therefore differ in
-    neighboring elements. The mapping required to transform these DoFs can be
-    implemented as series of 2x2 linear transformations. The raw data for these
-    linear transformations is stored in the T_data and TInv_data arrays and can
-    be accessed as DenseMatrices using the GetFaceTransform() and
-    GetFaceInverseTransform() methods.
-*/
-class ND_DofTransformation : public DofTransformation
-{
-protected:
-   static const double T_data[24];
-   static const double TInv_data[24];
-   static const DenseTensor T, TInv;
-   int order;
-
-   ND_DofTransformation(int size, int order);
-
-public:
-   // Return the 2x2 transformation operator for the given face orientation
-   static const DenseMatrix & GetFaceTransform(int ori) { return T(ori); }
-
-   // Return the 2x2 inverse transformation operator
-   static const DenseMatrix & GetFaceInverseTransform(int ori)
-   { return TInv(ori); }
-};
-
-/// DoF transformation implementation for the Nedelec basis on triangles
-class ND_TriDofTransformation : public ND_DofTransformation
-{
-public:
-   ND_TriDofTransformation(int order);
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-};
-
-/// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetDofTransformation : public ND_DofTransformation
-{
-public:
-   ND_TetDofTransformation(int order);
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-};
-
-/// DoF transformation implementation for the Nedelec basis on wedge elements
-/** TODO: (Under development) */
-class ND_WedgeDofTransformation : public ND_DofTransformation
-{
-public:
-   ND_WedgeDofTransformation(int order);
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-};
-
-} // namespace mfem
-
-#endif // MFEM_DOFTRANSFORM
@@ -329,7 +329,7 @@ void KellyErrorEstimator::ComputeEstimates()
         error_estimates(e) = sqrt(factor * error_estimates(e));
      }

-      total_error = error_estimates.Norml2();
+      total_error = error_estimates.Sum();
      delete flux;
      return;
   }
@@ -452,10 +452,9 @@ void KellyErrorEstimator::ComputeEstimates()
   auto pfes = dynamic_cast<ParFiniteElementSpace*>(xfes);
   MFEM_VERIFY(pfes, "xfes is not a ParFiniteElementSpace pointer");

-   double process_local_error = pow(error_estimates.Norml2(),2.0);
+   double process_local_error = error_estimates.Sum();
   MPI_Allreduce(&process_local_error, &total_error, 1, MPI_DOUBLE,
                 MPI_SUM, pfes->GetComm());
-   total_error = sqrt(total_error);
 #endif // MFEM_USE_MPI
 }

@@ -2656,31 +2656,18 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
      {
         for (int i = 0; i + j <= pm2; i++)
         {
-            int k0 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
-            int k1 = 2*pm2 - 2*i + ((2*p-3)-j)*j;
-            int k2 = 2*pm2 - 2*j + ((2*p-3)-i)*i;
-            int k3 = p*pm1 - 2 - 3*j - i - (i+j)*(i+j);
-            int k4 = p*pm1 - 2 - 3*i - j - (i+j)*(i+j);
-            int k5 = p*pm1 - (p - i)*(pm1 - i) + 2*j;
-
+            int k1 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
+            int k2 = p*pm1 - (p - i)*(pm1 - i) + 2*j;
            // (0,1,2)
-            TriDofOrd[0][k0  ] = k0;
-            TriDofOrd[0][k0+1] = k0 + 1;
-            // (1,0,2)
-            TriDofOrd[1][k0  ] = k1;
-            TriDofOrd[1][k0+1] = k1 + 1;
-            // (2,0,1)
-            TriDofOrd[2][k0  ] = k2;
-            TriDofOrd[2][k0+1] = k2 + 1;
-            // (2,1,0)
-            TriDofOrd[3][k0  ] = k3;
-            TriDofOrd[3][k0+1] = k3 + 1;
-            // (1,2,0)
-            TriDofOrd[4][k0  ] = k4;
-            TriDofOrd[4][k0+1] = k4 + 1;
+            TriDofOrd[0][k1  ] = k1;
+            TriDofOrd[0][k1+1] = k1 + 1;
            // (0,2,1)
-            TriDofOrd[5][k0  ] = k5;
-            TriDofOrd[5][k0+1] = k5 + 1;
+            TriDofOrd[5][k1  ] = k2 + 1;
+            TriDofOrd[5][k1+1] = k2;
+
+            // The other orientations can not be supported with the current
+            // interface. The method Mesh::ReorientTetMesh will ensure that
+            // only orientations 0 and 5 are generated.
         }
      }
   }
@@ -2705,6 +2692,11 @@ const int *ND_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
   }
   else if (GeomType == Geometry::TRIANGLE)
   {
+      if (Or != 0 && Or != 5)
+      {
+         MFEM_ABORT("triangle face orientation " << Or << " is not supported! "
+                    "Use Mesh::ReorientTetMesh to fix it.");
+      }
      return TriDofOrd[Or%6];
   }
   else if (GeomType == Geometry::SQUARE)
@@ -16,7 +16,6 @@
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
-#include "doftrans.hpp"
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 #include "complex_fem.hpp"
@@ -35,7 +34,6 @@
 #include "staticcond.hpp"
 #include "tmop.hpp"
 #include "tmop_tools.hpp"
-#include "tmop_amr.hpp"
 #include "gslib.hpp"
 #include "restriction.hpp"
 #include "quadinterpolator.hpp"
@@ -58,12 +58,9 @@ DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)

 FiniteElementSpace::FiniteElementSpace()
   : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
-     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
-     bdofs(NULL),
-     elem_dof(NULL), elem_fos(NULL), bdr_elem_dof(NULL), bdr_elem_fos(NULL),
-     face_dof(NULL),
+     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0), bdofs(NULL),
+     elem_dof(NULL), bdr_elem_dof(NULL), face_dof(NULL),
     NURBSext(NULL), own_ext(false),
-     DoFTrans(0), VDoFTrans(vdim, ordering),
     cP(NULL), cR(NULL), cR_hp(NULL), cP_is_set(false),
     Th(Operator::ANY_TYPE),
     sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
@@ -72,7 +69,6 @@ FiniteElementSpace::FiniteElementSpace()
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
                                       Mesh *mesh,
                                       const FiniteElementCollection *fec)
-   : VDoFTrans(orig.vdim, orig.ordering)
 {
   mesh = mesh ? mesh : orig.mesh;
   fec = fec ? fec : orig.fec;
@@ -263,36 +259,16 @@ void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
   }
 }

-DofTransformation *
-FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
+void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
 {
-   DofTransformation * doftrans = GetElementDofs(i, vdofs);
+   GetElementDofs(i, vdofs);
   DofsToVDofs(vdofs);
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
 }

-DofTransformation *
-FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
+void FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
 {
-   DofTransformation * doftrans = GetBdrElementDofs(i, vdofs);
+   GetBdrElementDofs(i, vdofs);
   DofsToVDofs(vdofs);
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
 }

 void FiniteElementSpace::GetFaceVDofs(int i, Array<int> &vdofs) const
@@ -331,39 +307,21 @@ void FiniteElementSpace::BuildElementToDofTable() const

   // TODO: can we call GetElementDofs only once per element?
   Table *el_dof = new Table;
-   Table *el_fos = (mesh->Dimension() > 2) ? (new Table) : NULL;
   Array<int> dofs;
-   Array<int> F, Fo;
   el_dof -> MakeI (mesh -> GetNE());
-   if (el_fos) { el_fos -> MakeI (mesh -> GetNE()); }
   for (int i = 0; i < mesh -> GetNE(); i++)
   {
      GetElementDofs (i, dofs);
      el_dof -> AddColumnsInRow (i, dofs.Size());
-
-      if (el_fos)
-      {
-         mesh->GetElementFaces(i, F, Fo);
-         el_fos -> AddColumnsInRow (i, Fo.Size());
-      }
   }
   el_dof -> MakeJ();
-   if (el_fos) { el_fos -> MakeJ(); }
   for (int i = 0; i < mesh -> GetNE(); i++)
   {
      GetElementDofs (i, dofs);
      el_dof -> AddConnections (i, (int *)dofs, dofs.Size());
-
-      if (el_fos)
-      {
-         mesh->GetElementFaces(i, F, Fo);
-         el_fos -> AddConnections (i, (int *)Fo, Fo.Size());
-      }
   }
   el_dof -> ShiftUpI();
-   if (el_fos) { el_fos -> ShiftUpI(); }
   elem_dof = el_dof;
-   elem_fos = el_fos;
 }

 void FiniteElementSpace::BuildBdrElementToDofTable() const
@@ -417,9 +375,7 @@ void FiniteElementSpace::BuildFaceToDofTable() const
 void FiniteElementSpace::RebuildElementToDofTable()
 {
   delete elem_dof;
-   delete elem_fos;
   elem_dof = NULL;
-   elem_fos = NULL;
   BuildElementToDofTable();
 }

@@ -1359,10 +1315,8 @@ const FaceQuadratureInterpolator

 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
   const int coarse_ndofs, const Table &coarse_elem_dof,
-   const Table *coarse_elem_fos, const DenseTensor localP[]) const
+   const DenseTensor localP[]) const
 {
-   /// TODO: Implement DofTransformation support
-
   MFEM_VERIFY(mesh->GetLastOperation() == Mesh::REFINE, "");

   Array<int> dofs, coarse_dofs, coarse_vdofs;
@@ -1445,8 +1399,7 @@ void FiniteElementSpace::GetLocalRefinementMatrices(
 }

 SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
-                                                   const Table* old_elem_dof,
-                                                   const Table* old_elem_fos)
+                                                   const Table* old_elem_dof)
 {
   MFEM_VERIFY(GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1459,16 +1412,13 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
      GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }

-   return RefinementMatrix_main(old_ndofs, *old_elem_dof, old_elem_fos,
-                                localP);
+   return RefinementMatrix_main(old_ndofs, *old_elem_dof, localP);
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator
-(const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
- int old_ndofs)
+(const FiniteElementSpace* fespace, Table* old_elem_dof, int old_ndofs)
   : fespace(fespace)
   , old_elem_dof(old_elem_dof)
-   , old_elem_fos(old_elem_fos)
 {
   MFEM_VERIFY(fespace->GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1482,14 +1432,12 @@ FiniteElementSpace::RefinementOperator::RefinementOperator
   {
      fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }
-
-   ConstructDoFTrans();
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator(
   const FiniteElementSpace *fespace, const FiniteElementSpace *coarse_fes)
   : Operator(fespace->GetVSize(), coarse_fes->GetVSize()),
-     fespace(fespace), old_elem_dof(NULL), old_elem_fos(NULL)
+     fespace(fespace), old_elem_dof(NULL)
 {
   Mesh::GeometryList elem_geoms(*fespace->GetMesh());

@@ -1501,50 +1449,11 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(

   // Make a copy of the coarse elem_dof Table.
   old_elem_dof = new Table(coarse_fes->GetElementToDofTable());
-
-   // Make a copy of the coarse elem_fos Table if it exists.
-   if (coarse_fes->GetElementToFaceOrientationTable())
-   {
-      old_elem_fos = new Table(*coarse_fes->GetElementToFaceOrientationTable());
-   }
-
-   ConstructDoFTrans();
 }

 FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
   delete old_elem_dof;
-   delete old_elem_fos;
-}
-
-void FiniteElementSpace::RefinementOperator
-::ConstructDoFTrans()
-{
-   old_DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<old_DoFTrans.Size(); i++)
-   {
-      old_DoFTrans[i] = NULL;
-   }
-
-   const FiniteElementCollection *fec = fespace->FEColl();
-   if (dynamic_cast<const ND_FECollection*>(fec))
-   {
-      const FiniteElement * nd_tri =
-         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
-      if (nd_tri)
-      {
-         old_DoFTrans[Geometry::TRIANGLE] =
-            new ND_TriDofTransformation(nd_tri->GetOrder());
-      }
-
-      const FiniteElement * nd_tet =
-         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
-      if (nd_tet)
-      {
-         old_DoFTrans[Geometry::TETRAHEDRON] =
-            new ND_TetDofTransformation(nd_tet->GetOrder());
-      }
-   }
 }

 void FiniteElementSpace::RefinementOperator
@@ -1553,7 +1462,7 @@ void FiniteElementSpace::RefinementOperator
   Mesh* mesh = fespace->GetMesh();
   const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();

-   Array<int> dofs, vdofs, old_dofs, old_vdofs, old_Fo;
+   Array<int> dofs, vdofs, old_dofs, old_vdofs;

   int vdim = fespace->GetVDim();
   int old_ndofs = width / vdim;
@@ -1568,53 +1477,18 @@ void FiniteElementSpace::RefinementOperator

      subY.SetSize(lP.Height());

-      DofTransformation *doftrans = fespace->GetElementDofs(k, dofs);
+      fespace->GetElementDofs(k, dofs);
      old_elem_dof->GetRow(emb.parent, old_dofs);

-      if (!doftrans)
+      for (int vd = 0; vd < vdim; vd++)
      {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            dofs.Copy(vdofs);
-            fespace->DofsToVDofs(vd, vdofs);
-            old_dofs.Copy(old_vdofs);
-            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-            x.GetSubVector(old_vdofs, subX);
-            lP.Mult(subX, subY);
-            y.SetSubVector(vdofs, subY);
-         }
-      }
-      else
-      {
-         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
-
-         DofTransformation *new_doftrans = NULL;
-         VDofTransformation *vdoftrans =
-            dynamic_cast<VDofTransformation*>(doftrans);
-         if (vdoftrans)
-         {
-            new_doftrans = doftrans;
-            doftrans = vdoftrans->GetDofTransformation();
-         }
-
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            dofs.Copy(vdofs);
-            fespace->DofsToVDofs(vd, vdofs);
-            old_dofs.Copy(old_vdofs);
-            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-            x.GetSubVector(old_vdofs, subX);
-            old_DoFTrans[geom]->InvTransformPrimal(subX);
-            lP.Mult(subX, subY);
-            doftrans->TransformPrimal(subY);
-            y.SetSubVector(vdofs, subY);
-         }
-
-         if (vdoftrans)
-         {
-            doftrans = new_doftrans;
-         }
+         dofs.Copy(vdofs);
+         fespace->DofsToVDofs(vd, vdofs);
+         old_dofs.Copy(old_vdofs);
+         fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+         x.GetSubVector(old_vdofs, subX);
+         lP.Mult(subX, subY);
+         y.SetSubVector(vdofs, subY);
      }
   }
 }
@@ -1630,12 +1504,12 @@ void FiniteElementSpace::RefinementOperator
   Array<char> processed(fespace->GetVSize());
   processed = 0;

-   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs, old_Fo;
+   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs;

   int vdim = fespace->GetVDim();
   int old_ndofs = width / vdim;

-   Vector subY, subX, subYt, subXt;
+   Vector subY, subX;

   for (int k = 0; k < mesh->GetNE(); k++)
   {
@@ -1643,77 +1517,30 @@ void FiniteElementSpace::RefinementOperator
      const Geometry::Type geom = mesh->GetElementBaseGeometry(k);
      const DenseMatrix &lP = localP[geom](emb.matrix);

-      DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
+      fespace->GetElementDofs(k, f_dofs);
      old_elem_dof->GetRow(emb.parent, c_dofs);

-      if (!doftrans)
+      subY.SetSize(lP.Width());
+
+      for (int vd = 0; vd < vdim; vd++)
      {
-         subY.SetSize(lP.Width());
+         f_dofs.Copy(f_vdofs);
+         fespace->DofsToVDofs(vd, f_vdofs);
+         c_dofs.Copy(c_vdofs);
+         fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);

-         for (int vd = 0; vd < vdim; vd++)
+         x.GetSubVector(f_vdofs, subX);
+
+         for (int p = 0; p < f_dofs.Size(); ++p)
         {
-            f_dofs.Copy(f_vdofs);
-            fespace->DofsToVDofs(vd, f_vdofs);
-            c_dofs.Copy(c_vdofs);
-            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
-
-            x.GetSubVector(f_vdofs, subX);
-
-            for (int p = 0; p < f_dofs.Size(); ++p)
+            if (processed[DecodeDof(f_dofs[p])])
            {
-               if (processed[DecodeDof(f_dofs[p])])
-               {
-                  subX[p] = 0.0;
-               }
+               subX[p] = 0.0;
            }
-
-            lP.MultTranspose(subX, subY);
-            y.AddElementVector(c_vdofs, subY);
-         }
-      }
-      else
-      {
-         subYt.SetSize(lP.Width());
-
-         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
-
-         DofTransformation *new_doftrans = NULL;
-         VDofTransformation *vdoftrans =
-            dynamic_cast<VDofTransformation*>(doftrans);
-         if (vdoftrans)
-         {
-            new_doftrans = doftrans;
-            doftrans = vdoftrans->GetDofTransformation();
         }

-         for (int vd = 0; vd < vdim; vd++)
-         {
-            f_dofs.Copy(f_vdofs);
-            fespace->DofsToVDofs(vd, f_vdofs);
-            c_dofs.Copy(c_vdofs);
-            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
-
-            x.GetSubVector(f_vdofs, subX);
-            old_DoFTrans[geom]->InvTransformPrimal(subX);
-
-            for (int p = 0; p < f_dofs.Size(); ++p)
-            {
-               if (processed[DecodeDof(f_dofs[p])])
-               {
-                  subX[p] = 0.0;
-               }
-            }
-
-            lP.MultTranspose(subX, subY);
-            doftrans->TransformPrimal(subY);
-            y.AddElementVector(c_vdofs, subY);
-         }
-
-         if (vdoftrans)
-         {
-            doftrans = new_doftrans;
-         }
+         lP.MultTranspose(subX, subY);
+         y.AddElementVector(c_vdofs, subY);
      }

      for (int p = 0; p < f_dofs.Size(); ++p)
@@ -1723,7 +1550,6 @@ void FiniteElementSpace::RefinementOperator
   }
 }

-/// TODO: Implement DofTransformation support
 FiniteElementSpace::DerefinementOperator::DerefinementOperator(
   const FiniteElementSpace *f_fes, const FiniteElementSpace *c_fes,
   BilinearFormIntegrator *mass_integ)
@@ -1881,11 +1707,8 @@ void FiniteElementSpace::GetLocalDerefinementMatrices(Geometry::Type geom,
 }

 SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
-                                                     const Table* old_elem_dof,
-                                                     const Table* old_elem_fos)
+                                                     const Table* old_elem_dof)
 {
-   /// TODO: Implement DofTransformation support
-
   MFEM_VERIFY(Nonconforming(), "Not implemented for conforming meshes.");
   MFEM_VERIFY(old_ndofs, "Missing previous (finer) space.");
   MFEM_VERIFY(ndofs <= old_ndofs, "Previous space is not finer.");
@@ -1991,7 +1814,6 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
   this->ordering = (Ordering::Type) ordering;

   elem_dof = NULL;
-   elem_fos = NULL;
   face_dof = NULL;

   sequence = 0;
@@ -2019,8 +1841,6 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
      UpdateNURBS();
      cP = cR = cR_hp = NULL;
      cP_is_set = false;
-
-      ConstructDoFTrans();
   }
   else
   {
@@ -2028,41 +1848,9 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
      own_ext = 0;
      Construct();
   }
-
   BuildElementToDofTable();
 }

-void FiniteElementSpace::ConstructDoFTrans()
-{
-   DestroyDoFTrans();
-
-   VDoFTrans.SetVDim(vdim);
-   DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<DoFTrans.Size(); i++)
-   {
-      DoFTrans[i] = NULL;
-   }
-   if (mesh->Dimension() < 3) { return; }
-   if (dynamic_cast<const ND_FECollection*>(fec))
-   {
-      const FiniteElement * nd_tri =
-         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
-      if (nd_tri)
-      {
-         DoFTrans[Geometry::TRIANGLE] =
-            new ND_TriDofTransformation(nd_tri->GetOrder());
-      }
-
-      const FiniteElement * nd_tet =
-         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
-      if (nd_tet)
-      {
-         DoFTrans[Geometry::TETRAHEDRON] =
-            new ND_TetDofTransformation(nd_tet->GetOrder());
-      }
-   }
-}
-
 NURBSExtension *FiniteElementSpace::StealNURBSext()
 {
   if (NURBSext && !own_ext)
@@ -2158,9 +1946,7 @@ void FiniteElementSpace::Construct()
               "Variable order space requires a nonconforming mesh.");

   elem_dof = NULL;
-   elem_fos = NULL;
   bdr_elem_dof = NULL;
-   bdr_elem_fos = NULL;
   face_dof = NULL;

   ndofs = 0;
@@ -2258,8 +2044,6 @@ void FiniteElementSpace::Construct()

   ndofs = nvdofs + nedofs + nfdofs + nbdofs;

-   ConstructDoFTrans();
-
   // record the current mesh sequence number to detect refinement etc.
   mesh_sequence = mesh->GetSequence();

@@ -2511,22 +2295,14 @@ int FiniteElementSpace::GetNVariants(int entity, int index) const
 static const char* msg_orders_changed =
   "Element orders changed, you need to Update() the space first.";

-DofTransformation *
-FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
+void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

   if (elem_dof)
   {
      elem_dof->GetRow(elem, dofs);
-
-      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
-      {
-         Array<int> Fo;
-         elem_fos -> GetRow (elem, Fo);
-         DoFTrans[mesh->GetElementBaseGeometry(elem)]->SetFaceOrientations(Fo);
-      }
-      return DoFTrans[mesh->GetElementBaseGeometry(elem)];
+      return;
   }

   Array<int> V, E, Eo, F, Fo; // TODO: LocalArray
@@ -2550,11 +2326,6 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
      {
         nfd += fec->GetNumDof(mesh->GetFaceGeometry(F[i]), order);
      }
-      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
-      {
-         DoFTrans[mesh->GetElementBaseGeometry(elem)]
-         -> SetFaceOrientations(Fo);
-      }
   }

   dofs.SetSize(0);
@@ -2612,7 +2383,6 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
         dofs.Append(bbase + j);
      }
   }
-   return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }

 const FiniteElement *FiniteElementSpace::GetFE(int i) const
@@ -2645,27 +2415,18 @@ const FiniteElement *FiniteElementSpace::GetFE(int i) const
   return FE;
 }

-DofTransformation *
-FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
+void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

   if (bdr_elem_dof)
   {
      bdr_elem_dof->GetRow(bel, dofs);
-
-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
-      {
-         Array<int> Fo;
-         bdr_elem_fos -> GetRow (bel, Fo);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
-         SetFaceOrientations(Fo);
-      }
-      return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
+      return;
   }

-   Array<int> V, E, Eo, Fo; // TODO: LocalArray
-   int F, oF;
+   Array<int> V, E, Eo; // TODO: LocalArray
+   int F, Fo;

   int dim = mesh->Dimension();
   auto geom = mesh->GetBdrElementGeometry(bel);
@@ -2684,17 +2445,7 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const

   if (nv) { mesh->GetBdrElementVertices(bel, V); }
   if (ne) { mesh->GetBdrElementEdges(bel, E, Eo); }
-   if (nf)
-   {
-      mesh->GetBdrElementFace(bel, &F, &oF);
-
-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
-      {
-         Fo.Append(oF);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
-         SetFaceOrientations(Fo);
-      }
-   }
+   if (nf) { mesh->GetBdrElementFace(bel, &F, &Fo); }

   dofs.SetSize(0);
   dofs.Reserve(nv*V.Size() + ne*E.Size() + nf);
@@ -2727,15 +2478,13 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
   if (nf) // face DOFs
   {
      int fbase = (var_face_dofs.Size() > 0) ? FindFaceDof(F, nf) : F*nf;
-      const int *ind = fec->GetDofOrdering(geom, order, oF);
+      const int *ind = fec->GetDofOrdering(geom, order, Fo);

      for (int j = 0; j < nf; j++)
      {
         dofs.Append(EncodeDof(nvdofs + nedofs + fbase, ind[j]));
      }
   }
-
-   return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
 }

 int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,
@@ -3045,8 +2794,6 @@ void FiniteElementSpace::Destroy()
   }
   E2BFQ_array.SetSize(0);

-   DestroyDoFTrans();
-
   dof_elem_array.DeleteAll();
   dof_ldof_array.DeleteAll();

@@ -3059,9 +2806,7 @@ void FiniteElementSpace::Destroy()
   else
   {
      delete elem_dof;
-      delete elem_fos;
      delete bdr_elem_dof;
-      delete bdr_elem_fos;
      delete face_dof;

      delete [] bdofs;
@@ -3069,15 +2814,6 @@ void FiniteElementSpace::Destroy()
   ceed::RemoveBasisAndRestriction(this);
 }

-void FiniteElementSpace::DestroyDoFTrans()
-{
-   for (int i = 0; i < DoFTrans.Size(); i++)
-   {
-      delete DoFTrans[i];
-   }
-   DoFTrans.SetSize(0);
-}
-
 void FiniteElementSpace::GetTransferOperator(
   const FiniteElementSpace &coarse_fes, OperatorHandle &T) const
 {
@@ -3095,8 +2831,6 @@ void FiniteElementSpace::GetTransferOperator(
      }
      T.Reset(RefinementMatrix_main(coarse_fes.GetNDofs(),
                                    coarse_fes.GetElementToDofTable(),
-                                    coarse_fes.
-                                    GetElementToFaceOrientationTable(),
                                    localP));
   }
   else
@@ -3199,7 +2933,6 @@ void FiniteElementSpace::Update(bool want_transform)
   }

   Table* old_elem_dof = NULL;
-   Table* old_elem_fos = NULL;
   int old_ndofs;
   bool old_orders_changed = orders_changed;

@@ -3207,9 +2940,7 @@ void FiniteElementSpace::Update(bool want_transform)
   if (want_transform)
   {
      old_elem_dof = elem_dof;
-      old_elem_fos = elem_fos;
      elem_dof = NULL;
-      elem_fos = NULL;
      old_ndofs = ndofs;
   }

@@ -3235,18 +2966,15 @@ void FiniteElementSpace::Update(bool want_transform)
         {
            if (Th.Type() != Operator::MFEM_SPARSEMAT)
            {
-               Th.Reset(new RefinementOperator(this, old_elem_dof,
-                                               old_elem_fos, old_ndofs));
+               Th.Reset(new RefinementOperator(this, old_elem_dof, old_ndofs));
               // The RefinementOperator takes ownership of 'old_elem_dof', so
               // we no longer own it:
               old_elem_dof = NULL;
-               old_elem_fos = NULL;
            }
            else
            {
               // calculate fully assembled matrix
-               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof,
-                                         old_elem_fos));
+               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof));
            }
            break;
         }
@@ -3254,7 +2982,7 @@ void FiniteElementSpace::Update(bool want_transform)
         case Mesh::DEREFINE:
         {
            BuildConformingInterpolation();
-            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
+            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof));
            if (cP && cR)
            {
               Th.SetOperatorOwner(false);
@@ -3269,7 +2997,6 @@ void FiniteElementSpace::Update(bool want_transform)
      }

      delete old_elem_dof;
-      delete old_elem_fos;
   }
 }

@@ -16,7 +16,6 @@
 #include "../linalg/sparsemat.hpp"
 #include "../mesh/mesh.hpp"
 #include "fe_coll.hpp"
-#include "doftrans.hpp"
 #include "restriction.hpp"
 #include <iostream>
 #include <unordered_map>
@@ -129,9 +128,7 @@ protected:

   // precalculated DOFs for each element, boundary element, and face
   mutable Table *elem_dof; // owned (except in NURBS FE space)
-   mutable Table *elem_fos; // face orientations by element index
   mutable Table *bdr_elem_dof; // owned (except in NURBS FE space)
-   mutable Table *bdr_elem_fos; // bdr face orientations by bdr element index
   mutable Table *face_dof; // owned; in var-order space contains variant 0 DOFs

   Array<int> dof_elem_array, dof_ldof_array;
@@ -140,9 +137,6 @@ protected:
   int own_ext;
   mutable Array<int> face_to_be; // NURBS FE space only

-   Array<DofTransformation*> DoFTrans;
-   mutable VDofTransformation VDoFTrans;
-
   /** Matrix representing the prolongation from the global conforming dofs to
       a set of intermediate partially conforming dofs, e.g. the dofs associated
       with a "cut" space on a non-conforming mesh. */
@@ -195,9 +189,6 @@ protected:
   void Construct();
   void Destroy();

-   void ConstructDoFTrans();
-   void DestroyDoFTrans();
-
   void BuildElementToDofTable() const;
   void BuildBdrElementToDofTable() const;
   void BuildFaceToDofTable() const;
@@ -292,19 +283,12 @@ protected:
      const FiniteElementSpace* fespace;
      DenseTensor localP[Geometry::NumGeom];
      Table* old_elem_dof; // Owned.
-      Table* old_elem_fos; // Owned.
-
-      Array<DofTransformation*> old_DoFTrans;
-      mutable VDofTransformation old_VDoFTrans;
-
-      void ConstructDoFTrans();

   public:
      /** Construct the operator based on the elem_dof table of the original
          (coarse) space. The class takes ownership of the table. */
      RefinementOperator(const FiniteElementSpace* fespace,
-                         Table *old_elem_dof/*takes ownership*/,
-                         Table *old_elem_fos/*takes ownership*/, int old_ndofs);
+                         Table *old_elem_dof/*takes ownership*/, int old_ndofs);
      RefinementOperator(const FiniteElementSpace *fespace,
                         const FiniteElementSpace *coarse_fes);
      virtual void Mult(const Vector &x, Vector &y) const;
@@ -318,7 +302,6 @@ protected:
      const FiniteElementSpace *fine_fes; // Not owned.
      DenseTensor localR[Geometry::NumGeom];
      Table *coarse_elem_dof; // Owned.
-      // Table *coarse_elem_fos; // Owned.
      Table coarse_to_fine;
      Array<int> coarse_to_ref_type;
      Array<Geometry::Type> ref_type_to_geom;
@@ -340,7 +323,6 @@ protected:
       the same vector dimension, vdim. */
   SparseMatrix *RefinementMatrix_main(const int coarse_ndofs,
                                       const Table &coarse_elem_dof,
-                                       const Table *coarse_elem_fos,
                                       const DenseTensor localP[]) const;

   void GetLocalRefinementMatrices(Geometry::Type geom,
@@ -351,12 +333,10 @@ protected:
   /** Calculate explicit GridFunction interpolation matrix (after mesh
       refinement). NOTE: consider using the RefinementOperator class instead
       of the fully assembled matrix, which can take a lot of memory. */
-   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof,
-                                  const Table* old_elem_fos);
+   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof);

   /// Calculate GridFunction restriction matrix after mesh derefinement.
-   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof,
-                                    const Table* old_elem_fos);
+   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof);

   /** @brief Return in @a localP the local refinement matrices that map
       between fespaces after mesh refinement. */
@@ -634,11 +614,10 @@ public:
   int GetBdrAttribute(int i) const { return mesh->GetBdrAttribute(i); }

   /// Returns indices of degrees of freedom of element 'elem'.
-   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
+   virtual void GetElementDofs(int elem, Array<int> &dofs) const;

   /// Returns indices of degrees of freedom for boundary element 'bel'.
-   virtual DofTransformation *GetBdrElementDofs(int bel,
-                                                Array<int> &dofs) const;
+   virtual void GetBdrElementDofs(int bel, Array<int> &dofs) const;

   /** @brief Returns the indices of the degrees of freedom for the specified
       face, including the DOFs for the edges and the vertices of the face. */
@@ -687,10 +666,10 @@ public:
   static void AdjustVDofs(Array<int> &vdofs);

   /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;
+   void GetElementVDofs(int i, Array<int> &vdofs) const;

   /// Returns indexes of degrees of freedom for i'th boundary element.
-   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;
+   void GetBdrElementVDofs(int i, Array<int> &vdofs) const;

   /// Returns indexes of degrees of freedom for i'th face element (2D and 3D).
   void GetFaceVDofs(int i, Array<int> &vdofs) const;
@@ -716,8 +695,6 @@ public:
       is preserved. */
   void ReorderElementToDofTable();

-   const Table *GetElementToFaceOrientationTable() const { return elem_fos; }
-
   /** @brief Return a reference to the internal Table that stores the lists of
       scalar dofs, for each mesh element, as returned by GetElementDofs(). */
   const Table &GetElementToDofTable() const { return *elem_dof; }
@@ -255,8 +255,6 @@ void GridFunction::SumFluxAndCount(BilinearFormIntegrator &blfi,
   GridFunction &u = *this;

   ElementTransformation *Transf;
-   DofTransformation *udoftrans;
-   DofTransformation *fdoftrans;

   FiniteElementSpace *ufes = u.FESpace();
   FiniteElementSpace *ffes = flux.FESpace();
@@ -276,23 +274,15 @@ void GridFunction::SumFluxAndCount(BilinearFormIntegrator &blfi,
         continue;
      }

-      udoftrans = ufes->GetElementVDofs(i, udofs);
-      fdoftrans = ffes->GetElementVDofs(i, fdofs);
+      ufes->GetElementVDofs(i, udofs);
+      ffes->GetElementVDofs(i, fdofs);

      u.GetSubVector(udofs, ul);
-      if (udoftrans)
-      {
-         udoftrans->InvTransformPrimal(ul);
-      }

      Transf = ufes->GetElementTransformation(i);
      blfi.ComputeElementFlux(*ufes->GetFE(i), *Transf, ul,
                              *ffes->GetFE(i), fl, wcoef);

-      if (fdoftrans)
-      {
-         fdoftrans->TransformPrimal(fl);
-      }
      flux.AddElementVector(fdofs, fl);

      FiniteElementSpace::AdjustVDofs(fdofs);
@@ -374,7 +364,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const

   int k;

-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
   const FiniteElement *FElem = fes->GetFE(i);
   const IntegrationRule *ElemVert =
      Geometries.GetVertices(FElem->GetGeomType());
@@ -384,10 +374,6 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
   vdim--;
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }

   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
@@ -417,7 +403,7 @@ double GridFunction::GetValue(int i, const IntegrationPoint &ip, int vdim)
 const
 {
   Array<int> dofs;
-   DofTransformation * doftrans = fes->GetElementDofs(i, dofs);
+   fes->GetElementDofs(i, dofs);
   fes->DofsToVDofs(vdim-1, dofs);
   Vector DofVal(dofs.Size()), LocVec;
   const FiniteElement *fe = fes->GetFE(i);
@@ -432,10 +418,6 @@ const
      fe->CalcPhysShape(*Tr, DofVal);
   }
   GetSubVector(dofs, LocVec);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(LocVec);
-   }

   return (DofVal * LocVec);
 }
@@ -446,13 +428,9 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
   const FiniteElement *FElem = fes->GetFE(i);
   int dof = FElem->GetDof();
   Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
      Vector shape(dof);
@@ -492,35 +470,30 @@ const
   Array<int> dofs;
   int n = ir.GetNPoints();
   vals.SetSize(n);
-   DofTransformation * doftrans = fes->GetElementDofs(i, dofs);
+   fes->GetElementDofs(i, dofs);
   fes->DofsToVDofs(vdim-1, dofs);
   const FiniteElement *FElem = fes->GetFE(i);
   int dof = FElem->GetDof();
   Vector DofVal(dof), loc_data(dof);
   GetSubVector(dofs, loc_data);
-   if (doftrans)
+   if (FElem->GetMapType() == FiniteElement::VALUE)
   {
-      doftrans->InvTransformPrimal(loc_data);
+      for (int k = 0; k < n; k++)
+      {
+         FElem->CalcShape(ir.IntPoint(k), DofVal);
+         vals(k) = DofVal * loc_data;
+      }
   }
-   for (int k = 0; k < n; k++)
-      if (FElem->GetMapType() == FiniteElement::VALUE)
+   else
+   {
+      ElementTransformation *Tr = fes->GetElementTransformation(i);
+      for (int k = 0; k < n; k++)
      {
-         for (int k = 0; k < n; k++)
-         {
-            FElem->CalcShape(ir.IntPoint(k), DofVal);
-            vals(k) = DofVal * loc_data;
-         }
-      }
-      else
-      {
-         ElementTransformation *Tr = fes->GetElementTransformation(i);
-         for (int k = 0; k < n; k++)
-         {
-            Tr->SetIntPoint(&ir.IntPoint(k));
-            FElem->CalcPhysShape(*Tr, DofVal);
-            vals(k) = DofVal * loc_data;
-         }
+         Tr->SetIntPoint(&ir.IntPoint(k));
+         FElem->CalcPhysShape(*Tr, DofVal);
+         vals(k) = DofVal * loc_data;
      }
+   }
 }

 void GridFunction::GetValues(int i, const IntegrationRule &ir, Vector &vals,
@@ -888,12 +861,11 @@ void GridFunction::GetVectorValue(ElementTransformation &T,

   Array<int> vdofs;
   const FiniteElement *fe = NULL;
-   DofTransformation * doftrans = NULL;

   switch (T.ElementType)
   {
      case ElementTransformation::ELEMENT:
-         doftrans = fes->GetElementVDofs(T.ElementNo, vdofs);
+         fes->GetElementVDofs(T.ElementNo, vdofs);
         fe = fes->GetFE(T.ElementNo);
         break;
      case ElementTransformation::EDGE:
@@ -984,10 +956,6 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
   int dof = fe->GetDof();
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
   if (fe->GetRangeType() == FiniteElement::SCALAR)
   {
      Vector shape(dof);
@@ -1030,14 +998,10 @@ void GridFunction::GetVectorValues(ElementTransformation &T,
   int dof = FElem->GetDof();

   Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(T.ElementNo, vdofs);
+   fes->GetElementVDofs(T.ElementNo, vdofs);
+
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
-
   int nip = ir.GetNPoints();

   if (FElem->GetRangeType() == FiniteElement::SCALAR)
@@ -1125,8 +1089,6 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
   // Without averaging ...

   const FiniteElementSpace *orig_fes = orig_func.FESpace();
-   DofTransformation * doftrans;
-   DofTransformation * orig_doftrans;
   Array<int> vdofs, orig_vdofs;
   Vector shape, loc_values, orig_loc_values;
   int i, j, d, ne, dof, odof, vdim;
@@ -1135,13 +1097,9 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
   vdim = fes->GetVDim();
   for (i = 0; i < ne; i++)
   {
-      doftrans = fes->GetElementVDofs(i, vdofs);
-      orig_doftrans = orig_fes->GetElementVDofs(i, orig_vdofs);
+      fes->GetElementVDofs(i, vdofs);
+      orig_fes->GetElementVDofs(i, orig_vdofs);
      orig_func.GetSubVector(orig_vdofs, orig_loc_values);
-      if (orig_doftrans)
-      {
-         orig_doftrans->InvTransformPrimal(orig_loc_values);
-      }
      const FiniteElement *fe = fes->GetFE(i);
      const FiniteElement *orig_fe = orig_fes->GetFE(i);
      dof = fe->GetDof();
@@ -1159,10 +1117,6 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
               shape * ((const double *)orig_loc_values + d * odof) ;
         }
      }
-      if (doftrans)
-      {
-         doftrans->TransformPrimal(loc_values);
-      }
      SetSubVector(vdofs, loc_values);
   }
 }
@@ -1172,10 +1126,8 @@ void GridFunction::GetBdrValuesFrom(const GridFunction &orig_func)
   // Without averaging ...

   const FiniteElementSpace *orig_fes = orig_func.FESpace();
-   // DofTransformation * doftrans;
-   // DofTransformation * orig_doftrans;
   Array<int> vdofs, orig_vdofs;
-   Vector shape, loc_values, loc_values_t, orig_loc_values, orig_loc_values_t;
+   Vector shape, loc_values, orig_loc_values;
   int i, j, d, nbe, dof, odof, vdim;

   nbe = fes->GetNBE();
@@ -1213,33 +1165,37 @@ void GridFunction::GetVectorFieldValues(
   Array<int> vdofs;
   ElementTransformation *transf;

-   int d, k, n, sdim, dof;
+   int d, j, k, n, sdim, dof, ind;

   n = ir.GetNPoints();
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
   const FiniteElement *fe = fes->GetFE(i);
   dof = fe->GetDof();
   sdim = fes->GetMesh()->SpaceDimension();
-   // int *dofs = &vdofs[comp*dof];
+   int *dofs = &vdofs[comp*dof];
   transf = fes->GetElementTransformation(i);
   transf->Transform(ir, tr);
   vals.SetSize(n, sdim);
   DenseMatrix vshape(dof, sdim);
-   Vector loc_data, val(sdim);
-   GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
+   double a;
   for (k = 0; k < n; k++)
   {
      const IntegrationPoint &ip = ir.IntPoint(k);
      transf->SetIntPoint(&ip);
      fe->CalcVShape(*transf, vshape);
-      vshape.MultTranspose(loc_data, val);
      for (d = 0; d < sdim; d++)
      {
-         vals(k,d) = val(d);
+         a = 0.0;
+         for (j = 0; j < dof; j++)
+            if ( (ind=dofs[j]) >= 0 )
+            {
+               a += vshape(j, d) * data[ind];
+            }
+            else
+            {
+               a -= vshape(j, d) * data[-1-ind];
+            }
+         vals(k, d) = a;
      }
   }
 }
@@ -1409,13 +1365,9 @@ void GridFunction::GetVectorGradientHat(
   const FiniteElement *FElem = fes->GetFE(elNo);
   int dim = FElem->GetDim(), dof = FElem->GetDof();
   Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(elNo, vdofs);
+   fes->GetElementVDofs(elNo, vdofs);
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
   // assuming scalar FE
   int vdim = fes->GetVDim();
   DenseMatrix dshape(dof, dim);
@@ -1454,13 +1406,9 @@ double GridFunction::GetDivergence(ElementTransformation &T) const
         {
            // Assuming RT-type space
            Array<int> dofs;
-            DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
+            fes->GetElementDofs(elNo, dofs);
            Vector loc_data, divshape(fe->GetDof());
            GetSubVector(dofs, loc_data);
-            if (doftrans)
-            {
-               doftrans->InvTransformPrimal(loc_data);
-            }
            fe->CalcDivShape(T.GetIntPoint(), divshape);
            return (loc_data * divshape) / T.Weight();
         }
@@ -1551,13 +1499,9 @@ void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
         {
            // Assuming ND-type space
            Array<int> dofs;
-            DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
+            fes->GetElementDofs(elNo, dofs);
            Vector loc_data;
            GetSubVector(dofs, loc_data);
-            if (doftrans)
-            {
-               doftrans->InvTransformPrimal(loc_data);
-            }
            DenseMatrix curl_shape(fe->GetDof(), fe->GetDim() == 3 ? 3 : 1);
            fe->CalcCurlShape(T.GetIntPoint(), curl_shape);
            curl.SetSize(curl_shape.Width());
@@ -1699,12 +1643,8 @@ void GridFunction::GetGradients(ElementTransformation &tr,
   DenseMatrix dshape(fe->GetDof(), fe->GetDim());
   Vector lval, gh(fe->GetDim()), gcol;
   Array<int> dofs;
-   DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
+   fes->GetElementDofs(elNo, dofs);
   GetSubVector(dofs, lval);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(lval);
-   }
   grad.SetSize(fe->GetDim(), ir.GetNPoints());
   for (int i = 0; i < ir.GetNPoints(); i++)
   {
@@ -1784,8 +1724,6 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
 {
   MassIntegrator Mi;
   DenseMatrix loc_mass;
-   DofTransformation * te_doftrans;
-   DofTransformation * tr_doftrans;
   Array<int> te_dofs, tr_dofs;
   Vector loc_avgs, loc_this;
   Vector int_psi(avgs.Size());
@@ -1796,19 +1734,11 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
   {
      Mi.AssembleElementMatrix2(*fes->GetFE(i), *avgs.FESpace()->GetFE(i),
                                *fes->GetElementTransformation(i), loc_mass);
-      tr_doftrans = fes->GetElementDofs(i, tr_dofs);
-      te_doftrans = avgs.FESpace()->GetElementDofs(i, te_dofs);
+      fes->GetElementDofs(i, tr_dofs);
+      avgs.FESpace()->GetElementDofs(i, te_dofs);
      GetSubVector(tr_dofs, loc_this);
-      if (tr_doftrans)
-      {
-         tr_doftrans->InvTransformPrimal(loc_this);
-      }
      loc_avgs.SetSize(te_dofs.Size());
      loc_mass.Mult(loc_this, loc_avgs);
-      if (te_doftrans)
-      {
-         te_doftrans->TransformPrimal(loc_avgs);
-      }
      avgs.AddElementVector(te_dofs, loc_avgs);
      loc_this = 1.0; // assume the local basis for 'this' sums to 1
      loc_mass.Mult(loc_this, loc_avgs);
@@ -1823,12 +1753,8 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
 void GridFunction::GetElementDofValues(int el, Vector &dof_vals) const
 {
   Array<int> dof_idx;
-   DofTransformation * doftrans = fes->GetElementVDofs(el, dof_idx);
+   fes->GetElementVDofs(el, dof_idx);
   GetSubVector(dof_idx, dof_vals);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(dof_vals);
-   }
 }

 void GridFunction::ProjectGridFunction(const GridFunction &src)
@@ -1864,21 +1790,13 @@ void GridFunction::ProjectGridFunction(const GridFunction &src)
         cached_geom = geom;
      }

-      DofTransformation * src_doftrans = src.fes->GetElementVDofs(i, src_vdofs);
+      src.fes->GetElementVDofs(i, src_vdofs);
      src.GetSubVector(src_vdofs, src_lvec);
-      if (src_doftrans)
-      {
-         src_doftrans->InvTransformPrimal(src_lvec);
-      }
      for (int vd = 0; vd < vdim; vd++)
      {
         P.Mult(&src_lvec[vd*P.Width()], &dest_lvec[vd*P.Height()]);
      }
-      DofTransformation * doftrans = fes->GetElementVDofs(i, dest_vdofs);
-      if (doftrans)
-      {
-         doftrans->TransformPrimal(dest_lvec);
-      }
+      fes->GetElementVDofs(i, dest_vdofs);
      SetSubVector(dest_vdofs, dest_lvec);
   }
 }
@@ -1887,15 +1805,10 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
                                const Vector &lo_, const Vector &hi_)
 {
   Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
   int size = vdofs.Size();
   Vector vals, new_vals(size);
-
   GetSubVector(vdofs, vals);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(vals);
-   }

   MFEM_ASSERT(weights.Size() == size, "Different # of weights and dofs.");
   MFEM_ASSERT(lo_.Size() == size, "Different # of lower bounds and dofs.");
@@ -1912,10 +1825,6 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
   slbqp.SetPrintLevel(0); // print messages only if not converged
   slbqp.Mult(vals, new_vals);

-   if (doftrans)
-   {
-      doftrans->TransformPrimal(new_vals);
-   }
   SetSubVector(vdofs, new_vals);
 }

@@ -1923,14 +1832,10 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
                                double min_, double max_)
 {
   Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
   int size = vdofs.Size();
   Vector vals, new_vals(size);
   GetSubVector(vdofs, vals);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(vals);
-   }

   double max_val = vals.Max();
   double min_val = vals.Min();
@@ -1938,10 +1843,6 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
   if (max_val <= min_)
   {
      new_vals = min_;
-      if (doftrans)
-      {
-         doftrans->TransformPrimal(new_vals);
-      }
      SetSubVector(vdofs, new_vals);
      return;
   }
@@ -2377,7 +2278,6 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
 void GridFunction::ProjectCoefficient(Coefficient &coeff)
 {
   DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
-   DofTransformation * doftrans = NULL;

   if (delta_c == NULL)
   {
@@ -2386,13 +2286,9 @@ void GridFunction::ProjectCoefficient(Coefficient &coeff)

      for (int i = 0; i < fes->GetNE(); i++)
      {
-         doftrans = fes->GetElementVDofs(i, vdofs);
+         fes->GetElementVDofs(i, vdofs);
         vals.SetSize(vdofs.Size());
         fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
-         if (doftrans)
-         {
-            doftrans->TransformPrimal(vals);
-         }
         SetSubVector(vdofs, vals);
      }
   }
@@ -2438,17 +2334,11 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
   Array<int> vdofs;
   Vector vals;

-   DofTransformation * doftrans = NULL;
-
   for (i = 0; i < fes->GetNE(); i++)
   {
-      doftrans = fes->GetElementVDofs(i, vdofs);
+      fes->GetElementVDofs(i, vdofs);
      vals.SetSize(vdofs.Size());
      fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
-      if (doftrans)
-      {
-         doftrans->TransformPrimal(vals);
-      }
      SetSubVector(vdofs, vals);
   }
 }
@@ -2511,7 +2401,6 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
   double val;
   const FiniteElement *fe;
   ElementTransformation *transf;
-   // DofTransformation * doftrans;
   Array<int> vdofs;

   vdim = fes->GetVDim();
@@ -2521,7 +2410,6 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
      fdof = fe->GetDof();
      transf = fes->GetElementTransformation(i);
      const IntegrationRule &ir = fe->GetNodes();
-      // doftrans = fes->GetElementVDofs(i, vdofs);
      fes->GetElementVDofs(i, vdofs);
      for (j = 0; j < fdof; j++)
      {
@@ -95,12 +95,6 @@ public:
      : Vector(data, f->GetVSize())
   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

-   /** @brief Construct a GridFunction using previously allocated Vector @a base
-       starting at the given offset, @a base_offset. */
-   GridFunction(FiniteElementSpace *f, Vector &base, int base_offset = 0)
-      : Vector(base, base_offset, f->GetVSize())
-   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
-
   /// Construct a GridFunction on the given Mesh, using the data from @a input.
   /** The content of @a input should be in the format created by the method
       Save(). The reconstructed FiniteElementSpace and FiniteElementCollection
@@ -610,12 +610,7 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
      {
         if (gsl_code[i] == 1) { indl2.Append(i); }
      }
-      int borderPts = indl2.Size();
-#ifdef MFEM_USE_MPI
-      MPI_Allreduce(MPI_IN_PLACE, &borderPts, 1, MPI_INT, MPI_SUM, gsl_comm->c);
-#endif
-      if (borderPts == 0) { return; } // no points on element borders
-
+      if (indl2.Size() == 0) { return; } // no points on element borders

      Vector field_out_l2(field_out.Size());
      VectorGridFunctionCoefficient field_in_dg(&field_in);
@@ -103,7 +103,6 @@ void LinearForm::Assemble()
 {
   Array<int> vdofs;
   ElementTransformation *eltrans;
-   DofTransformation *doftrans;
   Vector elemvect;

   int i;
@@ -135,14 +134,10 @@ void LinearForm::Assemble()
            if ( domain_integs_marker[k] == NULL ||
                 (*(domain_integs_marker[k]))[elem_attr-1] == 1 )
            {
-               doftrans = fes -> GetElementVDofs (i, vdofs);
+               fes -> GetElementVDofs (i, vdofs);
               eltrans = fes -> GetElementTransformation (i);
               domain_integs[k]->AssembleRHSElementVect(*fes->GetFE(i),
                                                        *eltrans, elemvect);
-               if (doftrans)
-               {
-                  doftrans->TransformDual(elemvect);
-               }
               AddElementVector (vdofs, elemvect);
            }
         }
@@ -179,7 +174,7 @@ void LinearForm::Assemble()
      {
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
+         fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
         for (int k=0; k < boundary_integs.Size(); k++)
         {
@@ -189,10 +184,6 @@ void LinearForm::Assemble()
            boundary_integs[k]->AssembleRHSElementVect(*fes->GetBE(i),
                                                       *eltrans, elemvect);

-            if (doftrans)
-            {
-               doftrans->TransformDual(elemvect);
-            }
            AddElementVector (vdofs, elemvect);
         }
      }
@@ -317,11 +317,14 @@ ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
 void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const double a)
 const
 {
+   MFEM_VERIFY(interior_face_integs.Size() == 0,
+               "the case of interior face integrators is not"
+               " implemented");
+
   if (X.ParFESpace() != pfes)
   {
      X.SetSpace(pfes);
      Y.SetSpace(pfes);
-      Ytmp.SetSize(pfes->GetTrueVSize());
   }

   X.Distribute(&x);
@@ -331,13 +334,9 @@ const
   }
   else
   {
-      MFEM_VERIFY(interior_face_integs.Size() == 0,
-                  "the case of interior face integrators is not"
-                  " implemented");
      mat->Mult(X, Y);
   }
-   pfes->GetProlongationMatrix()->MultTranspose(Y, Ytmp);
-   y.Add(a,Ytmp);
+   pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Y, 1.0, y);
 }

 void ParBilinearForm::FormLinearSystem(
@@ -33,7 +33,6 @@ protected:

   /// Auxiliary objects used in TrueAddMult().
   mutable ParGridFunction X, Y;
-   mutable Vector Ytmp;

   OperatorHandle p_mat, p_mat_e;

@@ -128,9 +128,6 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
   {
      ApplyLDofSigns(*elem_dof);
   }
-
-   // Check for shared trianglular faces with interior Nedelec DoFs
-   CheckNDSTriaDofs();
 }

 void ParFiniteElementSpace::Construct()
@@ -467,53 +464,32 @@ void ParFiniteElementSpace::ApplyLDofSigns(Table &el_dof) const
   ApplyLDofSigns(all_dofs);
 }

-DofTransformation *
-ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
+void ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
 {
   if (elem_dof)
   {
      elem_dof->GetRow(i, dofs);
-
-      if (DoFTrans[mesh->GetElementBaseGeometry(i)])
-      {
-         Array<int> Fo;
-         elem_fos->GetRow(i, Fo);
-         DoFTrans[mesh->GetElementBaseGeometry(i)]->SetFaceOrientations(Fo);
-         return DoFTrans[mesh->GetElementBaseGeometry(i)];
-      }
-      return NULL;
+      return;
   }
-   DofTransformation * doftrans = FiniteElementSpace::GetElementDofs(i, dofs);
+   FiniteElementSpace::GetElementDofs(i, dofs);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
-   return doftrans;
 }

-DofTransformation *
-ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
+void ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
 {
   if (bdr_elem_dof)
   {
      bdr_elem_dof->GetRow(i, dofs);
-
-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(i)])
-      {
-         Array<int> Fo;
-         bdr_elem_fos -> GetRow (i, Fo);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(i)]->SetFaceOrientations(Fo);
-         return DoFTrans[mesh->GetBdrElementBaseGeometry(i)];
-      }
-      return NULL;
+      return;
   }
-   DofTransformation * doftrans =
-      FiniteElementSpace::GetBdrElementDofs(i, dofs);
+   FiniteElementSpace::GetBdrElementDofs(i, dofs);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
-   return doftrans;
 }

 int ParFiniteElementSpace::GetFaceDofs(int i, Array<int> &dofs,
@@ -681,267 +657,60 @@ void ParFiniteElementSpace::GenerateGlobalOffsets() const
   }
 }

-void ParFiniteElementSpace::CheckNDSTriaDofs()
-{
-   // Check for Nedelec basis
-   bool nd_basis = dynamic_cast<const ND_FECollection*>(fec);
-   if (!nd_basis)
-   {
-      nd_strias = false;
-      return;
-   }
-
-   // Check for interior face dofs on triangles (the use of TETRAHEDRON
-   // is not an error)
-   bool nd_fdof  = fec->HasFaceDofs(Geometry::TETRAHEDRON,
-                                    GetMaxElementOrder());
-   if (!nd_fdof)
-   {
-      nd_strias = false;
-      return;
-   }
-
-   // Check for shared triangle faces
-   bool strias   = false;
-   {
-      int ngrps = pmesh->GetNGroups();
-      for (int g = 1; g < ngrps; g++)
-      {
-         strias |= pmesh->GroupNTriangles(g);
-      }
-   }
-
-   // Combine results
-   int loc_nd_strias = strias ? 1 : 0;
-   int glb_nd_strias = 0;
-   MPI_Allreduce(&loc_nd_strias, &glb_nd_strias, 1,
-                 MPI_INTEGER, MPI_SUM, MyComm);
-   nd_strias = glb_nd_strias > 0;
-}
-
 void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
 {
   MFEM_ASSERT(Conforming(), "wrong code path");

   if (P) { return; }

-   if (!nd_strias)
+   int ldof  = GetVSize();
+   int ltdof = TrueVSize();
+
+   HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
+   HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
+   int diag_counter;
+
+   HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
+   HYPRE_Int *j_offd = Memory<HYPRE_Int>(ldof-ltdof);
+   int offd_counter;
+
+   HYPRE_BigInt *cmap = Memory<HYPRE_BigInt>(ldof-ltdof);
+
+   HYPRE_BigInt *col_starts = GetTrueDofOffsets();
+   HYPRE_BigInt *row_starts = GetDofOffsets();
+
+   Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
+
+   i_diag[0] = i_offd[0] = 0;
+   diag_counter = offd_counter = 0;
+   for (int i = 0; i < ldof; i++)
   {
-      // Safe to assume 1-1 correspondence between shared dofs
-      int ldof  = GetVSize();
-      int ltdof = TrueVSize();
-
-      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
-      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
-      int diag_counter;
-
-      HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
-      HYPRE_Int *j_offd = Memory<HYPRE_Int>(ldof-ltdof);
-      int offd_counter;
-
-      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);
-
-      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
-      HYPRE_BigInt *row_starts = GetDofOffsets();
-
-      Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
-
-      i_diag[0] = i_offd[0] = 0;
-      diag_counter = offd_counter = 0;
-      for (int i = 0; i < ldof; i++)
+      int ltdof = GetLocalTDofNumber(i);
+      if (ltdof >= 0)
      {
-         int ltdof = GetLocalTDofNumber(i);
-         if (ltdof >= 0)
-         {
-            j_diag[diag_counter++] = ltdof;
-         }
-         else
-         {
-            cmap_j_offd[offd_counter].one = GetGlobalTDofNumber(i);
-            cmap_j_offd[offd_counter].two = offd_counter;
-            offd_counter++;
-         }
-         i_diag[i+1] = diag_counter;
-         i_offd[i+1] = offd_counter;
+         j_diag[diag_counter++] = ltdof;
      }
-
-      SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);
-
-      for (int i = 0; i < offd_counter; i++)
+      else
      {
-         cmap[i] = cmap_j_offd[i].one;
-         j_offd[cmap_j_offd[i].two] = i;
+         cmap_j_offd[offd_counter].one = GetGlobalTDofNumber(i);
+         cmap_j_offd[offd_counter].two = offd_counter;
+         offd_counter++;
      }
-
-      P = new HypreParMatrix(MyComm, MyRank, NRanks, row_starts, col_starts,
-                             i_diag, j_diag, i_offd, j_offd,
-                             cmap, offd_counter);
+      i_diag[i+1] = diag_counter;
+      i_offd[i+1] = offd_counter;
   }
-   else
+
+   SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);
+
+   for (int i = 0; i < offd_counter; i++)
   {
-      // Some shared dofs will be linear combinations of others
-      int ldof  = GetVSize();
-      int ltdof = TrueVSize();
-
-      HYPRE_Int gdof  = -1;
-      HYPRE_Int gtdof = -1;
-
-      MPI_Allreduce(&ldof, &gdof, 1, HYPRE_MPI_INT, MPI_SUM, MyComm);
-      MPI_Allreduce(&ltdof, &gtdof, 1, HYPRE_MPI_INT, MPI_SUM, MyComm);
-
-      // Ensure face orientations have been communicated
-      pmesh->ExchangeFaceNbrData();
-
-      // Locate and count non-zeros in off-diagonal portion of P
-      int nnz_offd = 0;
-      Array<int> ldsize(ldof); ldsize = 0;
-      Array<int> ltori(ldof);  ltori = 0; // Local triangle orientations
-      {
-         int ngrps = pmesh->GetNGroups();
-         int nedofs = fec->DofForGeometry(Geometry::SEGMENT);
-         Array<int> sdofs;
-         for (int g = 1; g < ngrps; g++)
-         {
-            if (pmesh->gtopo.IAmMaster(g))
-            {
-               continue;
-            }
-            for (int ei=0; ei<pmesh->GroupNEdges(g); ei++)
-            {
-               this->GetSharedEdgeDofs(g, ei, sdofs);
-               for (int i=0; i<sdofs.Size(); i++)
-               {
-                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
-                  if (ldsize[ind] == 0) { nnz_offd++; }
-                  ldsize[ind] = 1;
-               }
-            }
-            for (int fi=0; fi<pmesh->GroupNTriangles(g); fi++)
-            {
-               int face, ori, info1, info2;
-               pmesh->GroupTriangle(g, fi, face, ori);
-               pmesh->GetFaceInfos(face, &info1, &info2);
-               this->GetSharedTriangleDofs(g, fi, sdofs);
-               for (int i=0; i<3*nedofs; i++)
-               {
-                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
-                  if (ldsize[ind] == 0) { nnz_offd++; }
-                  ldsize[ind] = 1;
-               }
-               for (int i=3*nedofs; i<sdofs.Size(); i++)
-               {
-                  if (ldsize[sdofs[i]] == 0) { nnz_offd += 2; }
-                  ldsize[sdofs[i]] = 2;
-                  ltori[sdofs[i]]  = info2 % 64;
-               }
-            }
-            for (int fi=0; fi<pmesh->GroupNQuadrilaterals(g); fi++)
-            {
-               this->GetSharedQuadrilateralDofs(g, fi, sdofs);
-               for (int i=0; i<sdofs.Size(); i++)
-               {
-                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
-                  if (ldsize[ind] == 0) { nnz_offd++; }
-                  ldsize[ind] = 1;
-               }
-            }
-         }
-      }
-
-      HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
-      double    *d_diag = new double[ltdof];
-      int diag_counter;
-
-      HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_offd = new HYPRE_Int[nnz_offd];
-      double    *d_offd = new double[nnz_offd];
-      int offd_counter;
-
-      HYPRE_BigInt *cmap   = new HYPRE_BigInt[ldof-ltdof];
-
-      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
-      HYPRE_BigInt *row_starts = GetDofOffsets();
-
-      Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
-
-      i_diag[0] = i_offd[0] = 0;
-      diag_counter = offd_counter = 0;
-      int offd_col_counter = 0;
-      for (int i = 0; i < ldof; i++)
-      {
-         int ltdof = GetLocalTDofNumber(i);
-         if (ltdof >= 0)
-         {
-            j_diag[diag_counter]   = ltdof;
-            d_diag[diag_counter++] = 1.0;
-         }
-         else
-         {
-            if (ldsize[i] == 1)
-            {
-               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
-               cmap_j_offd[offd_col_counter].two = offd_counter;
-               offd_counter++;
-               offd_col_counter++;
-            }
-            else
-            {
-               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
-               cmap_j_offd[offd_col_counter].two = offd_counter;
-               offd_counter += 2;
-               offd_col_counter++;
-               i_diag[i+1] = diag_counter;
-               i_offd[i+1] = offd_counter;
-               i++;
-               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
-               cmap_j_offd[offd_col_counter].two = offd_counter;
-               offd_counter += 2;
-               offd_col_counter++;
-            }
-         }
-         i_diag[i+1] = diag_counter;
-         i_offd[i+1] = offd_counter;
-      }
-
-      SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_col_counter);
-
-      for (int i = 0; i < nnz_offd; i++)
-      {
-         j_offd[i] = -1;
-         d_offd[i] = 0.0;
-      }
-
-      for (int i = 0; i < offd_col_counter; i++)
-      {
-         cmap[i] = cmap_j_offd[i].one;
-         j_offd[cmap_j_offd[i].two] = i;
-      }
-
-      for (int i = 0; i < ldof; i++)
-      {
-         if (i_offd[i+1] == i_offd[i] + 1)
-         {
-            d_offd[i_offd[i]] = 1.0;
-         }
-         else if (i_offd[i+1] == i_offd[i] + 2)
-         {
-            const double * T = ND_DofTransformation
-                               ::GetFaceTransform(ltori[i]).GetData();
-            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]] + 1;
-            d_offd[i_offd[i]] = T[0]; d_offd[i_offd[i] + 1] = T[2];
-            i++;
-            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]];
-            j_offd[i_offd[i]] = j_offd[i_offd[i] + 1] - 1;
-            d_offd[i_offd[i]] = T[1]; d_offd[i_offd[i] + 1] = T[3];
-         }
-      }
-
-      P = new HypreParMatrix(MyComm, gdof, gtdof, row_starts, col_starts,
-                             i_diag, j_diag, d_diag, i_offd, j_offd, d_offd,
-                             offd_col_counter, cmap);
+      cmap[i] = cmap_j_offd[i].one;
+      j_offd[cmap_j_offd[i].two] = i;
   }

+   P = new HypreParMatrix(MyComm, MyRank, NRanks, row_starts, col_starts,
+                          i_diag, j_diag, i_offd, j_offd, cmap, offd_counter);
+
   SparseMatrix Pdiag;
   P->GetDiag(Pdiag);
   R = Transpose(Pdiag);
@@ -1144,8 +913,6 @@ const Operator *ParFiniteElementSpace::GetProlongationMatrix() const
   {
      if (Pconf) { return Pconf; }

-      if (nd_strias) { return Dof_TrueDof_Matrix(); }
-
      if (NRanks == 1)
      {
         Pconf = new IdentityOperator(GetTrueVSize());
@@ -1449,29 +1216,10 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
   delete [] requests;
 }

-DofTransformation *ParFiniteElementSpace::GetFaceNbrElementVDofs(
+void ParFiniteElementSpace::GetFaceNbrElementVDofs(
   int i, Array<int> &vdofs) const
 {
   face_nbr_element_dof.GetRow(i, vdofs);
-
-   DofTransformation *doftrans = NULL;
-   Geometry::Type geom = GetFaceNbrFE(i)->GetGeomType();
-   if (DoFTrans[geom])
-   {
-      Array<int> F, Fo;
-      pmesh->GetFaceNbrElementFaces(pmesh->GetNE() + i, F, Fo);
-      doftrans = DoFTrans[geom];
-      doftrans->SetFaceOrientations(Fo);
-   }
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
 }

 void ParFiniteElementSpace::GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const
@@ -1530,17 +1278,12 @@ const FiniteElement *ParFiniteElementSpace::GetFaceNbrFaceFE(int i) const

 void ParFiniteElementSpace::Lose_Dof_TrueDof_Matrix()
 {
-   P -> StealData();
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrix *csrP = (hypre_ParCSRMatrix*)(*P);
   hypre_ParCSRMatrixOwnsRowStarts(csrP) = 1;
   hypre_ParCSRMatrixOwnsColStarts(csrP) = 1;
+   P -> StealData();
   dof_offsets.LoseData();
   tdof_offsets.LoseData();
-#else
-   dof_offsets.DeleteAll();
-   tdof_offsets.DeleteAll();
-#endif
 }

 void ParFiniteElementSpace::ConstructTrueDofs()
@@ -2783,8 +2526,7 @@ static int_type* make_j_array(int_type* I, int nrows)

 HypreParMatrix*
 ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
-                                       const Table* old_elem_dof,
-                                       const Table* old_elem_fos)
+                                       const Table* old_elem_dof)
 {
   MFEM_VERIFY(Nonconforming(), "Only supported for nonconforming meshes.");
   MFEM_VERIFY(old_dof_offsets.Size(), "ParFiniteElementSpace::Update needs to "
@@ -2909,8 +2651,7 @@ struct DerefDofMessage

 HypreParMatrix*
 ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
-                                                  const Table* old_elem_dof,
-                                                  const Table *old_elem_fos)
+                                                  const Table* old_elem_dof)
 {
   int nrk = HYPRE_AssumedPartitionCheck() ? 2 : NRanks;

@@ -3266,16 +3007,13 @@ void ParFiniteElementSpace::Update(bool want_transform)
   }

   Table* old_elem_dof = NULL;
-   Table* old_elem_fos = NULL;
   int old_ndofs;

   // save old DOF table
   if (want_transform)
   {
      old_elem_dof = elem_dof;
-      old_elem_fos = elem_fos;
      elem_dof = NULL;
-      elem_fos = NULL;
      old_ndofs = ndofs;
      Swap(dof_offsets, old_dof_offsets);
   }
@@ -3297,25 +3035,22 @@ void ParFiniteElementSpace::Update(bool want_transform)
         {
            if (Th.Type() != Operator::MFEM_SPARSEMAT)
            {
-               Th.Reset(new RefinementOperator(this, old_elem_dof,
-                                               old_elem_fos, old_ndofs));
+               Th.Reset(new RefinementOperator(this, old_elem_dof, old_ndofs));
               // The RefinementOperator takes ownership of 'old_elem_dofs', so
               // we no longer own it:
               old_elem_dof = NULL;
-               old_elem_fos = NULL;
            }
            else
            {
               // calculate fully assembled matrix
-               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
+               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof));
            }
            break;
         }

         case Mesh::DEREFINE:
         {
-            Th.Reset(ParallelDerefinementMatrix(old_ndofs, old_elem_dof,
-                                                old_elem_fos));
+            Th.Reset(ParallelDerefinementMatrix(old_ndofs, old_elem_dof));
            if (Nonconforming())
            {
               Th.SetOperatorOwner(false);
@@ -3327,7 +3062,7 @@ void ParFiniteElementSpace::Update(bool want_transform)

         case Mesh::REBALANCE:
         {
-            Th.Reset(RebalanceMatrix(old_ndofs, old_elem_dof, old_elem_fos));
+            Th.Reset(RebalanceMatrix(old_ndofs, old_elem_dof));
            break;
         }

@@ -3336,7 +3071,6 @@ void ParFiniteElementSpace::Update(bool want_transform)
      }

      delete old_elem_dof;
-      delete old_elem_fos;
   }
 }

@@ -87,12 +87,6 @@ private:
       this is a TransposeOperator wrapping R. */
   mutable Operator *R_transpose;

-   /// Flag indicating the existence of shared triangles with interior ND dofs
-   bool nd_strias;
-
-   /// Resets nd_strias flag at constuction or after rebalancing
-   void CheckNDSTriaDofs();
-
   ParNURBSExtension *pNURBSext() const
   { return dynamic_cast<ParNURBSExtension *>(NURBSext); }

@@ -180,16 +174,14 @@ private:
       The result is a parallel permutation matrix that can be used to update
       all grid functions defined on this space. */
   HypreParMatrix* RebalanceMatrix(int old_ndofs,
-                                   const Table* old_elem_dof,
-                                   const Table* old_elem_fos);
+                                   const Table* old_elem_dof);

   /** Calculate a GridFunction restriction matrix after mesh derefinement.
       The matrix is constructed so that the new grid function interpolates
       the original function, i.e., the original function is evaluated at the
       nodes of the coarse function. */
   HypreParMatrix* ParallelDerefinementMatrix(int old_ndofs,
-                                              const Table *old_elem_dof,
-                                              const Table *old_elem_fos);
+                                              const Table *old_elem_dof);

   /// Updates the internal mesh pointer. @warning @a new_mesh must be
   /// <b>topologically identical</b> to the existing mesh. Used if the address
@@ -210,8 +202,6 @@ public:
   int num_face_nbr_dofs;
   // Face-neighbor-element to face-neighbor dof
   Table face_nbr_element_dof;
-   // Face-neighbor-element face orientations
-   Table face_nbr_element_fos;
   // Face-neighbor to ldof in the face-neighbor numbering
   Table face_nbr_ldof;
   // The global ldof indices of the face-neighbor dofs
@@ -289,10 +279,10 @@ public:
   virtual int GetTrueVSize() const { return ltdof_size; }

   /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   virtual DofTransformation *GetElementDofs(int i, Array<int> &dofs) const;
+   virtual void GetElementDofs(int i, Array<int> &dofs) const;

   /// Returns indexes of degrees of freedom for i'th boundary element.
-   virtual DofTransformation *GetBdrElementDofs(int i, Array<int> &dofs) const;
+   virtual void GetBdrElementDofs(int i, Array<int> &dofs) const;

   /** Returns the indexes of the degrees of freedom for i'th face
       including the dofs for the edges and the vertices of the face. */
@@ -392,7 +382,7 @@ public:
   // Face-neighbor functions
   void ExchangeFaceNbrData();
   int GetFaceNbrVSize() const { return num_face_nbr_dofs; }
-   DofTransformation *GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
+   void GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
   void GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const;
   const FiniteElement *GetFaceNbrFE(int i) const;
   const FiniteElement *GetFaceNbrFaceFE(int i) const;
@@ -407,8 +397,6 @@ public:
   bool Conforming() const { return pmesh->pncmesh == NULL && !nonconf_P; }
   bool Nonconforming() const { return pmesh->pncmesh != NULL || nonconf_P; }

-   bool SharedNDTriangleDofs() const { return nd_strias; }
-
   // Transfer parallel true-dof data from coarse_fes, defined on a coarse mesh,
   // to this FE space, defined on a refined mesh. See full documentation in the
   // base class, FiniteElementSpace::GetTrueTransferOperator.
@@ -325,14 +325,9 @@ void ParGridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
   if (nbr_el_no >= 0)
   {
      Array<int> dofs;
-      DofTransformation * doftrans = pfes->GetFaceNbrElementVDofs(nbr_el_no,
-                                                                  dofs);
+      pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
      Vector loc_data;
      face_nbr_data.GetSubVector(dofs, loc_data);
-      if (doftrans)
-      {
-         doftrans->InvTransformPrimal(loc_data);
-      }
      const FiniteElement *FElem = pfes->GetFaceNbrFE(nbr_el_no);
      int dof = FElem->GetDof();
      if (FElem->GetRangeType() == FiniteElement::SCALAR)
@@ -442,17 +437,12 @@ void ParGridFunction::GetVectorValue(ElementTransformation &T,
   }

   Array<int> vdofs;
-   DofTransformation * doftrans = pfes->GetFaceNbrElementVDofs(nbr_el_no,
-                                                               vdofs);
+   pfes->GetFaceNbrElementVDofs(nbr_el_no, vdofs);
   const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);

   int dof = fe->GetDof();
   Vector loc_data;
   face_nbr_data.GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
   if (fe->GetRangeType() == FiniteElement::SCALAR)
   {
      Vector shape(dof);
@@ -65,11 +65,6 @@ public:
   ParGridFunction(ParFiniteElementSpace *pf, double *data) :
      GridFunction(pf, data), pfes(pf) { }

-   /** @brief Construct a ParGridFunction using previously allocated Vector
-       @a base starting at the given offset, @a base_offset. */
-   ParGridFunction(ParFiniteElementSpace *pf, Vector &base, int base_offset = 0)
-      : GridFunction(pf, base, base_offset), pfes(pf) { }
-
   /// Construct a ParGridFunction using a GridFunction as external data.
   /** The parallel space @a *pf and the space used by @a *gf should match. The
       data from @a *gf is used as the local data of the ParGridFunction on each
@@ -33,10 +33,6 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
   // If fespace == L2
   const ParFiniteElementSpace &pfes =
      static_cast<const ParFiniteElementSpace&>(this->fes);
-
-   // Ensure the face neighbor data is constructed
-   pfes.GetParMesh()->ExchangeFaceNbrData();
-
   const FiniteElement *fe = pfes.GetFE(0);
   const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);
   MFEM_VERIFY(tfe != NULL &&
@@ -15,12 +15,6 @@
 #include "../general/forall.hpp"
 #include <climits>

-#ifdef MFEM_USE_MPI
-
-#include "pfespace.hpp"
-
-#endif
-
 namespace mfem
 {

@@ -681,19 +675,6 @@ H1FaceRestriction::H1FaceRestriction(const FiniteElementSpace &fes,
     gather_indices(nf*dof)
 {
   if (nf==0) { return; }
-
-#ifdef MFEM_USE_MPI
-
-   // If the underlying finite element space is parallel, ensure the face
-   // neighbor information is generated.
-   if (const ParFiniteElementSpace *pfes
-       = dynamic_cast<const ParFiniteElementSpace*>(&fes))
-   {
-      pfes->GetParMesh()->ExchangeFaceNbrData();
-   }
-
-#endif
-
   // If fespace == H1
   const FiniteElement *fe = fes.GetFE(0);
   const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);
@@ -1314,61 +1314,33 @@ static inline void device_copy(double *d_dest, const double *d_src, int size)
 } // namespace internal

 #ifdef MFEM_USE_MPI
-void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction &t)
+void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction
+                                                    &tspec_)
 {
   MFEM_VERIFY(adapt_eval, "SetAdaptivityEvaluator() has not been called!")
   MFEM_VERIFY(ncomp > 0, "No target specifications have been set!");

-   ParFiniteElementSpace *ptspec_fes = t.ParFESpace();
+   ParFiniteElementSpace *ptspec_fes = tspec_.ParFESpace();

   adapt_eval->SetParMetaInfo(*ptspec_fes->GetParMesh(),
                              *ptspec_fes->FEColl(), ncomp);
-   adapt_eval->SetInitialField(*ptspec_fes->GetMesh()->GetNodes(), tspec);
+   adapt_eval->SetInitialField(*tspec_fes->GetMesh()->GetNodes(), tspec);

   tspec_sav = tspec;

   delete tspec_fesv;
-   tspec_fesv = new FiniteElementSpace(ptspec_fes->GetMesh(),
-                                       ptspec_fes->FEColl(), ncomp);
-
-   delete ptspec_fesv;
-   ptspec_fesv = new ParFiniteElementSpace(ptspec_fes->GetParMesh(),
-                                           ptspec_fes->FEColl(), ncomp);
-
-   delete tspec_pgf;
-   tspec_pgf = new ParGridFunction(ptspec_fesv, tspec);
-   tspec_gf = tspec_pgf;
-}
-
-void DiscreteAdaptTC::ParUpdateAfterMeshTopologyChange()
-{
-   ptspec_fesv->Update();
-   if (tspec_fesv)
-   {
-      delete tspec_fesv;
-      tspec_fesv = new FiniteElementSpace(ptspec_fesv->GetMesh(),
-                                          ptspec_fesv->FEColl(), ncomp);
-   }
-   tspec_pgf->Update();
-   tspec_gf = tspec_pgf;
-   tspec.SetDataAndSize(tspec_pgf->GetData(), tspec_pgf->Size());
-   tspec_sav = tspec;
-
-   adapt_eval->SetParMetaInfo(*ptspec_fesv->GetParMesh(),
-                              *ptspec_fesv->FEColl(), ncomp);
-   adapt_eval->SetInitialField(*ptspec_fesv->GetMesh()->GetNodes(), tspec);
+   tspec_fesv = new FiniteElementSpace(tspec_fes->GetMesh(),
+                                       tspec_fes->FEColl(), ncomp);
 }

 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const ParGridFunction &tspec_)
 {
-   const int vdim = tspec_.FESpace()->GetVDim(),
-             ndof = tspec_.FESpace()->GetNDofs();
-   MFEM_VERIFY(ndof == tspec.Size()/ncomp, "Inconsistency in SetTspecAtIndex.");
-
+   const int vdim     = tspec_.FESpace()->GetVDim(),
+             dof_cnt  = tspec_.Size()/vdim;
   const auto tspec__d = tspec_.Read();
   auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*ndof;
-   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
+   const int offset = idx*dof_cnt;
+   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
   FinalizeParDiscreteTargetSpec(tspec_);
 }

@@ -1388,71 +1360,78 @@ void DiscreteAdaptTC::SetParDiscreteTargetSkew(const ParGridFunction &tspec_)
   FinalizeParDiscreteTargetSpec(tspec_);
 }

-void DiscreteAdaptTC::SetParDiscreteTargetAspectRatio(const ParGridFunction &ar)
+void DiscreteAdaptTC::SetParDiscreteTargetAspectRatio(const ParGridFunction
+                                                      &tspec_)
 {
-   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, ar); return; }
+   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, tspec_); return; }
   aspectratioidx = ncomp;
-   SetDiscreteTargetBase(ar);
-   FinalizeParDiscreteTargetSpec(ar);
+   SetDiscreteTargetBase(tspec_);
+   FinalizeParDiscreteTargetSpec(tspec_);
 }

-void DiscreteAdaptTC::SetParDiscreteTargetOrientation(const ParGridFunction &o)
+void DiscreteAdaptTC::SetParDiscreteTargetOrientation(const ParGridFunction
+                                                      &tspec_)
 {
-   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, o); return; }
+   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, tspec_); return; }
   orientationidx = ncomp;
-   SetDiscreteTargetBase(o);
-   FinalizeParDiscreteTargetSpec(o);
+   SetDiscreteTargetBase(tspec_);
+   FinalizeParDiscreteTargetSpec(tspec_);
 }

 void DiscreteAdaptTC::SetParDiscreteTargetSpec(const ParGridFunction &tspec_)
 {
   SetParDiscreteTargetSize(tspec_);
+   FinalizeParDiscreteTargetSpec(tspec_);
 }
 #endif // MFEM_USE_MPI

 void DiscreteAdaptTC::SetDiscreteTargetBase(const GridFunction &tspec_)
 {
-   const int vdim = tspec_.FESpace()->GetVDim(),
-             ndof = tspec_.FESpace()->GetNDofs();
+   const int vdim     = tspec_.FESpace()->GetVDim(),
+             dof_cnt  = tspec_.Size()/vdim;

   ncomp += vdim;

+   delete tspec_fes;
+   tspec_fes = new FiniteElementSpace(tspec_.FESpace()->GetMesh(),
+                                      tspec_.FESpace()->FEColl(), 1);
+
   // need to append data to tspec
   // make a copy of tspec->tspec_temp, increase its size, and
   // copy data from tspec_temp -> tspec, then add new entries
   Vector tspec_temp = tspec;
   tspec.UseDevice(true);
   tspec_sav.UseDevice(true);
-   tspec.SetSize(ncomp*ndof);
+   tspec.SetSize(ncomp*dof_cnt);

   const auto tspec_temp_d = tspec_temp.Read();
   auto tspec_d = tspec.ReadWrite();
   internal::device_copy(tspec_d, tspec_temp_d, tspec_temp.Size());

   const auto tspec__d = tspec_.Read();
-   const int offset = (ncomp-vdim)*ndof;
-   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
+   const int offset = (ncomp-vdim)*dof_cnt;
+   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
 }

 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const GridFunction &tspec_)
 {
-   const int vdim = tspec_.FESpace()->GetVDim(),
-             ndof = tspec_.FESpace()->GetNDofs();
-   MFEM_VERIFY(ndof == tspec.Size()/ncomp, "Inconsistency in SetTargetSpec.");
+   const int vdim     = tspec_.FESpace()->GetVDim(),
+             dof_cnt  = tspec_.Size()/vdim;

   const auto tspec__d = tspec_.Read();
   auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*ndof;
-   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
-   FinalizeSerialDiscreteTargetSpec(tspec_);
+   const int offset = idx*dof_cnt;
+   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
+   FinalizeSerialDiscreteTargetSpec();
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSize(const GridFunction &tspec_)
 {
+
   if (sizeidx > -1) { SetTspecAtIndex(sizeidx, tspec_); return; }
   sizeidx = ncomp;
   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec(tspec_);
+   FinalizeSerialDiscreteTargetSpec();
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSkew(const GridFunction &tspec_)
@@ -1460,31 +1439,32 @@ void DiscreteAdaptTC::SetSerialDiscreteTargetSkew(const GridFunction &tspec_)
   if (skewidx > -1) { SetTspecAtIndex(skewidx, tspec_); return; }
   skewidx = ncomp;
   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec(tspec_);
+   FinalizeSerialDiscreteTargetSpec();
 }

-void DiscreteAdaptTC::SetSerialDiscreteTargetAspectRatio(const GridFunction &ar)
+void DiscreteAdaptTC::SetSerialDiscreteTargetAspectRatio(
+   const GridFunction &tspec_)
 {
-   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, ar); return; }
+   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, tspec_); return; }
   aspectratioidx = ncomp;
-   SetDiscreteTargetBase(ar);
-   FinalizeSerialDiscreteTargetSpec(ar);
+   SetDiscreteTargetBase(tspec_);
+   FinalizeSerialDiscreteTargetSpec();
 }

-void DiscreteAdaptTC::SetSerialDiscreteTargetOrientation(const GridFunction &o)
+void DiscreteAdaptTC::SetSerialDiscreteTargetOrientation(
+   const GridFunction &tspec_)
 {
-   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, o); return; }
+   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, tspec_); return; }
   orientationidx = ncomp;
-   SetDiscreteTargetBase(o);
-   FinalizeSerialDiscreteTargetSpec(o);
+   SetDiscreteTargetBase(tspec_);
+   FinalizeSerialDiscreteTargetSpec();
 }

-void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec(const GridFunction &t)
+void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec()
 {
   MFEM_VERIFY(adapt_eval, "SetAdaptivityEvaluator() has not been called!")
   MFEM_VERIFY(ncomp > 0, "No target specifications have been set!");

-   const FiniteElementSpace *tspec_fes = t.FESpace();
   adapt_eval->SetSerialMetaInfo(*tspec_fes->GetMesh(),
                                 *tspec_fes->FEColl(), ncomp);
   adapt_eval->SetInitialField(*tspec_fes->GetMesh()->GetNodes(), tspec);
@@ -1494,40 +1474,12 @@ void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec(const GridFunction &t)
   delete tspec_fesv;
   tspec_fesv = new FiniteElementSpace(tspec_fes->GetMesh(),
                                       tspec_fes->FEColl(), ncomp);
-
-   delete tspec_gf;
-   tspec_gf = new GridFunction(tspec_fesv, tspec);
-}
-
-void DiscreteAdaptTC::GetDiscreteTargetSpec(GridFunction &tspec_, int idx)
-{
-   if (idx < 0) { return; }
-   const int ndof = tspec_.FESpace()->GetNDofs(),
-             vdim = tspec_.FESpace()->GetVDim();
-   MFEM_VERIFY(ndof == tspec.Size()/ncomp,
-               "Inconsistency in GetSerialDiscreteTargetSpec.");
-
-   for (int i = 0; i < ndof*vdim; i++)
-   {
-      tspec_(i) = tspec(i + idx*ndof);
-   }
-}
-
-void DiscreteAdaptTC::UpdateAfterMeshTopologyChange()
-{
-   tspec_fesv->Update();
-   tspec_gf->Update();
-   tspec.SetDataAndSize(tspec_gf->GetData(), tspec_gf->Size());
-   tspec_sav = tspec;
-
-   adapt_eval->SetSerialMetaInfo(*tspec_fesv->GetMesh(),
-                                 *tspec_fesv->FEColl(), ncomp);
-   adapt_eval->SetInitialField(*tspec_fesv->GetMesh()->GetNodes(), tspec);
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSpec(const GridFunction &tspec_)
 {
   SetSerialDiscreteTargetSize(tspec_);
+   FinalizeSerialDiscreteTargetSpec();
 }


@@ -1557,7 +1509,7 @@ void DiscreteAdaptTC::UpdateTargetSpecificationAtNode(const FiniteElement &el,
   MFEM_VERIFY(tspec.Size() > 0, "Target specification is not set!");

   Array<int> dofs;
-   tspec_fesv->GetElementDofs(T.ElementNo, dofs);
+   tspec_fes->GetElementDofs(T.ElementNo, dofs);
   const int cnt = tspec.Size()/ncomp; // dofs per scalar-field

   for (int i = 0; i < ncomp; i++)
@@ -1572,7 +1524,7 @@ void DiscreteAdaptTC::RestoreTargetSpecificationAtNode(ElementTransformation &T,
   MFEM_VERIFY(tspec.Size() > 0, "Target specification is not set!");

   Array<int> dofs;
-   tspec_fesv->GetElementDofs(T.ElementNo, dofs);
+   tspec_fes->GetElementDofs(T.ElementNo, dofs);
   const int cnt = tspec.Size()/ncomp;
   for (int i = 0; i < ncomp; i++)
   {
@@ -1580,40 +1532,6 @@ void DiscreteAdaptTC::RestoreTargetSpecificationAtNode(ElementTransformation &T,
   }
 }

-void DiscreteAdaptTC::SetTspecFromIntRule(int e_id,
-                                          const IntegrationRule &intrule)
-{
-   switch (target_type)
-   {
-      case IDEAL_SHAPE_GIVEN_SIZE:
-      case GIVEN_SHAPE_AND_SIZE:
-      {
-         const int ndofs = tspec_fesv->GetFE(e_id)->GetDof(),
-                   ntspec_dofs = ndofs*ncomp;
-
-         Vector tspec_vals(ntspec_dofs);
-
-         Array<int> dofs;
-         tspec_fesv->GetElementVDofs(e_id, dofs);
-         tspec.GetSubVector(dofs, tspec_vals);
-         DenseMatrix tr;
-         tspec_gf->GetVectorValues(e_id, intrule, tspec_refine, tr);
-         tspec_refine.Transpose();
-         break;
-      }
-      default:
-         MFEM_ABORT("Incompatible target type for discrete adaptation!");
-   }
-}
-
-void DiscreteAdaptTC::SetTspecDataForDerefinement(FiniteElementSpace *fes)
-{
-   coarse_tspec_fesv = fes;
-   const Operator *c_op = fes->GetUpdateOperator();
-   tspec_derefine.SetSize(c_op->Height());
-   c_op->Mult(tspec, tspec_derefine);
-}
-
 void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
                                            const IntegrationRule &ir,
                                            const Vector &elfun,
@@ -1624,8 +1542,6 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
             nqp = ir.GetNPoints();
   Jtrcomp.SetSize(dim, dim, 4*nqp);

-   FiniteElementSpace *src_fes = tspec_fesv;
-
   switch (target_type)
   {
      case IDEAL_SHAPE_GIVEN_SIZE:
@@ -1634,7 +1550,7 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
         const DenseMatrix &Wideal =
            Geometries.GetGeomToPerfGeomJac(fe.GetGeomType());
         const int dim = Wideal.Height(),
-                   ndofs = tspec_fesv->GetFE(e_id)->GetDof(),
+                   ndofs = tspec_fes->GetFE(e_id)->GetDof(),
                   ntspec_dofs = ndofs*ncomp;

         Vector shape(ndofs), tspec_vals(ntspec_dofs), par_vals,
@@ -1645,29 +1561,11 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
         tspec_fesv->GetElementVDofs(e_id, dofs);
         tspec.UseDevice(true);
         tspec.GetSubVector(dofs, tspec_vals);
-         if (tspec_refine.NumCols() > 0) // Refinement
-         {
-            MFEM_VERIFY(amr_el >= 0, " Target being constructed for an AMR element.");
-            for (int i = 0; i < ncomp; i++)
-            {
-               for (int j = 0; j < ndofs; j++)
-               {
-                  tspec_vals(j + i*ndofs) = tspec_refine(j + amr_el*ndofs, i);
-               }
-            }
-         }
-         else if (tspec_derefine.Size() > 0) // Derefinement
-         {
-            dofs.SetSize(0);
-            coarse_tspec_fesv->GetElementVDofs(e_id, dofs);
-            tspec_derefine.GetSubVector(dofs, tspec_vals);
-            src_fes = coarse_tspec_fesv;
-         }

         for (int q = 0; q < nqp; q++)
         {
            const IntegrationPoint &ip = ir.IntPoint(q);
-            src_fes->GetFE(e_id)->CalcShape(ip, shape);
+            tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
            Jtr(q) = Wideal; // Initialize to identity
            for (int d = 0; d < 4; d++)
            {
@@ -1678,16 +1576,9 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
            if (sizeidx != -1) // Set size
            {
               par_vals.SetDataAndSize(tspec_vals.GetData()+sizeidx*ndofs, ndofs);
-               double min_size = par_vals.Min();//0.001; //
-               if (lim_min_size > 0.)
-               {
-                  min_size = lim_min_size;
-               }
-               else
-               {
-                  MFEM_VERIFY(min_size > 0.0,
-                              "Non-positive size propagated in the target definition.");
-               }
+               const double min_size = par_vals.Min();
+               MFEM_VERIFY(min_size > 0.0,
+                           "Non-positive size propagated in the target definition.");
               const double size = std::max(shape * par_vals, min_size);
               Jtr(q).Set(std::pow(size, 1.0/dim), Jtr(q));
               DenseMatrix Jtrcomp_q(Jtrcomp.GetData(0 + 4*q), dim, dim);
@@ -1702,9 +1593,6 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
               {
                  par_vals.SetDataAndSize(tspec_vals.GetData()+
                                          aspectratioidx*ndofs, ndofs);
-                  const double min_size = par_vals.Min();
-                  MFEM_VERIFY(min_size > 0.0,
-                              "Non-positive aspect-ratio propagated in the target definition.");

                  const double aspectratio = shape * par_vals;
                  D_rho = 0.;
@@ -1889,7 +1777,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

               grad_phys.Mult(par_vals, grad_ptr_c1);
               Vector grad_q(dim);
-               tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
+               tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
               grad_e_c1.MultTranspose(shape, grad_q);

               const double min_size = par_vals.Min();
@@ -1922,7 +1810,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double aspectratio = shape * par_vals;
@@ -1953,7 +1841,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -1992,7 +1880,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double skew = shape * par_vals;
@@ -2025,7 +1913,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -2072,7 +1960,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double theta = shape * par_vals;
@@ -2103,7 +1991,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -2183,7 +2071,7 @@ void DiscreteAdaptTC::UpdateGradientTargetSpecification(const Vector &x,
 {
   if (use_flag && good_tspec_grad) { return; }

-   const int dim = tspec_fesv->GetFE(0)->GetDim(),
+   const int dim = tspec_fes->GetFE(0)->GetDim(),
             cnt = x.Size()/dim;

   tspec_pert1h.SetSize(x.Size()*ncomp);
@@ -2209,7 +2097,7 @@ void DiscreteAdaptTC::UpdateHessianTargetSpecification(const Vector &x,

   if (use_flag && good_tspec_hess) { return; }

-   const int dim    = tspec_fesv->GetFE(0)->GetDim(),
+   const int dim    = tspec_fes->GetFE(0)->GetDim(),
             cnt    = x.Size()/dim,
             totmix = 1+2*(dim-2);

@@ -2257,16 +2145,6 @@ void DiscreteAdaptTC::UpdateHessianTargetSpecification(const Vector &x,
   good_tspec_hess = use_flag;
 }

-DiscreteAdaptTC::~DiscreteAdaptTC()
-{
-   delete tspec_gf;
-   delete adapt_eval;
-   delete tspec_fesv;
-#ifdef MFEM_USE_MPI
-   delete ptspec_fesv;
-#endif
-}
-
 void AdaptivityEvaluator::SetSerialMetaInfo(const Mesh &m,
                                            const FiniteElementCollection &fec,
                                            int num_comp)
@@ -2380,7 +2258,6 @@ void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
                                             AdaptivityEvaluator &ae)
 {
   zeta_0 = &z0;
-   pzeta_0 = &z0;
   delete zeta;
   zeta   = new GridFunction(z0);
   coeff_zeta = &coeff;
@@ -2393,33 +2270,6 @@ void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
 }
 #endif

-void TMOP_Integrator::UpdateAfterMeshTopologyChange()
-{
-   if (zeta)
-   {
-      zeta->Update();
-      adapt_eval->SetSerialMetaInfo(*zeta->FESpace()->GetMesh(),
-                                    *zeta->FESpace()->FEColl(), 1);
-      adapt_eval->SetInitialField
-      (*zeta->FESpace()->GetMesh()->GetNodes(), *zeta);
-   }
-}
-
-#ifdef MFEM_USE_MPI
-void TMOP_Integrator::ParUpdateAfterMeshTopologyChange()
-{
-   if (zeta)
-   {
-      zeta->Update();
-      adapt_eval->SetParMetaInfo(*pzeta_0->ParFESpace()->GetParMesh(),
-                                 *pzeta_0->ParFESpace()->FEColl(), 1);
-      adapt_eval->SetInitialField
-      (*zeta->FESpace()->GetMesh()->GetNodes(), *zeta);
-   }
-}
-#endif
-
-
 double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
                                         ElementTransformation &T,
                                         const Vector &elfun)
@@ -2528,145 +2378,6 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,

   return energy;
 }
-
-double TMOP_Integrator::GetRefinementElementEnergy(const FiniteElement &el,
-                                                   ElementTransformation &T,
-                                                   const Vector &elfun,
-                                                   const IntegrationRule &irule)
-{
-   int dof = el.GetDof(), dim = el.GetDim(),
-       NEsplit = elfun.Size() / (dof*dim), el_id = T.ElementNo;
-   double energy = 0.;
-
-   TargetConstructor *tc = const_cast<TargetConstructor *>(targetC);
-   DiscreteAdaptTC *dtc = dynamic_cast<DiscreteAdaptTC *>(tc);
-   // For DiscreteAdaptTC the GridFunctions used to set the targets must be
-   // mapped onto the fine elements.
-   if (dtc) { dtc->SetTspecFromIntRule(el_id, irule); }
-
-   for (int e = 0; e < NEsplit; e++)
-   {
-      DSh.SetSize(dof, dim);
-      Jrt.SetSize(dim);
-      Jpr.SetSize(dim);
-      Jpt.SetSize(dim);
-      Vector elfun_child(dof*dim);
-      for (int i = 0; i < dof; i++)
-      {
-         for (int d = 0; d < dim; d++)
-         {
-            // elfun is (xe1,xe2,...xen,ye1,ye2...yen) and has nodal coordinates
-            // for all the children element of the parent element being considered.
-            // So we must index and get (xek, yek) i.e. nodal coordinates for
-            // the fine element being considered.
-            elfun_child(i + d*dof) = elfun(i + e*dof + d*dof*NEsplit);
-         }
-      }
-      PMatI.UseExternalData(elfun_child.GetData(), dof, dim);
-
-      const IntegrationRule &ir = EnergyIntegrationRule(el);
-
-      double el_energy = 0;
-      DenseTensor Jtr(dim, dim, ir.GetNPoints());
-      if (dtc)
-      {
-         // This is used to index into the tspec vector inside DiscreteAdaptTC.
-         dtc->SetRefinementSubElement(e);
-      }
-      targetC->ComputeElementTargets(el_id, el, ir, elfun_child, Jtr);
-
-      // Define ref->physical transformation, wn a Coefficient is specified.
-      IsoparametricTransformation *Tpr = NULL;
-      if (coeff1 || coeff0)
-      {
-         Tpr = new IsoparametricTransformation;
-         Tpr->SetFE(&el);
-         Tpr->ElementNo = T.ElementNo;
-         Tpr->ElementType = ElementTransformation::ELEMENT;
-         Tpr->Attribute = T.Attribute;
-         Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
-      }
-
-      for (int i = 0; i < ir.GetNPoints(); i++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         const DenseMatrix &Jtr_i = Jtr(i);
-         h_metric->SetTargetJacobian(Jtr_i);
-         CalcInverse(Jtr_i, Jrt);
-         const double weight = ip.weight * Jtr_i.Det();
-
-         el.CalcDShape(ip, DSh);
-         MultAtB(PMatI, DSh, Jpr);
-         Mult(Jpr, Jrt, Jpt);
-
-         double val = metric_normal * h_metric->EvalW(Jpt);
-         if (coeff1) { val *= coeff1->Eval(*Tpr, ip); }
-
-         el_energy += weight * val;
-         delete Tpr;
-      }
-      energy += el_energy;
-   }
-   energy /= NEsplit;
-
-   if (dtc) { dtc->ResetRefinementTspecData(); }
-
-   return energy;
-}
-
-double TMOP_Integrator::GetDerefinementElementEnergy(const FiniteElement &el,
-                                                     ElementTransformation &T,
-                                                     const Vector &elfun)
-{
-   int dof = el.GetDof(), dim = el.GetDim();
-   double energy = 0.;
-
-   DSh.SetSize(dof, dim);
-   Jrt.SetSize(dim);
-   Jpr.SetSize(dim);
-   Jpt.SetSize(dim);
-   PMatI.UseExternalData(elfun.GetData(), dof, dim);
-
-   const IntegrationRule &ir = EnergyIntegrationRule(el);
-
-   energy = 0.0;
-   DenseTensor Jtr(dim, dim, ir.GetNPoints());
-   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
-
-   // Define ref->physical transformation, wn a Coefficient is specified.
-   IsoparametricTransformation *Tpr = NULL;
-   if (coeff1)
-   {
-      Tpr = new IsoparametricTransformation;
-      Tpr->SetFE(&el);
-      Tpr->ElementNo = T.ElementNo;
-      Tpr->ElementType = ElementTransformation::ELEMENT;
-      Tpr->Attribute = T.Attribute;
-      Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
-   }
-
-   for (int i = 0; i < ir.GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
-      const DenseMatrix &Jtr_i = Jtr(i);
-      h_metric->SetTargetJacobian(Jtr_i);
-      CalcInverse(Jtr_i, Jrt);
-      const double weight = ip.weight * Jtr_i.Det();
-
-      el.CalcDShape(ip, DSh);
-      MultAtB(PMatI, DSh, Jpr);
-      Mult(Jpr, Jrt, Jpt);
-
-      double val = metric_normal * h_metric->EvalW(Jpt);
-      if (coeff1) { val *= coeff1->Eval(*Tpr, ip); }
-
-      energy += weight * val;
-   }
-
-   delete Tpr;
-   return energy;
-}
-
 void TMOP_Integrator::AssembleElementVector(const FiniteElement &el,
                                            ElementTransformation &T,
                                            const Vector &elfun, Vector &elvect)
@@ -3328,7 +3039,7 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
   dx = detv_avg_min / dxscale;
 }

-void TMOP_Integrator::UpdateAfterMeshPositionChange(const Vector &new_x)
+void TMOP_Integrator::UpdateAfterMeshChange(const Vector &new_x)
 {
   if (discr_tc)
   {
@@ -3457,32 +3168,6 @@ void TMOPComboIntegrator::AssembleElementGrad(const FiniteElement &el,
   }
 }

-double TMOPComboIntegrator::GetRefinementElementEnergy(const FiniteElement &el,
-                                                       ElementTransformation &T,
-                                                       const Vector &elfun,
-                                                       const IntegrationRule &irule)
-{
-   double energy= 0.0;
-   for (int i = 0; i < tmopi.Size(); i++)
-   {
-      energy += tmopi[i]->GetRefinementElementEnergy(el, T, elfun, irule);
-   }
-   return energy;
-}
-
-double TMOPComboIntegrator::GetDerefinementElementEnergy(
-   const FiniteElement &el,
-   ElementTransformation &T,
-   const Vector &elfun)
-{
-   double energy= 0.0;
-   for (int i = 0; i < tmopi.Size(); i++)
-   {
-      energy += tmopi[i]->GetDerefinementElementEnergy(el, T, elfun);
-   }
-   return energy;
-}
-
 void TMOPComboIntegrator::EnableNormalization(const GridFunction &x)
 {
   const int cnt = tmopi.Size();
@@ -1057,31 +1057,14 @@ protected:
   // eta1(x+h,y), eta2(x+h,y) ... etan(x+h,y), eta1(x,y+h), eta2(x,y+h) ...
   // same for tspec_pert2h and tspec_pertmix.

-   // DenseMatrix to hold target_spec values for the (children of the)
-   // element being refined to consider for h-refinement.
-   DenseMatrix tspec_refine;
-   // Vector to hold the target_spec values for the coarse version of the
-   // current mesh. Used for derefinement decision with hr-adaptivity.
-   Vector tspec_derefine;
-
   // Components of Target Jacobian at each quadrature point of an element. This
   // is required for computation of the derivative using chain rule.
   mutable DenseTensor Jtrcomp;

   // Note: do not use the Nodes of this space as they may not be on the
   // positions corresponding to the values of tspec.
-   FiniteElementSpace *tspec_fesv;         //owned
-   FiniteElementSpace *coarse_tspec_fesv;  //not owned, derefinement FESpace
-   GridFunction *tspec_gf;                 //owned, uses tspec and tspec_fes
-   // discrete adaptivity
-#ifdef MFEM_USE_MPI
-   ParFiniteElementSpace *ptspec_fesv;     //owned, needed for derefinement to
-   // get update operator.
-   ParGridFunction *tspec_pgf;             // similar to tspec_gf
-#endif
-
-   int amr_el;
-   double lim_min_size;
+   const FiniteElementSpace *tspec_fes;
+   const FiniteElementSpace *tspec_fesv;

   // These flags can be used by outside functions to avoid recomputing the
   // tspec and tspec_perth fields again on the same mesh.
@@ -1093,7 +1076,7 @@ protected:

   void SetDiscreteTargetBase(const GridFunction &tspec_);
   void SetTspecAtIndex(int idx, const GridFunction &tspec_);
-   void FinalizeSerialDiscreteTargetSpec(const GridFunction &tspec_);
+   void FinalizeSerialDiscreteTargetSpec();
 #ifdef MFEM_USE_MPI
   void SetTspecAtIndex(int idx, const ParGridFunction &tspec_);
   void FinalizeParDiscreteTargetSpec(const ParGridFunction &tspec_);
@@ -1105,16 +1088,16 @@ public:
        ncomp(0),
        sizeidx(-1), skewidx(-1), aspectratioidx(-1), orientationidx(-1),
        tspec(), tspec_sav(), tspec_pert1h(), tspec_pert2h(), tspec_pertmix(),
-        tspec_refine(), tspec_derefine(),
-        tspec_fesv(NULL), coarse_tspec_fesv(NULL), tspec_gf(NULL),
-#ifdef MFEM_USE_MPI
-        ptspec_fesv(NULL), tspec_pgf(NULL),
-#endif
-        amr_el(-1), lim_min_size(-0.1),
+        tspec_fes(NULL), tspec_fesv(NULL),
        good_tspec(false), good_tspec_grad(false), good_tspec_hess(false),
        adapt_eval(NULL) { }

-   virtual ~DiscreteAdaptTC();
+   virtual ~DiscreteAdaptTC()
+   {
+      delete adapt_eval;
+      delete tspec_fes;
+      delete tspec_fesv;
+   }

   /** @name Target specification methods.
       The following methods are used to specify geometric parameters of the
@@ -1145,20 +1128,6 @@ public:
   void ResetUpdateFlags()
   { good_tspec = good_tspec_grad = good_tspec_hess = false; }

-   /// Get one of the discrete fields from tspec.
-   void GetDiscreteTargetSpec(GridFunction &tspec_, int idx);
-   /// Get the FESpace associated with tspec.
-   FiniteElementSpace *GetTSpecFESpace() { return tspec_fesv; }
-   /// Get the entire tspec.
-   GridFunction *GetTSpecData() { return tspec_gf; }
-   /// Update all discrete fields based on tspec and update for AMR
-   void UpdateAfterMeshTopologyChange();
-
-#ifdef MFEM_USE_MPI
-   ParFiniteElementSpace *GetTSpecParFESpace() { return ptspec_fesv; }
-   void ParUpdateAfterMeshTopologyChange();
-#endif
-
   /** Used to update the target specification after the mesh has changed. The
       new mesh positions are given by new_x. If @a use_flags is true, repeated
       calls won't do anything until ResetUpdateFlags() is called. */
@@ -1215,36 +1184,6 @@ public:
                                              const Vector &elfun,
                                              IsoparametricTransformation &Tpr,
                                              DenseTensor &dJtr) const;
-
-   // Generates tspec_vals for target construction using intrule
-   // Used for the refinement component in hr-adaptivity.
-   void SetTspecFromIntRule(int e_id, const IntegrationRule &intrule);
-
-   // Targets based on discrete functions can result in invalid (negative)
-   // size at the quadrature points. This method can be used to set a
-   // minimum target size.
-   void SetMinSizeForTargets(double min_size_) { lim_min_size = min_size_; }
-
-   /// Computes target specification data with respect to the coarse FE space.
-   void SetTspecDataForDerefinement(FiniteElementSpace *fes);
-
-   // Reset refinement data associated with h-adaptivity component.
-   void ResetRefinementTspecData()
-   {
-      tspec_refine.Clear();
-      amr_el = -1;
-   }
-
-   // Reset derefinement data associated with h-adaptivity component.
-   void ResetDerefinementTspecData()
-   {
-      tspec_derefine.Destroy();
-      coarse_tspec_fesv = NULL;
-   }
-
-   // Used to specify the fine element for determining energy of children of a
-   // parent element.
-   void SetRefinementSubElement(int amr_el_) { amr_el = amr_el_; }
 };

 class TMOPNewtonSolver;
@@ -1262,7 +1201,6 @@ protected:
   friend class TMOPNewtonSolver;
   friend class TMOPComboIntegrator;

-   TMOP_QualityMetric *h_metric;
   TMOP_QualityMetric *metric;        // not owned
   const TargetConstructor *targetC;  // not owned

@@ -1289,9 +1227,6 @@ protected:

   // Adaptive limiting.
   const GridFunction *zeta_0;       // Not owned.
-#ifdef MFEM_USE_MPI
-   const ParGridFunction *pzeta_0;
-#endif
   GridFunction *zeta;               // Owned. Updated by adapt_eval.
   Coefficient *coeff_zeta;          // Not owned.
   AdaptivityEvaluator *adapt_eval;  // Not owned.
@@ -1402,7 +1337,7 @@ protected:
 #endif
   void ComputeMinJac(const Vector &x, const FiniteElementSpace &fes);

-   void UpdateAfterMeshPositionChange(const Vector &new_x);
+   void UpdateAfterMeshChange(const Vector &new_x);

   void DisableLimiting()
   {
@@ -1460,13 +1395,11 @@ protected:
   void ComputeAllElementTargets(const Vector &xe = Vector()) const;

 public:
-   /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
-       @param[in] tc   Target-matrix construction algorithm to use (not owned).
-       @param[in] hm   TMOP_QualityMetric for h-adaptivity (not owned). */
-   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc,
-                   TMOP_QualityMetric *hm)
-      : h_metric(hm), metric(m), targetC(tc), IntegRules(NULL),
-        integ_order(-1), coeff1(NULL), metric_normal(1.0),
+   /** @param[in] m  TMOP_QualityMetric that will be integrated (not owned).
+       @param[in] tc Target-matrix construction algorithm to use (not owned). */
+   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
+      : metric(m), targetC(tc), IntegRules(NULL), integ_order(-1),
+        coeff1(NULL), metric_normal(1.0),
        nodes0(NULL), coeff0(NULL),
        lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
        zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
@@ -1474,9 +1407,6 @@ public:
        fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
   { PA.enabled = false; }

-   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
-      : TMOP_Integrator(m, tc, m) { }
-
   ~TMOP_Integrator();

   /// Release the device memory of large PA allocations. This will copy device
@@ -1548,22 +1478,6 @@ public:
                                   ElementTransformation &T,
                                   const Vector &elfun);

-   /** @brief Computes the mean of the energies of the given element's children.
-
-       In addition to the inputs for GetElementEnergy, this function requires an
-       IntegrationRule to be specified that will give the decomposition of the
-       given element based on the refinement type being considered. */
-   virtual double GetRefinementElementEnergy(const FiniteElement &el,
-                                             ElementTransformation &T,
-                                             const Vector &elfun,
-                                             const IntegrationRule &irule);
-
-   /// This function is similar to GetElementEnergy, but ignores components
-   /// such as limiting etc. to compute the element energy.
-   virtual double GetDerefinementElementEnergy(const FiniteElement &el,
-                                               ElementTransformation &T,
-                                               const Vector &elfun);
-
   virtual void AssembleElementVector(const FiniteElement &el,
                                      ElementTransformation &T,
                                      const Vector &elfun, Vector &elvect);
@@ -1572,13 +1486,6 @@ public:
                                    ElementTransformation &T,
                                    const Vector &elfun, DenseMatrix &elmat);

-   TMOP_QualityMetric &GetAMRQualityMetric() { return *h_metric; }
-
-   void UpdateAfterMeshTopologyChange();
-#ifdef MFEM_USE_MPI
-   void ParUpdateAfterMeshTopologyChange();
-#endif
-
   // PA extension
   using NonlinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace&);
@@ -1657,15 +1564,6 @@ public:
                                    ElementTransformation &T,
                                    const Vector &elfun, DenseMatrix &elmat);

-   virtual double GetRefinementElementEnergy(const FiniteElement &el,
-                                             ElementTransformation &T,
-                                             const Vector &elfun,
-                                             const IntegrationRule &irule);
-
-   virtual double GetDerefinementElementEnergy(const FiniteElement &el,
-                                               ElementTransformation &T,
-                                               const Vector &elfun);
-
   /// Normalization factor that considers all integrators in the combination.
   void EnableNormalization(const GridFunction &x);
 #ifdef MFEM_USE_MPI
@@ -1,896 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "tmop_amr.hpp"
-
-namespace mfem
-{
-
-using namespace mfem;
-
-void TMOPRefinerEstimator::ComputeEstimates()
-{
-   bool iso = false;
-   bool aniso = false;
-   if (amrmetric == 1 || amrmetric == 2 || amrmetric == 58)
-   {
-      aniso = true;
-   }
-   if (amrmetric == 55 || amrmetric == 56 || amrmetric == 77 ||
-       amrmetric == 315 || amrmetric == 316 || amrmetric == 321)
-   {
-      iso = true;
-   }
-   if (amrmetric == 7 || amrmetric == 9)
-   {
-      iso = true; aniso = true;
-   }
-
-   MFEM_VERIFY(iso || aniso, "Metric type not supported in hr-adaptivity.");
-
-   const int dim = mesh->Dimension();
-   const int num_ref_types = 3 + 4*(dim-2);
-   const int NEorig = mesh->GetNE();
-
-   aniso_flags.SetSize(NEorig);
-   error_estimates.SetSize(NEorig);
-   Vector amr_base_energy(NEorig), amr_temp_energy(NEorig);
-   error_estimates = 1.*std::numeric_limits<float>::max();
-   aniso_flags = -1;
-   GetTMOPRefinementEnergy(0, amr_base_energy);
-
-   for (int i = 1; i < num_ref_types+1; i++)
-   {
-      if ( dim == 2 && i < 3  && aniso != true ) { continue; }
-      if ( dim == 2 && i == 3 && iso   != true ) { continue; }
-      if ( dim == 3 && i < 7  && aniso != true ) { continue; }
-      if ( dim == 3 && i == 7 && iso   != true ) { continue; }
-
-      GetTMOPRefinementEnergy(i, amr_temp_energy);
-
-      for (int e = 0; e < NEorig; e++)
-      {
-         if ( amr_temp_energy(e) < error_estimates(e) )
-         {
-            error_estimates(e) = amr_temp_energy(e);
-            aniso_flags[e] = i;
-         }
-      }
-   }
-   error_estimates *= energy_scaling_factor;
-
-   if (spat_gf)
-   {
-      L2_FECollection avg_fec(0, mesh->Dimension());
-      FiniteElementSpace avg_fes(spat_gf->FESpace()->GetMesh(), &avg_fec);
-      GridFunction elem_avg(&avg_fes);
-      spat_gf->GetElementAverages(elem_avg);
-      for (int i = 0; i < amr_base_energy.Size(); i++)
-      {
-         if (elem_avg(i) < spat_gf_critical) { amr_base_energy(i) = 0.; }
-      }
-   }
-
-   error_estimates -= amr_base_energy;
-   error_estimates *= -1; // error = E(parent) - scaling_factor*mean(E(children))
-   current_sequence = mesh->GetSequence();
-}
-
-void TMOPRefinerEstimator::GetTMOPRefinementEnergy(int reftype,
-                                                   Vector &el_energy_vec)
-{
-   const FiniteElementSpace *fes = mesh->GetNodalFESpace();
-   const int NE = fes->GetNE();
-   GridFunction *xdof = mesh->GetNodes();
-   xdof->SetTrueVector();
-   xdof->SetFromTrueVector();
-
-   el_energy_vec.SetSize(NE);
-   el_energy_vec = std::numeric_limits<float>::max();
-
-   for (int e = 0; e < NE; e++)
-   {
-      Geometry::Type gtype = fes->GetFE(e)->GetGeomType();
-      DenseMatrix tr, xsplit;
-      IntegrationRule *irule = NULL;
-
-      if ( (gtype == Geometry::TRIANGLE && reftype > 0 && reftype < 3) ||
-           (gtype == Geometry::CUBE && reftype > 0 && reftype < 7) ||
-           (gtype == Geometry::TETRAHEDRON && reftype > 0 && reftype < 7) )
-      {
-         continue;
-      }
-
-      switch (gtype)
-      {
-         case Geometry::TRIANGLE:
-         {
-            int ref_access = reftype == 0 ? 0 : 1;
-            xdof->GetVectorValues(e, *TriIntRule[ref_access], xsplit, tr);
-            irule = TriIntRule[ref_access];
-            break;
-         }
-         case Geometry::TETRAHEDRON:
-         {
-            int ref_access = reftype == 0 ? 0 : 1;
-            xdof->GetVectorValues(e, *TetIntRule[ref_access], xsplit, tr);
-            irule = TetIntRule[ref_access];
-            break;
-         }
-         case Geometry::SQUARE:
-         {
-            MFEM_VERIFY(QuadIntRule[reftype], " Integration rule does not exist.");
-            xdof->GetVectorValues(e, *QuadIntRule[reftype], xsplit, tr);
-            irule = QuadIntRule[reftype];
-            break;
-         }
-         case Geometry::CUBE:
-         {
-            int ref_access = reftype == 0 ? 0 : 1;
-            xdof->GetVectorValues(e, *HexIntRule[ref_access], xsplit, tr);
-            irule = HexIntRule[ref_access];
-            break;
-         }
-         default:
-            MFEM_ABORT("Incompatible geometry type!");
-      }
-      xsplit.Transpose();
-
-      el_energy_vec(e) = 0.; // Re-set to 0
-
-      // The data format is xe1,xe2,..xen,ye1,ye2..yen.
-      // We will reformat it inside GetRefinementElementEnergy
-      Vector elfun(xsplit.GetData(), xsplit.NumCols()*xsplit.NumRows());
-
-      Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
-      TMOP_Integrator *ti  = NULL;
-      TMOPComboIntegrator *co = NULL;
-      for (int i = 0; i < integs.Size(); i++)
-      {
-         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-         if (ti)
-         {
-            el_energy_vec(e) = ti->GetRefinementElementEnergy(*fes->GetFE(e),
-                                                              *mesh->GetElementTransformation(e),
-                                                              elfun,
-                                                              *irule);
-         }
-         co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
-         if (co)
-         {
-            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
-            for (int j = 0; j < ati.Size(); j++)
-            {
-               el_energy_vec(e) += ati[j]->GetRefinementElementEnergy(*fes->GetFE(e),
-                                                                      *mesh->GetElementTransformation(e),
-                                                                      elfun,
-                                                                      *irule);
-            }
-         }
-      }
-   }
-}
-
-void TMOPRefinerEstimator::SetHexIntRules()
-{
-   HexIntRule.SetSize(1+1);
-   // Reftype = 0 -> original element
-   Mesh meshsplit = Mesh::MakeCartesian3D(1, 1, 1, Element::HEXAHEDRON);
-   Mesh base_mesh_copy(meshsplit);
-   HexIntRule[0] = SetIntRulesFromMesh(meshsplit);
-   meshsplit.Clear();
-
-   // Reftype = 7
-   for (int i = 7; i < 8; i++)
-   {
-      Array<Refinement> marked_elements;
-      Mesh mesh_ref(base_mesh_copy);
-      for (int e = 0; e < mesh_ref.GetNE(); e++)
-      {
-         marked_elements.Append(Refinement(e, i));
-      }
-      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
-      HexIntRule[1] = SetIntRulesFromMesh(mesh_ref);
-      mesh_ref.Clear();
-   }
-}
-
-void TMOPRefinerEstimator::SetQuadIntRules()
-{
-   QuadIntRule.SetSize(3+1);
-
-   // Reftype = 0 -> original element
-   Mesh meshsplit = Mesh::MakeCartesian2D(1, 1, Element::QUADRILATERAL);
-   Mesh base_mesh_copy(meshsplit);
-   QuadIntRule[0] = SetIntRulesFromMesh(meshsplit);
-   meshsplit.Clear();
-
-   // Reftype = 1-3
-   for (int i = 1; i < 4; i++)
-   {
-      Array<Refinement> marked_elements;
-      Mesh mesh_ref(base_mesh_copy);
-      for (int e = 0; e < mesh_ref.GetNE(); e++)
-      {
-         marked_elements.Append(Refinement(e, i));
-      }
-      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
-      QuadIntRule[i] = SetIntRulesFromMesh(mesh_ref);
-      mesh_ref.Clear();
-   }
-}
-
-void TMOPRefinerEstimator::SetTriIntRules()
-{
-   TriIntRule.SetSize(1+1);
-
-   // Reftype = 0 // original element
-   const int Nvert = 3, NEsplit = 1;
-   Mesh meshsplit(2, Nvert, NEsplit, 0 ,2);
-   const double tri_v[3][2] =
-   {
-      {0, 0}, {1, 0}, {0, 1}
-   };
-   const int tri_e[1][3] =
-   {
-      {0, 1, 2}
-   };
-
-   for (int j = 0; j < Nvert; j++)
-   {
-      meshsplit.AddVertex(tri_v[j]);
-   }
-   meshsplit.AddTriangle(tri_e[0], 1);
-   meshsplit.FinalizeTriMesh(1, 1, true);
-
-   Mesh base_mesh_copy(meshsplit);
-   TriIntRule[0] = SetIntRulesFromMesh(meshsplit);
-   meshsplit.Clear();
-
-   // no anisotropic refinements for triangle
-   // Reftype = 3
-   for (int i = 1; i < 2; i++)
-   {
-      Array<Refinement> marked_elements;
-      Mesh mesh_ref(base_mesh_copy);
-      for (int e = 0; e < mesh_ref.GetNE(); e++)
-      {
-         marked_elements.Append(Refinement(e, i));
-      }
-      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
-      TriIntRule[i] = SetIntRulesFromMesh(mesh_ref);
-      mesh_ref.Clear();
-   }
-}
-
-void TMOPRefinerEstimator::SetTetIntRules()
-{
-   TetIntRule.SetSize(1+1);
-
-   // Reftype = 0 // original element
-   const int Nvert = 4, NEsplit = 1;
-   Mesh meshsplit(3, Nvert, NEsplit, 0, 3);
-   const double tet_v[4][3] =
-   {
-      {0, 0, 0}, {1, 0, 0}, {0, 1, 0}, {0, 0, 1}
-   };
-   const int tet_e[1][4] =
-   {
-      {0, 1, 2, 3}
-   };
-
-   for (int j = 0; j < Nvert; j++)
-   {
-      meshsplit.AddVertex(tet_v[j]);
-   }
-   meshsplit.AddTet(tet_e[0], 1);
-   meshsplit.FinalizeTetMesh(1, 1, true);
-
-   Mesh base_mesh_copy(meshsplit);
-   TetIntRule[0] = SetIntRulesFromMesh(meshsplit);
-   meshsplit.Clear();
-
-   // no anisotropic refinements for triangle
-   // Reftype = 7
-   for (int i = 1; i < 2; i++)
-   {
-      Array<Refinement> marked_elements;
-      Mesh mesh_ref(base_mesh_copy);
-      for (int e = 0; e < mesh_ref.GetNE(); e++)
-      {
-         marked_elements.Append(Refinement(e, i)); //ref_type will default to 7
-      }
-      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
-      TetIntRule[i] = SetIntRulesFromMesh(mesh_ref);
-      mesh_ref.Clear();
-   }
-}
-
-IntegrationRule* TMOPRefinerEstimator::SetIntRulesFromMesh(Mesh &meshsplit)
-{
-   const int dim = meshsplit.Dimension();
-   H1_FECollection fec(order, dim);
-   FiniteElementSpace nodal_fes(&meshsplit, &fec, dim);
-   meshsplit.SetNodalFESpace(&nodal_fes);
-
-   const int NEsplit = meshsplit.GetNE();
-   const int dof_cnt = nodal_fes.GetFE(0)->GetDof(),
-             pts_cnt = NEsplit * dof_cnt;
-
-   DenseMatrix pos(dof_cnt, dim);
-   Vector posV(pos.Data(), dof_cnt * dim);
-   Array<int> xdofs(dof_cnt * dim);
-
-   // Create an IntegrationRule on the nodes of the reference submesh.
-   IntegrationRule *irule = new IntegrationRule(pts_cnt);
-   GridFunction *nodesplit = meshsplit.GetNodes();
-
-   int pt_id = 0;
-   for (int i = 0; i < NEsplit; i++)
-   {
-      nodal_fes.GetElementVDofs(i, xdofs);
-      nodesplit->GetSubVector(xdofs, posV);
-      for (int j = 0; j < dof_cnt; j++)
-      {
-         if (dim == 2)
-         {
-            irule->IntPoint(pt_id).Set2(pos(j, 0), pos(j, 1));
-         }
-         else if (dim == 3)
-         {
-            irule->IntPoint(pt_id).Set3(pos(j, 0), pos(j, 1), pos(j, 2));
-         }
-         pt_id++;
-      }
-   }
-   return irule;
-}
-
-bool TMOPDeRefinerEstimator::GetDerefineEnergyForIntegrator(
-   TMOP_Integrator &tmopi,
-   Vector &fine_energy)
-{
-   DiscreteAdaptTC *tcd = tmopi.GetDiscreteAdaptTC();
-   fine_energy.SetSize(mesh->GetNE());
-
-   if (serial)
-   {
-      Mesh meshcopy(*mesh);
-      FiniteElementSpace *tcdfes = NULL;
-      if (tcd)
-      {
-         tcdfes = new FiniteElementSpace(*tcd->GetTSpecFESpace(), &meshcopy);
-      }
-
-      Vector local_err(meshcopy.GetNE());
-      local_err = 0.;
-      double threshold = std::numeric_limits<float>::max();
-      meshcopy.DerefineByError(local_err, threshold, 0, 1);
-
-      if (meshcopy.GetGlobalNE() == mesh->GetGlobalNE())
-      {
-         delete tcdfes;
-         return false;
-      }
-
-      if (tcd)
-      {
-         tcdfes->Update();
-         tcd->SetTspecDataForDerefinement(tcdfes);
-      }
-
-      Vector coarse_energy(meshcopy.GetNE());
-      GetTMOPDerefinementEnergy(meshcopy, tmopi, coarse_energy);
-      if (tcd) { tcd->ResetDerefinementTspecData(); }
-      GetTMOPDerefinementEnergy(*mesh, tmopi, fine_energy);
-
-      const CoarseFineTransformations &dtrans =
-         meshcopy.ncmesh->GetDerefinementTransforms();
-      Table coarse_to_fine;
-      dtrans.GetCoarseToFineMap(meshcopy, coarse_to_fine);
-
-      for (int pe = 0; pe < coarse_to_fine.Size(); pe++)
-      {
-         Array<int> tabrow;
-         coarse_to_fine.GetRow(pe, tabrow);
-         int nchild = tabrow.Size();
-         double parent_energy = coarse_energy(pe);
-         for (int fe = 0; fe < nchild; fe++)
-         {
-            int child = tabrow[fe];
-            MFEM_VERIFY(child < mesh->GetNE(), " invalid coarse to fine mapping");
-            fine_energy(child) -= parent_energy;
-         }
-      }
-      delete tcdfes;
-   }
-   else
-   {
-#ifdef MFEM_USE_MPI
-      ParMesh meshcopy(*pmesh);
-      ParFiniteElementSpace *tcdfes = NULL;
-      if (tcd)
-      {
-         tcdfes = new ParFiniteElementSpace(*tcd->GetTSpecParFESpace(), meshcopy);
-      }
-
-      Vector local_err(meshcopy.GetNE());
-      local_err = 0.;
-      double threshold = std::numeric_limits<float>::max();
-      meshcopy.DerefineByError(local_err, threshold, 0, 1);
-
-      if (meshcopy.GetGlobalNE() == pmesh->GetGlobalNE())
-      {
-         delete tcdfes;
-         return false;
-      }
-
-      if (tcd)
-      {
-         tcdfes->Update();
-         tcd->SetTspecDataForDerefinement(tcdfes);
-      }
-
-      Vector coarse_energy(meshcopy.GetNE());
-      GetTMOPDerefinementEnergy(meshcopy, tmopi, coarse_energy);
-      if (tcd) { tcd->ResetDerefinementTspecData(); }
-      GetTMOPDerefinementEnergy(*pmesh, tmopi, fine_energy);
-
-      const CoarseFineTransformations &dtrans =
-         meshcopy.pncmesh->GetDerefinementTransforms();
-      Table coarse_to_fine;
-      dtrans.GetCoarseToFineMap(meshcopy, coarse_to_fine);
-
-      for (int pe = 0; pe < meshcopy.GetNE(); pe++)
-      {
-         Array<int> tabrow;
-         coarse_to_fine.GetRow(pe, tabrow);
-         int nchild = tabrow.Size();
-         double parent_energy = coarse_energy(pe);
-         for (int fe = 0; fe < nchild; fe++)
-         {
-            int child = tabrow[fe];
-            MFEM_VERIFY(child < pmesh->GetNE(), " invalid coarse to fine mapping");
-            fine_energy(child) -= parent_energy;
-         }
-      }
-      delete tcdfes;
-#endif
-   }
-
-   // error_estimate(e) = energy(parent_of_e)-energy(e)
-   // Negative energy means derefinement is desirable.
-   fine_energy *= -1;
-   return true;
-}
-
-void TMOPDeRefinerEstimator::ComputeEstimates()
-{
-   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
-   TMOP_Integrator *ti  = NULL;
-   TMOPComboIntegrator *co = NULL;
-   error_estimates.SetSize(mesh->GetNE());
-   error_estimates = 0.;
-   Vector fine_energy(mesh->GetNE());
-
-   for (int i = 0; i < integs.Size(); i++)
-   {
-      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti)
-      {
-         bool deref = GetDerefineEnergyForIntegrator(*ti, fine_energy);
-         if (!deref) { error_estimates = 1; return; }
-         error_estimates += fine_energy;
-      }
-      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
-      if (co)
-      {
-         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
-         for (int j = 0; j < ati.Size(); j++)
-         {
-            bool deref = GetDerefineEnergyForIntegrator(*ati[j], fine_energy);
-            if (!deref) { error_estimates = 1; return; }
-            error_estimates += fine_energy;
-         }
-      }
-   }
-}
-
-void TMOPDeRefinerEstimator::GetTMOPDerefinementEnergy(Mesh &cmesh,
-                                                       TMOP_Integrator &tmopi,
-                                                       Vector &el_energy_vec)
-{
-   const int cNE = cmesh.GetNE();
-   el_energy_vec.SetSize(cNE);
-   const FiniteElementSpace *fespace = cmesh.GetNodalFESpace();
-
-   GridFunction *cxdof = cmesh.GetNodes();
-
-   Array<int> vdofs;
-   Vector el_x;
-   const FiniteElement *fe;
-   ElementTransformation *T;
-
-   for (int j = 0; j < cNE; j++)
-   {
-      fe = fespace->GetFE(j);
-      fespace->GetElementVDofs(j, vdofs);
-      T = cmesh.GetElementTransformation(j);
-      cxdof->GetSubVector(vdofs, el_x);
-      el_energy_vec(j) = tmopi.GetDerefinementElementEnergy(*fe, *T, el_x);
-   }
-}
-
-
-TMOPHRSolver::TMOPHRSolver(Mesh &mesh_, NonlinearForm &nlf_,
-                           TMOPNewtonSolver &tmopns_, GridFunction &x_,
-                           bool move_bnd_, bool hradaptivity_,
-                           int mesh_poly_deg_, int amr_metric_id_,
-                           int hr_iter_, int h_per_r_iter_) :
-   mesh(&mesh_), nlf(&nlf_), tmopns(&tmopns_), x(&x_),
-   gridfuncarr(), fespacearr(),
-   move_bnd(move_bnd_), hradaptivity(hradaptivity_),
-   mesh_poly_deg(mesh_poly_deg_), amr_metric_id(amr_metric_id_),
-   serial(true), hr_iter(hr_iter_), h_per_r_iter(h_per_r_iter_)
-{
-   if (!hradaptivity) { return; }
-   tmop_r_est = new TMOPRefinerEstimator(*mesh, *nlf, mesh_poly_deg,
-                                         amr_metric_id);
-   tmop_r = new ThresholdRefiner(*tmop_r_est);
-   tmop_r->SetTotalErrorFraction(0.0);
-   tmop_r_est->SetEnergyScalingFactor(1.);
-   tmop_dr_est= new TMOPDeRefinerEstimator(*mesh, *nlf);
-   tmop_dr = new ThresholdDerefiner(*tmop_dr_est);
-   AddGridFunctionForUpdate(x);
-}
-
-#ifdef MFEM_USE_MPI
-TMOPHRSolver::TMOPHRSolver(ParMesh &pmesh_, ParNonlinearForm &pnlf_,
-                           TMOPNewtonSolver &tmopns_, ParGridFunction &px_,
-                           bool move_bnd_, bool hradaptivity_,
-                           int mesh_poly_deg_, int amr_metric_id_,
-                           int hr_iter_, int h_per_r_iter_) :
-   mesh(&pmesh_), nlf(&pnlf_), tmopns(&tmopns_), x(&px_),
-   gridfuncarr(), fespacearr(),
-   move_bnd(move_bnd_), hradaptivity(hradaptivity_),
-   mesh_poly_deg(mesh_poly_deg_), amr_metric_id(amr_metric_id_),
-   pmesh(&pmesh_), pnlf(&pnlf_), pgridfuncarr(), pfespacearr(),
-   serial(false), hr_iter(hr_iter_), h_per_r_iter(h_per_r_iter_)
-{
-   if (!hradaptivity) { return; }
-   tmop_r_est = new TMOPRefinerEstimator(*pmesh, *pnlf, mesh_poly_deg,
-                                         amr_metric_id);
-   tmop_r = new ThresholdRefiner(*tmop_r_est);
-   tmop_r->SetTotalErrorFraction(0.0);
-   tmop_r_est->SetEnergyScalingFactor(1.);
-   tmop_dr_est= new TMOPDeRefinerEstimator(*pmesh, *pnlf);
-   tmop_dr = new ThresholdDerefiner(*tmop_dr_est);
-   AddGridFunctionForUpdate(&px_);
-}
-#endif
-
-void TMOPHRSolver::Mult()
-{
-   Vector b(0);
-   int myid = 0;
-   if (serial)
-   {
-      tmopns->SetOperator(*nlf);
-   }
-   else
-   {
-#ifdef MFEM_USE_MPI
-      myid = pnlf->ParFESpace()->GetMyRank();
-      tmopns->SetOperator(*pnlf);
-#endif
-   }
-   if (!hradaptivity)
-   {
-      tmopns->Mult(b, x->GetTrueVector());
-      if (tmopns->GetConverged() == false)
-      {
-         if (myid == 0) { mfem::out << "Nonlinear solver: rtol not achieved.\n"; }
-      }
-      x->SetFromTrueVector();
-      return;
-   }
-
-   bool radaptivity = true;
-
-   tmop_dr->Reset();
-   tmop_r->Reset();
-
-   if (serial)
-   {
-      for (int i_hr = 0; i_hr < hr_iter; i_hr++)
-      {
-         if (!radaptivity)
-         {
-            break;
-         }
-         mfem::out << i_hr << " r-adaptivity iteration.\n";
-
-         tmopns->SetOperator(*nlf);
-         tmopns->Mult(b, x->GetTrueVector());
-         x->SetFromTrueVector();
-
-         mfem::out << "TMOP energy after r-adaptivity: " <<
-                   nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
-                   ", Elements: " << mesh->GetNE() << std::endl;
-
-         for (int i_h = 0; i_h < h_per_r_iter; i_h++)
-         {
-            // Derefinement step.
-            if (mesh->ncmesh)
-            {
-               tmop_dr->Apply(*mesh);
-               Update();
-            }
-            mfem::out << "TMOP energy after derefinement: " <<
-                      nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
-                      ", Elements: " << mesh->GetNE() << std::endl;
-
-            // Refinement step.
-            tmop_r->Apply(*mesh);
-            Update();
-            mfem::out << "TMOP energy after   refinement: " <<
-                      nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
-                      ", Elements: " << mesh->GetNE() << std::endl;
-
-            if (!tmop_dr->Derefined() && tmop_r->Stop())
-            {
-               radaptivity = false;
-               mfem::out << "AMR stopping criterion satisfied. Stop.\n";
-               break;
-            }
-         } //n_h
-      } //n_hr
-   }
-   else
-   {
-#ifdef MFEM_USE_MPI
-      int NEGlob;
-      double tmopenergy;
-      for (int i_hr = 0; i_hr < hr_iter; i_hr++)
-      {
-         if (!radaptivity)
-         {
-            break;
-         }
-         if (myid == 0) { mfem::out << i_hr << " r-adaptivity iteration.\n"; }
-         tmopns->SetOperator(*pnlf);
-         tmopns->Mult(b, x->GetTrueVector());
-         x->SetFromTrueVector();
-
-         NEGlob = pmesh->GetGlobalNE();
-         tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
-         if (myid == 0)
-         {
-            mfem::out << "TMOP energy after r-adaptivity: " << tmopenergy <<
-                      ", Elements: " << NEGlob << std::endl;
-         }
-
-         for (int i_h = 0; i_h < h_per_r_iter; i_h++)
-         {
-            // Derefinement step.
-            if (pmesh->pncmesh)
-            {
-               RebalanceParNCMesh();
-               ParUpdate();
-
-               tmop_dr->Apply(*pmesh);
-               ParUpdate();
-            }
-            NEGlob = pmesh->GetGlobalNE();
-            tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
-            if (myid == 0)
-            {
-               mfem::out << "TMOP energy after derefinement: " << tmopenergy <<
-                         ", Elements: " << NEGlob << std::endl;
-            }
-
-            // Refinement step.
-            tmop_r->Apply(*pmesh);
-            ParUpdate();
-            NEGlob = pmesh->GetGlobalNE();
-            tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
-            if (myid == 0)
-            {
-               mfem::out << "TMOP energy after   refinement: " << tmopenergy <<
-                         ", Elements: " << NEGlob << std::endl;
-            }
-
-            if (!tmop_dr->Derefined() && tmop_r->Stop())
-            {
-               radaptivity = false;
-               if (myid == 0)
-               {
-                  mfem::out << "AMR stopping criterion satisfied. Stop.\n";
-               }
-               break;
-            }
-         } //n_r limit
-      } //n_hr
-#endif
-   }
-}
-
-#ifdef MFEM_USE_MPI
-void TMOPHRSolver::RebalanceParNCMesh()
-{
-   ParNCMesh *pncmesh = pmesh->pncmesh;
-   if (pncmesh)
-   {
-      const Table &dreftable = pncmesh->GetDerefinementTable();
-      Array<int> drefs, new_ranks;
-      for (int i = 0; i < dreftable.Size(); i++)
-      {
-         drefs.Append(i);
-      }
-      pncmesh->GetFineToCoarsePartitioning(drefs, new_ranks);
-      pmesh->Rebalance(new_ranks);
-   }
-}
-#endif
-
-void TMOPHRSolver::Update()
-{
-   // Update FESpace
-   for (int i = 0; i < fespacearr.Size(); i++)
-   {
-      fespacearr[i]->Update();
-   }
-   // Update nodal GF
-   for (int i = 0; i < gridfuncarr.Size(); i++)
-   {
-      gridfuncarr[i]->Update();
-      gridfuncarr[i]->SetTrueVector();
-      gridfuncarr[i]->SetFromTrueVector();
-   }
-
-   // Update Discrete Indicator for all the TMOP_Integrators in NonLinearForm
-   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
-   TMOP_Integrator *ti  = NULL;
-   TMOPComboIntegrator *co = NULL;
-   DiscreteAdaptTC *dtc = NULL;
-   for (int i = 0; i < integs.Size(); i++)
-   {
-      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti)
-      {
-         ti->UpdateAfterMeshTopologyChange();
-         dtc = ti->GetDiscreteAdaptTC();
-         if (dtc) { dtc->UpdateAfterMeshTopologyChange(); }
-      }
-      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
-      if (co)
-      {
-         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
-         for (int j = 0; j < ati.Size(); j++)
-         {
-            ati[j]->UpdateAfterMeshTopologyChange();
-            dtc = ati[j]->GetDiscreteAdaptTC();
-            if (dtc) { dtc->UpdateAfterMeshTopologyChange(); }
-         }
-      }
-   }
-
-   // Update the Nonlinear form and set Essential BC.
-   UpdateNonlinearFormAndBC(mesh, nlf);
-}
-
-#ifdef MFEM_USE_MPI
-void TMOPHRSolver::ParUpdate()
-{
-   // Update FESpace
-   for (int i = 0; i < pfespacearr.Size(); i++)
-   {
-      pfespacearr[i]->Update();
-   }
-   // Update nodal GF
-   for (int i = 0; i < pgridfuncarr.Size(); i++)
-   {
-      pgridfuncarr[i]->Update();
-      pgridfuncarr[i]->SetTrueVector();
-      pgridfuncarr[i]->SetFromTrueVector();
-   }
-
-   // Update Discrete Indicator
-   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
-   TMOP_Integrator *ti  = NULL;
-   TMOPComboIntegrator *co = NULL;
-   DiscreteAdaptTC *dtc = NULL;
-   for (int i = 0; i < integs.Size(); i++)
-   {
-      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti)
-      {
-         ti->ParUpdateAfterMeshTopologyChange();
-         dtc = ti->GetDiscreteAdaptTC();
-         if (dtc) { dtc->ParUpdateAfterMeshTopologyChange(); }
-      }
-      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
-      if (co)
-      {
-         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
-         for (int j = 0; j < ati.Size(); j++)
-         {
-            ati[j]->ParUpdateAfterMeshTopologyChange();
-            dtc = ati[j]->GetDiscreteAdaptTC();
-            if (dtc) { dtc->ParUpdateAfterMeshTopologyChange(); }
-         }
-      }
-   }
-
-   // Update the Nonlinear form and set Essential BC.
-   UpdateNonlinearFormAndBC(pmesh, pnlf);
-}
-#endif
-
-void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
-{
-   const FiniteElementSpace &fes = *mesh->GetNodalFESpace();
-
-   // Update Nonlinear form and Set Essential BC
-   nlf->Update();
-   const int dim = fes.GetFE(0)->GetDim();
-   if (move_bnd == false)
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      nlf->SetEssentialBC(ess_bdr);
-   }
-   else
-   {
-      const int nd  = fes.GetBE(0)->GetDof();
-      int n = 0;
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         const int attr = mesh->GetBdrElement(i)->GetAttribute();
-         MFEM_VERIFY(!(dim == 2 && attr == 3),
-                     "Boundary attribute 3 must be used only for 3D meshes. "
-                     "Adjust the attributes (1/2/3/4 for fixed x/y/z/all "
-                     "components, rest for free nodes), or use -fix-bnd.");
-         if (attr == 1 || attr == 2 || attr == 3) { n += nd; }
-         if (attr == 4) { n += nd * dim; }
-      }
-      Array<int> ess_vdofs(n), vdofs;
-      n = 0;
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         const int attr = mesh->GetBdrElement(i)->GetAttribute();
-         fes.GetBdrElementVDofs(i, vdofs);
-         if (attr == 1) // Fix x components.
-         {
-            for (int j = 0; j < nd; j++)
-            { ess_vdofs[n++] = vdofs[j]; }
-         }
-         else if (attr == 2) // Fix y components.
-         {
-            for (int j = 0; j < nd; j++)
-            { ess_vdofs[n++] = vdofs[j+nd]; }
-         }
-         else if (attr == 3) // Fix z components.
-         {
-            for (int j = 0; j < nd; j++)
-            { ess_vdofs[n++] = vdofs[j+2*nd]; }
-         }
-         else if (attr == 4) // Fix all components.
-         {
-            for (int j = 0; j < vdofs.Size(); j++)
-            { ess_vdofs[n++] = vdofs[j]; }
-         }
-      }
-      nlf->SetEssentialVDofs(ess_vdofs);
-   }
-}
-
-}
@@ -1,284 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_TMOP_AMR_HPP
-#define MFEM_TMOP_AMR_HPP
-
-#include "tmop_tools.hpp"
-#include "nonlinearform.hpp"
-#include "pnonlinearform.hpp"
-#include "estimators.hpp"
-#include "../mesh/mesh_operators.hpp"
-
-namespace mfem
-{
-
-class TMOPRefinerEstimator : public AnisotropicErrorEstimator
-{
-protected:
-   Mesh *mesh; // not owned
-   NonlinearForm *nlf; // not owned
-   int order;
-   int amrmetric;
-   Array<IntegrationRule *> TriIntRule, QuadIntRule, TetIntRule, HexIntRule;
-   long current_sequence;
-   Vector error_estimates;
-   Array<int> aniso_flags;
-   // An element is refined only if
-   // [mean TMOPEnergy(children)]*energy_scaling_factor < TMOPEnergy(parent)
-   double energy_scaling_factor;
-   GridFunction *spat_gf;   // If specified, can be used to specify the
-   double spat_gf_critical; // region where hr-adaptivity is done.
-
-   /// Check if the mesh of the solution was modified.
-   bool MeshIsModified()
-   {
-      long mesh_sequence = mesh->GetSequence();
-      MFEM_ASSERT(mesh_sequence >= current_sequence, "");
-      return (mesh_sequence > current_sequence);
-   }
-
-   /// Compute the element error estimates. For an element E in the mesh,
-   /// error(E) = TMOPEnergy(E)*energy_scaling_factor-Mean(TMOPEnergy(ChildofE)),
-   /// where TMOPEnergy of Children of E is obtained by assuming the element E
-   /// is refined using the refinement type being considered based on the TMOP
-   /// mesh quality metric.
-   void ComputeEstimates();
-
-   /// Construct the integration rules to model how each element type is split
-   /// using different refinement types. ref_type = 0 is the original element
-   /// and reftype \ in [1, 7] represent different refinement type based on
-   /// NCMesh class.
-   void SetQuadIntRules(); // supports ref_type = 1 to 3.
-   void SetTriIntRules(); // currently supports only isotropic refinement.
-   void SetHexIntRules(); // currently supports only isotropic refinement.
-   void SetTetIntRules(); // currently supports only isotropic refinement.
-
-   /// Get TMOP energy for each element corresponding to the refinement type
-   /// specified.
-   void GetTMOPRefinementEnergy(int reftype, Vector &el_energy_vec);
-
-   /// Use a mesh to setup an integration rule that will mimic the different
-   /// refinement types.
-   IntegrationRule* SetIntRulesFromMesh(Mesh &meshsplit);
-public:
-   TMOPRefinerEstimator(Mesh &mesh_, NonlinearForm &nlf_, int order_,
-                        int amrmetric_) :
-      mesh(&mesh_), nlf(&nlf_), order(order_), amrmetric(amrmetric_),
-      TriIntRule(0), QuadIntRule(0), TetIntRule(0), HexIntRule(0),
-      current_sequence(-1), error_estimates(), aniso_flags(),
-      energy_scaling_factor(1.), spat_gf(NULL), spat_gf_critical(0.)
-   {
-      if (mesh->Dimension() == 2)
-      {
-         SetQuadIntRules();
-         SetTriIntRules();
-      }
-      else
-      {
-         SetHexIntRules();
-         SetTetIntRules();
-      }
-   }
-
-   ~TMOPRefinerEstimator()
-   {
-      for (int i = 0; i < QuadIntRule.Size(); i++) { delete QuadIntRule[i]; }
-      for (int i = 0; i < TriIntRule.Size();  i++) { delete TriIntRule[i]; }
-      for (int i = 0; i < HexIntRule.Size();  i++) { delete HexIntRule[i]; }
-      for (int i = 0; i < TetIntRule.Size();  i++) { delete TetIntRule[i]; }
-   }
-
-   /// Get TMOP-based errors for each element in the mesh computed based on the
-   /// refinement types being considered.
-   virtual const Vector &GetLocalErrors()
-   {
-      if (MeshIsModified()) { ComputeEstimates(); }
-      return error_estimates;
-   }
-   /// For anisotropic refinements, get the refinement type (e.g., x or y)
-   virtual const Array<int> &GetAnisotropicFlags()
-   {
-      if (MeshIsModified()) { ComputeEstimates(); }
-      return aniso_flags;
-   }
-
-   /// Scaling factor for the TMOP refinement energy. An element is refined if
-   /// [mean TMOPEnergy(children)]*energy_scaling_factor < TMOPEnergy(parent)
-   void SetEnergyScalingFactor(double scale) { energy_scaling_factor = scale; }
-
-   /// Spatial indicator function (eta) that can be used to prevent elements
-   /// from being refined even if the energy criterion is met. Using this,
-   /// an element E is not refined if mean(@a spat_gf(E)) < @a spat_gf_critical.
-   void SetSpatialIndicator(GridFunction &spat_gf_,
-                            double spat_gf_critical_ = 0.5)
-   { spat_gf = &spat_gf_; spat_gf_critical = spat_gf_critical_; }
-   void SetSpatialIndicatorCritical(double val_) { spat_gf_critical = val_; }
-
-   /// Reset the error estimator.
-   virtual void Reset() { current_sequence = -1; }
-};
-
-class TMOPDeRefinerEstimator : public ErrorEstimator
-{
-protected:
-   Mesh *mesh;
-   NonlinearForm *nlf;
-#ifdef MFEM_USE_MPI
-   ParMesh *pmesh;
-   ParNonlinearForm *pnlf;
-#endif
-   int order;
-   int amrmetric;
-   long current_sequence;
-   Vector error_estimates;
-   bool serial;
-
-   /// Check if the mesh of the solution was modified.
-   bool MeshIsModified()
-   {
-      long mesh_sequence = mesh->GetSequence();
-      MFEM_ASSERT(mesh_sequence >= current_sequence, "");
-      return (mesh_sequence > current_sequence);
-   }
-
-   /// Compute the element error estimates. For a given element E in the mesh,
-   /// error(E) = TMOPEnergy(parent_of_E)-TMOPEnergy(E). Children element of an
-   /// element are derefined if the mean TMOP energy of children is greated than
-   /// the TMOP energy associated with their parent.
-   void ComputeEstimates();
-
-   void GetTMOPDerefinementEnergy(Mesh &cmesh,
-                                  TMOP_Integrator &tmopi,
-                                  Vector &el_energy_vec);
-
-   bool GetDerefineEnergyForIntegrator(TMOP_Integrator &tmopi,
-                                       Vector &fine_energy);
-public:
-   TMOPDeRefinerEstimator(Mesh &mesh_, NonlinearForm &nlf_) :
-      mesh(&mesh_), nlf(&nlf_),
-      current_sequence(-1), error_estimates(), serial(true)   { }
-#ifdef MFEM_USE_MPI
-   TMOPDeRefinerEstimator(ParMesh &pmesh_, ParNonlinearForm &pnlf_) :
-      mesh(&pmesh_), nlf(&pnlf_), pmesh(&pmesh_), pnlf(&pnlf_),
-      current_sequence(-1), error_estimates(), serial(false) { }
-#endif
-
-   ~TMOPDeRefinerEstimator() { }
-
-   virtual const Vector &GetLocalErrors()
-   {
-      if (MeshIsModified()) { ComputeEstimates(); }
-      return error_estimates;
-   }
-
-   /// Reset the error estimator.
-   virtual void Reset() { current_sequence = -1; }
-};
-
-// hr-adaptivity using TMOP.
-// If hr-adaptivity is disabled, r-adaptivity is done once using the
-// TMOPNewtonSolver.
-// Otherwise, "hr_iter" iterations of r-adaptivity are done followed by
-// "h_per_r_iter" iterations of h-adaptivity after each r-adaptivity iteration.
-// The solver terminates early if an h-adaptivity iteration does not
-// refine/derefine any element in the mesh.
-class TMOPHRSolver
-{
-protected:
-   Mesh *mesh;
-   NonlinearForm *nlf;
-   TMOPNewtonSolver *tmopns;
-   GridFunction *x;
-   Array<GridFunction *> gridfuncarr;
-   Array<FiniteElementSpace *> fespacearr;
-   bool move_bnd, hradaptivity;
-   const int mesh_poly_deg, amr_metric_id;
-#ifdef MFEM_USE_MPI
-   ParMesh *pmesh;
-   ParNonlinearForm *pnlf;
-   Array<ParGridFunction *> pgridfuncarr;
-   Array<ParFiniteElementSpace *> pfespacearr;
-#endif
-   bool serial;
-
-   // All are owned.
-   TMOPRefinerEstimator *tmop_r_est;
-   ThresholdRefiner *tmop_r;
-   TMOPDeRefinerEstimator *tmop_dr_est;
-   ThresholdDerefiner *tmop_dr;
-
-   int hr_iter, h_per_r_iter;
-
-   void Update();
-#ifdef MFEM_USE_MPI
-   void ParUpdate();
-#endif
-   void UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf);
-
-#ifdef MFEM_USE_MPI
-   // Rebalance ParMesh such that all the children elements are moved to the same
-   // MPI rank where the parent will be if the mesh were to be derefined.
-   void RebalanceParNCMesh();
-#endif
-
-public:
-   TMOPHRSolver(Mesh &mesh_, NonlinearForm &nlf_,
-                TMOPNewtonSolver &tmopns_, GridFunction &x_,
-                bool move_bnd_, bool hradaptivity_,
-                int mesh_poly_deg_, int amr_metric_id_,
-                int hr_iter_ = 5, int h_per_r_iter_ = 1);
-#ifdef MFEM_USE_MPI
-   TMOPHRSolver(ParMesh &pmesh_, ParNonlinearForm &pnlf_,
-                TMOPNewtonSolver &tmopns_, ParGridFunction &x_,
-                bool move_bnd_, bool hradaptivity_,
-                int mesh_poly_deg_, int amr_metric_id_,
-                int hr_iter_ = 5, int h_per_r_iter_ = 1);
-#endif
-
-   void Mult();
-
-   /// These are used to update spaces and functions that are not owned by the
-   /// TMOPIntegrator or DiscreteAdaptTC. The owned ones are updated in the
-   /// functions UpdateAfterMeshTopologyChange() of both classes.
-   void AddGridFunctionForUpdate(GridFunction *gf) { gridfuncarr.Append(gf); }
-   void AddFESpaceForUpdate(FiniteElementSpace *fes) { fespacearr.Append(fes); }
-
-#ifdef MFEM_USE_MPI
-   void AddGridFunctionForUpdate(ParGridFunction *pgf_)
-   {
-      pgridfuncarr.Append(pgf_);
-   }
-   void AddFESpaceForUpdate(ParFiniteElementSpace *pfes_)
-   {
-      pfespacearr.Append(pfes_);
-   }
-#endif
-
-   ~TMOPHRSolver()
-   {
-      if (!hradaptivity) { return; }
-      delete tmop_dr;
-      delete tmop_dr_est;
-      delete tmop_r;
-      delete tmop_r_est;
-   }
-
-   /// Total number of hr-adaptivity iterations. At each iteration, we do an
-   /// r-adaptivity iteration followed by a number of h-adaptivity iterations.
-   void SetHRAdaptivityIterations(int iter) { hr_iter = iter; }
-
-   /// Total number of h-adaptivity iterations per r-adaptivity iteration.
-   void SetHAdaptivityIterations(int iter) { h_per_r_iter = iter; }
-};
-
-}
-#endif
@@ -407,8 +407,6 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   {
      // Needed for the line search below. The untangling metrics see this
      // reference to detect deteriorations.
-      MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
-                  " intermediate mesh is invalid. Contact TMOP Developers.");
      *min_det_ptr = untangle_factor * min_detT_in;
   }

@@ -578,7 +576,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
         if (ti)
         {
-            ti->UpdateAfterMeshPositionChange(x_loc);
+            ti->UpdateAfterMeshChange(x_loc);
            ti->ComputeFDh(x_loc, *pfesc);
            UpdateDiscreteTC(*ti, x_loc);
         }
@@ -588,7 +586,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
            for (int j = 0; j < ati.Size(); j++)
            {
-               ati[j]->UpdateAfterMeshPositionChange(x_loc);
+               ati[j]->UpdateAfterMeshChange(x_loc);
               ati[j]->ComputeFDh(x_loc, *pfesc);
               UpdateDiscreteTC(*ati[j], x_loc);
            }
@@ -615,7 +613,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
         if (ti)
         {
-            ti->UpdateAfterMeshPositionChange(x_loc);
+            ti->UpdateAfterMeshChange(x_loc);
            ti->ComputeFDh(x_loc, *fesc);
            UpdateDiscreteTC(*ti, x_loc);
         }
@@ -625,7 +623,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
            for (int j = 0; j < ati.Size(); j++)
            {
-               ati[j]->UpdateAfterMeshPositionChange(x_loc);
+               ati[j]->UpdateAfterMeshChange(x_loc);
               ati[j]->ComputeFDh(x_loc, *fesc);
               UpdateDiscreteTC(*ati[j], x_loc);
            }
@@ -175,8 +175,7 @@ const Operator &InterpolationGridTransfer::ForwardOperator()
                                            localP[elem_geoms[i]]);
      }
      F.Reset(ran_fes.RefinementMatrix_main(
-                 dom_fes.GetNDofs(), dom_fes.GetElementToDofTable(),
-                 dom_fes.GetElementToFaceOrientationTable(), localP));
+                 dom_fes.GetNDofs(), dom_fes.GetElementToDofTable(), localP));
   }
   else
   {
@@ -21,16 +21,12 @@
 #define MFEM_PERF_FUNCTION CALI_CXX_MARK_FUNCTION
 #define MFEM_PERF_BEGIN(s) CALI_MARK_BEGIN(s)
 #define MFEM_PERF_END(s) CALI_MARK_END(s)
-#define MFEM_PERF_SCOPE(name) \
-  cali::Annotation::Guard cali_autogenerated_guard_name(cali::Annotation("function").begin(std::string(name).c_str()))
-

 #else

 #define MFEM_PERF_FUNCTION
 #define MFEM_PERF_BEGIN(s)
 #define MFEM_PERF_END(s)
-#define MFEM_PERF_SCOPE(name)

 #endif

@@ -13,7 +13,6 @@
 #define MFEM_FORALL_HPP

 #include "../config/config.hpp"
-#include "annotation.hpp"
 #include "error.hpp"
 #include "backends.hpp"
 #include "device.hpp"
@@ -494,7 +494,8 @@ public:
   /// Copy @a size entries from @a *this to @a dest.
   /** The given @a size should not exceed the Capacity() of @a *this and the
       destination, @a dest. */
-   inline void CopyTo(Memory &dest, int size) const;
+   inline void CopyTo(Memory &dest, int size) const
+   { dest.CopyFrom(*this, size); }

   /// Copy @a size entries from @a *this to the host pointer @a dest.
   /** The given @a size should not exceed the Capacity() of @a *this. */
@@ -922,11 +923,6 @@ inline void Memory<T>::Wrap(T *ptr, T *d_ptr, int size, MemoryType mt, bool own)
 template <typename T>
 inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
 {
-   MFEM_ASSERT(0 <= offset, "invalid offset = " << offset);
-   MFEM_ASSERT(0 <= size, "invalid size = " << size);
-   MFEM_ASSERT(offset + size <= base.capacity,
-               "invalid offset + size = " << offset + size
-               << " > base capacity = " << base.capacity);
   capacity = size;
   h_mt = base.h_mt;
   h_ptr = base.h_ptr + offset;
@@ -1140,7 +1136,6 @@ inline bool Memory<T>::DeviceIsValid() const
 template <typename T>
 inline void Memory<T>::CopyFrom(const Memory &src, int size)
 {
-   MFEM_VERIFY(src.capacity>=size && capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED) && !(src.flags & REGISTERED))
   {
      if (h_ptr != src.h_ptr && size != 0)
@@ -1160,7 +1155,6 @@ inline void Memory<T>::CopyFrom(const Memory &src, int size)
 template <typename T>
 inline void Memory<T>::CopyFromHost(const T *src, int size)
 {
-   MFEM_VERIFY(capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED))
   {
      if (h_ptr != src && size != 0)
@@ -1177,17 +1171,9 @@ inline void Memory<T>::CopyFromHost(const T *src, int size)
   }
 }

-template <typename T>
-inline void Memory<T>::CopyTo(Memory &dest, int size) const
-{
-   MFEM_VERIFY(capacity>=size, "Incorrect size");
-   dest.CopyFrom(*this, size);
-}
-
 template <typename T>
 inline void Memory<T>::CopyToHost(T *dest, int size) const
 {
-   MFEM_VERIFY(capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED))
   {
      if (h_ptr != dest && size != 0)
@@ -118,11 +118,6 @@ int STable3D::Index (int r, int c, int f) const
 {
   STable3DNode *node;

-   if (r >= Size)
-   {
-      return -1;
-   }
-
   Sort3 (r, c, f);

   for (node = Rows[r]; node != NULL; node = node->Prev)
@@ -82,6 +82,9 @@ const char *GetConfigStr()
 #ifdef MFEM_USE_CUDA
      "MFEM_USE_CUDA\n"
 #endif
+#ifdef MFEM_USE_EPIC
+      "MFEM_USE_EPIC\n"
+#endif
 #ifdef MFEM_USE_EXCEPTIONS
      "MFEM_USE_EXCEPTIONS\n"
 #endif
@@ -83,6 +83,11 @@ if (MFEM_USE_SUNDIALS)
  list(APPEND HDRS sundials.hpp)
 endif()

+if (MFEM_USE_EPIC)
+  list(APPEND SRCS epic.cpp)
+  list(APPEND HDRS epic.hpp)
+endif()
+
 if (MFEM_USE_SUPERLU)
  list(APPEND SRCS superlu.cpp)
  # If this list (HDRS -> HEADERS) is used for install, we probably want the
@@ -787,16 +787,10 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const
                                           2 * num_cols_offd, cmap,
                                           true);

-#if MFEM_HYPRE_VERSION <= 22200
   // Give the new matrix ownership of row_starts and col_starts
   hypre_ParCSRMatrix *hA = (hypre_ParCSRMatrix*)(*A);
-
   hypre_ParCSRMatrixSetRowStartsOwner(hA,1);
   hypre_ParCSRMatrixSetColStartsOwner(hA,1);
-#else
-   mfem_hypre_TFree_host(row_starts);
-   mfem_hypre_TFree_host(col_starts);
-#endif

   return A;
 }
@@ -763,13 +763,6 @@ public:
      tdata.New(i*j*k);
   }

-   DenseTensor(double *d, int i, int j, int k)
-      : Mk(NULL, i, j)
-   {
-      nk = k;
-      tdata.Wrap(d, i*j*k, false);
-   }
-
   DenseTensor(int i, int j, int k, MemoryType mt)
      : Mk(NULL, i, j)
   {
@@ -0,0 +1,171 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "epic.hpp"
+
+#ifdef MFEM_USE_EPIC
+
+namespace mfem
+{
+
+EPICSolver::EPICSolver(bool exactJacobian_, EPICNumJacDelta delta)
+{
+   // Allocate an empty serial N_Vector
+   temp = N_VNewEmpty_Serial(0);
+   m[0] = 10;
+   m[1] = 10;
+   MFEM_VERIFY(temp, "error in N_VNewEmpty_Serial()");
+   exactJacobian = exactJacobian_;
+   Jtv = NULL;
+   Delta = delta;
+}
+
+#ifdef MFEM_USE_MPI
+EPICSolver::EPICSolver(MPI_Comm comm)
+{
+   m[0] = 10;
+   m[1] = 10;
+
+   // Allocate an empty vector
+   if (comm == MPI_COMM_NULL)
+   {
+      // Allocate an empty serial N_Vector
+      temp = N_VNewEmpty_Serial(0);
+      MFEM_VERIFY(temp, "error in N_VNewEmpty_Serial()");
+   }
+   else
+   {
+      // Allocate an empty parallel N_Vector
+      temp = N_VNewEmpty_Parallel(comm, 0, 0);  // calls MPI_Allreduce()
+      MFEM_VERIFY(temp, "error in N_VNewEmpty_Parallel()");
+   }
+}
+#endif
+
+int EPICSolver::RHS(realtype t, const N_Vector y, N_Vector ydot, void *user_data)
+{
+   // Get data from N_Vectors
+   const Vector mfem_y(y);
+   Vector mfem_ydot(ydot);
+   EPICSolver *self = static_cast<EPICSolver*>(user_data);
+
+   // Compute y' = f(t, y)
+   self->f->SetTime(t);
+   self->f->Mult(mfem_y, mfem_ydot);
+
+   // Return success
+   return 0;
+}
+
+
+int EPICSolver::Jacobian(N_Vector v, N_Vector Jv, realtype t, N_Vector y, N_Vector fy, void *user_data, N_Vector tmp)
+{
+    // Get data from N_Vectors
+   const Vector mfem_v(v);
+   Vector mfem_Jv(Jv);
+   EPICSolver *self = static_cast<EPICSolver*>(user_data);
+
+   // Compute J(t, y) v
+   self->Jtv->Mult(mfem_v, mfem_Jv);
+
+   return 0;
+}
+
+void EPICSolver::Init(TimeDependentOperator &f)
+{
+    ODESolver::Init(f);
+	    
+    long local_size = f.Height();
+    long global_size = 0;
+#ifdef MFEM_USE_MPI
+    if (Parallel())
+    {
+        MPI_Allreduce(&local_size, &global_size, 1, MPI_LONG, MPI_SUM,
+                      NV_COMM_P(temp));
+    }
+#endif
+
+    Vector mfem_temp(local_size);
+    mfem_temp.ToNVector(temp, global_size);
+}
+
+EPI2::EPI2(bool exactJacobian, EPICNumJacDelta delta) : EPICSolver(exactJacobian, delta) {}
+
+void EPI2::Init(TimeDependentOperator &f)
+{
+    EPICSolver::Init(f);
+    long local_size = f.Height();
+    if (exactJacobian) {
+       integrator = new Epi2_KIOPS(EPICSolver::RHS, EPICSolver::Jacobian, this, 100, temp ,local_size);
+    } else {
+       integrator = new Epi2_KIOPS(EPICSolver::RHS, Delta, this, 100, temp ,local_size);
+    }
+
+}
+
+EPIRK4::EPIRK4(bool exactJacobian, EPICNumJacDelta delta) : EPICSolver(exactJacobian, delta) {}
+
+void EPIRK4::Init(TimeDependentOperator &f)
+{
+    EPICSolver::Init(f);
+    long local_size = f.Height();
+    if (exactJacobian) {
+       integrator = new EpiRK4SC_KIOPS(EPICSolver::RHS, EPICSolver::Jacobian, this, 100, temp ,local_size);
+    } else {
+       integrator = new EpiRK4SC_KIOPS(EPICSolver::RHS, Delta, this, 100, temp ,local_size);
+    }
+}
+
+void EPICSolver::Step(Vector &x, double &t, double &dt)
+{
+   if (!Parallel())
+   {
+      NV_DATA_S(temp) = x.GetData();
+      MFEM_VERIFY(NV_LENGTH_S(temp) == x.Size(), "");
+   }
+   else
+   {
+#ifdef MFEM_USE_MPI
+      NV_DATA_P(temp) = x.GetData();
+      MFEM_VERIFY(NV_LOCLENGTH_P(temp) == x.Size(), "");
+#endif
+   }
+
+   Jtv = &(this->f->GetGradient(x));
+}
+
+void EPI2::Step(Vector &x, double &t, double &dt)
+{
+    EPICSolver::Step(x, t, dt);
+    integrator->Integrate(dt, t, t+dt, 0, temp, 1e-10, m);
+    t += dt;
+}
+
+void EPIRK4::Step(Vector &x, double &t, double &dt)
+{
+    EPICSolver::Step(x, t, dt);
+    integrator->Integrate(dt, t, t+dt, 0, temp, 1e-10, m);
+    t += dt;
+}
+
+EPI2::~EPI2()
+{
+    delete integrator;
+}
+
+EPIRK4::~EPIRK4()
+{
+    delete integrator;
+}
+
+}
+
+#endif
@@ -0,0 +1,98 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_EPIC
+#define MFEM_EPIC
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_EPIC
+
+// SUNDIALS vectors
+#include <nvector/nvector_serial.h>
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#include <nvector/nvector_parallel.h>
+#endif
+
+#include "ode.hpp"
+#include "solvers.hpp"
+#include <Epic.h>
+
+namespace mfem
+{
+
+typedef void (*JacobianFun)(const realtype t, const Vector &y, const Vector& v, Vector& Jv, void* user_data);
+// ---------------------------------------------------------------------------
+// Interface to the EPIC library -- exponential methods
+// ---------------------------------------------------------------------------
+
+class EPICSolver : public ODESolver
+{
+protected:
+    EPICNumJacDelta Delta;
+    Operator* Jtv;
+    N_Vector temp;
+    int m[2];
+
+    bool exactJacobian;
+
+    #ifdef MFEM_USE_MPI
+    bool Parallel() const
+    {
+        return (N_VGetVectorID(temp) != SUNDIALS_NVEC_SERIAL);
+    }
+    #else
+    bool Parallel() const { return false; }
+    #endif
+
+public:
+    EPICSolver(bool exactJacobian, EPICNumJacDelta delta=&DefaultDelta);
+    EPICSolver(MPI_Comm comm);
+
+    static int RHS(realtype t, const N_Vector y, N_Vector ydot, void *user_data);
+    static int Jacobian(N_Vector v, N_Vector Jv, realtype t,
+                         N_Vector y, N_Vector fy, void *user_data, N_Vector tmp);
+    virtual void Init(TimeDependentOperator &f);
+    virtual void Step(Vector &x, double &t, double &dt);
+
+    virtual ~EPICSolver() {}
+};
+
+class EPI2 : public EPICSolver
+{
+protected:
+    Epi2_KIOPS* integrator;
+public:
+    EPI2(bool exactJacobian=true, EPICNumJacDelta delta=&DefaultDelta);
+    virtual void Init(TimeDependentOperator &f);
+    virtual void Step(Vector &x, double &t, double &dt);
+
+    virtual ~EPI2();
+};
+
+class EPIRK4 : public EPICSolver
+{
+protected:
+    EpiRK4SC_KIOPS* integrator;
+public:
+    EPIRK4(bool exactJacobian=true, EPICNumJacDelta delta=&DefaultDelta);
+    virtual void Init(TimeDependentOperator &f);
+    virtual void Step(Vector &x, double &t, double &dt);
+
+    virtual ~EPIRK4();
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_EPIC
+
+#endif // MFEM_EPIC
@@ -88,9 +88,7 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
 {
   x = hypre_ParVectorCreate(comm,glob_size,col);
   hypre_ParVectorInitialize(x);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParVectorSetPartitioningOwner(x,0);
-#endif
   // The data will be destroyed by hypre (this is the default)
   hypre_ParVectorSetDataOwner(x,1);
   hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
@@ -107,9 +105,7 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
   hypre_ParVectorSetDataOwner(x,1); // owns the seq vector
   hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
   hypre_SeqVectorSetDataOwner(x_loc,0);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParVectorSetPartitioningOwner(x,0);
-#endif
   double tmp = 0.0;
   hypre_VectorData(x_loc) = &tmp;
 #ifdef HYPRE_USING_CUDA
@@ -132,9 +128,7 @@ HypreParVector::HypreParVector(const HypreParVector &y) : Vector()
   x = hypre_ParVectorCreate(y.x -> comm, y.x -> global_size,
                             y.x -> partitioning);
   hypre_ParVectorInitialize(x);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParVectorSetPartitioningOwner(x,0);
-#endif
   hypre_ParVectorSetDataOwner(x,1);
   hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
   _SetDataAndSize_();
@@ -168,9 +162,7 @@ HypreParVector::HypreParVector(ParFiniteElementSpace *pfes)
   x = hypre_ParVectorCreate(pfes->GetComm(), pfes->GlobalTrueVSize(),
                             pfes->GetTrueDofOffsets());
   hypre_ParVectorInitialize(x);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParVectorSetPartitioningOwner(x,0);
-#endif
   // The data will be destroyed by hypre (this is the default)
   hypre_ParVectorSetDataOwner(x,1);
   hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
@@ -691,10 +683,8 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, HYPRE_BigInt glob_size,
   A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, row_starts,
                                row_starts, 0, diag->NumNonZeroElems(), 0);
   hypre_ParCSRMatrixSetDataOwner(A,1);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
-#endif

   hypre_CSRMatrixSetDataOwner(A->diag,0);
   diagOwner = CopyCSR(diag, mem_diag, A->diag, false);
@@ -736,10 +726,8 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
                                row_starts, col_starts,
                                0, diag->NumNonZeroElems(), 0);
   hypre_ParCSRMatrixSetDataOwner(A,1);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
-#endif

   hypre_CSRMatrixSetDataOwner(A->diag,0);
   diagOwner = CopyCSR(diag, mem_diag, A->diag, false);
@@ -782,10 +770,8 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
                                offd->Width(), diag->NumNonZeroElems(),
                                offd->NumNonZeroElems());
   hypre_ParCSRMatrixSetDataOwner(A,1);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
-#endif

   hypre_CSRMatrixSetDataOwner(A->diag,0);
   diagOwner = CopyCSR(diag, mem_diag, A->diag, own_diag_offd);
@@ -831,10 +817,8 @@ HypreParMatrix::HypreParMatrix(
   A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols,
                                row_starts, col_starts, offd_num_cols, 0, 0);
   hypre_ParCSRMatrixSetDataOwner(A,1);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
-#endif

   HYPRE_Int local_num_rows = hypre_CSRMatrixNumRows(A->diag);

@@ -947,10 +931,8 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
   A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols,
                                row_starts, col_starts, 0, nnz, 0);
   hypre_ParCSRMatrixSetDataOwner(A,1);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
-#endif

   hypre_CSRMatrixSetDataOwner(A->diag,0);
   diagOwner = CopyBoolCSR(diag, mem_diag, A->diag);
@@ -1007,10 +989,8 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
   }

   hypre_ParCSRMatrixSetDataOwner(A,1);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
-#endif

   mem_diag.data.New(diag_nnz);
   for (HYPRE_Int i = 0; i < diag_nnz; i++)
@@ -1197,13 +1177,6 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows,
   {
      hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
   }
-#if MFEM_HYPRE_VERSION > 22200
-   mfem_hypre_TFree_host(row_starts);
-   if (rows != cols)
-   {
-      mfem_hypre_TFree_host(col_starts);
-   }
-#endif
   hypre_MatvecCommPkgCreate(A);

   height = GetNumRows();
@@ -1290,7 +1263,6 @@ void HypreParMatrix::SetOwnerFlags(signed char diag, signed char offd,

 void HypreParMatrix::CopyRowStarts()
 {
-#if MFEM_HYPRE_VERSION <= 22200
   if (!A || hypre_ParCSRMatrixOwnsRowStarts(A) ||
       (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) &&
        hypre_ParCSRMatrixOwnsColStarts(A)))
@@ -1325,12 +1297,10 @@ void HypreParMatrix::CopyRowStarts()
      hypre_ParCSRMatrixColStarts(A) = new_row_starts;
      hypre_ParCSRMatrixOwnsColStarts(A) = 0;
   }
-#endif
 }

 void HypreParMatrix::CopyColStarts()
 {
-#if MFEM_HYPRE_VERSION <= 22200
   if (!A || hypre_ParCSRMatrixOwnsColStarts(A) ||
       (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) &&
        hypre_ParCSRMatrixOwnsRowStarts(A)))
@@ -1369,7 +1339,6 @@ void HypreParMatrix::CopyColStarts()
   {
      hypre_ParCSRMatrixOwnsColStarts(A) = 1;
   }
-#endif
 }

 void HypreParMatrix::GetDiag(Vector &diag) const
@@ -1822,14 +1791,9 @@ HypreParMatrix* HypreParMatrix::LeftDiagMult(const SparseMatrix &D,
                         DA_diag, DA_offd, new_col_map_offd,
                         own_diag_offd);

-#if MFEM_HYPRE_VERSION <= 22200
   // Give ownership of row_starts, col_starts, and col_map_offd to DA
   hypre_ParCSRMatrixSetRowStartsOwner(DA->A, 1);
   hypre_ParCSRMatrixSetColStartsOwner(DA->A, 1);
-#else
-   mfem_hypre_TFree_host(new_row_starts);
-   mfem_hypre_TFree_host(new_col_starts);
-#endif
   DA->colMapOwner = 1;

   return DA;
@@ -1984,22 +1948,18 @@ void HypreParMatrix::Threshold(double threshold)
   row_starts = hypre_ParCSRMatrixRowStarts(A);
   col_starts = hypre_ParCSRMatrixColStarts(A);

-#if MFEM_HYPRE_VERSION <= 22200
   bool old_owns_row = hypre_ParCSRMatrixOwnsRowStarts(A);
   bool old_owns_col = hypre_ParCSRMatrixOwnsColStarts(A);
-#endif
   HYPRE_BigInt global_num_rows = hypre_ParCSRMatrixGlobalNumRows(A);
   HYPRE_BigInt global_num_cols = hypre_ParCSRMatrixGlobalNumCols(A);
   parcsr_A_ptr = hypre_ParCSRMatrixCreate(comm, global_num_rows,
                                           global_num_cols,
                                           row_starts, col_starts,
                                           0, 0, 0);
-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixOwnsRowStarts(parcsr_A_ptr) = old_owns_row;
   hypre_ParCSRMatrixOwnsColStarts(parcsr_A_ptr) = old_owns_col;
   hypre_ParCSRMatrixOwnsRowStarts(A) = 0;
   hypre_ParCSRMatrixOwnsColStarts(A) = 0;
-#endif

   csr_A = hypre_MergeDiagAndOffd(A);

@@ -2034,12 +1994,7 @@ void HypreParMatrix::Threshold(double threshold)

   hypre_ParCSRMatrixSetNumNonzeros(A);
   /* Make sure that the first entry in each row is the diagonal one. */
-#if MFEM_HYPRE_VERSION <= 22200
   if (row_starts == col_starts)
-#else
-   if ((row_starts[0] == col_starts[0]) &&
-       (row_starts[1] == col_starts[1]))
-#endif
   {
      hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
   }
@@ -2548,14 +2503,11 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
      // hypre_ParCSRMatrixRAPKT
   }
 #else
-#if MFEM_HYPRE_VERSION <= 22200
   HYPRE_Int P_owns_its_col_starts =
      hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
-#endif

   hypre_BoomerAMGBuildCoarseOperator(*P,*A,*P,&rap);

-#if MFEM_HYPRE_VERSION <= 22200
   /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts
      from P (even if it does not own them)! */
   hypre_ParCSRMatrixSetRowStartsOwner(rap,0);
@@ -2564,7 +2516,6 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
   {
      hypre_ParCSRMatrixSetColStartsOwner(*P, 1);
   }
-#endif
 #endif

   hypre_ParCSRMatrixSetNumNonzeros(rap);
@@ -2585,16 +2536,13 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
      hypre_ParCSRMatrixDestroy(Q);
   }
 #else
-#if MFEM_HYPRE_VERSION <= 22200
   HYPRE_Int P_owns_its_col_starts =
      hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
   HYPRE_Int Rt_owns_its_col_starts =
      hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*Rt));
-#endif

   hypre_BoomerAMGBuildCoarseOperator(*Rt,*A,*P,&rap);

-#if MFEM_HYPRE_VERSION <= 22200
   /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts
      from Rt and P (even if they do not own them)! */
   hypre_ParCSRMatrixSetRowStartsOwner(rap,0);
@@ -2607,7 +2555,6 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
   {
      hypre_ParCSRMatrixSetColStartsOwner(*Rt, 1);
   }
-#endif
 #endif

   hypre_ParCSRMatrixSetNumNonzeros(rap);
@@ -3197,12 +3144,8 @@ void HypreSmoother::SetOperator(const Operator &op)
      }
      else
      {
-#if MFEM_HYPRE_VERSION <= 22200
         min_eig_est = 0;
         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est);
-#else
-         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est);
-#endif
      }
      Z = new HypreParVector(*A);
   }
@@ -3216,12 +3159,8 @@ void HypreSmoother::SetOperator(const Operator &op)
      }
      else
      {
-#if MFEM_HYPRE_VERSION <= 22200
         min_eig_est = 0;
         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est);
-#else
-         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est);
-#endif
      }

      // The Taubin and FIR polynomials are defined on [0, 2]
@@ -5058,11 +4997,11 @@ HypreADS::HypreADS(const HypreParMatrix &A, ParFiniteElementSpace *face_fespace)
 void HypreADS::Init(ParFiniteElementSpace *face_fespace)
 {
   int cycle_type       = 11;
+   int rlx_type         = 2;
   int rlx_sweeps       = 1;
   double rlx_weight    = 1.0;
   double rlx_omega     = 1.0;
 #ifndef HYPRE_USING_CUDA
-   int rlx_type         = 2;
   int amg_coarsen_type = 10;
   int amg_agg_levels   = 1;
   int amg_rlx_type     = 8;
@@ -5070,7 +5009,6 @@ void HypreADS::Init(ParFiniteElementSpace *face_fespace)
   int amg_interp_type  = 6;
   int amg_Pmax         = 4;
 #else
-   int rlx_type         = 1;
   int amg_coarsen_type = 8;
   int amg_agg_levels   = 0;
   int amg_rlx_type     = 18;
@@ -508,10 +508,8 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
                                  hypre_ParCSRMatrixColStarts(A),
                                  0, 0, 0);

-#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(*Ae, 0);
   hypre_ParCSRMatrixSetColStartsOwner(*Ae, 0);
-#endif

   hypre_CSRMatrix *Ae_diag = hypre_ParCSRMatrixDiag(*Ae);
   hypre_CSRMatrix *Ae_offd = hypre_ParCSRMatrixOffd(*Ae);
@@ -1004,17 +1002,10 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,

      hypre_ParCSRMatrixOwnsData(blocks[i]) = 1;

-#if MFEM_HYPRE_VERSION <= 22200
      /* only the first block will own the row/col_starts */
      hypre_ParCSRMatrixOwnsRowStarts(blocks[i]) = !i;
      hypre_ParCSRMatrixOwnsColStarts(blocks[i]) = !i;
-#endif
   }
-
-#if MFEM_HYPRE_VERSION > 22200
-   mfem_hypre_TFree_host(row_starts);
-   mfem_hypre_TFree_host(col_starts);
-#endif
 }

 /* Based on hypre_CSRMatrixMatvec in hypre's csr_matvec.c */
@@ -1925,12 +1916,9 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A,

   /* C owns diag, offd, and cmap. */
   hypre_ParCSRMatrixSetDataOwner(C, 1);
-
-#if MFEM_HYPRE_VERSION <= 22200
   /* C does not own row and column starts. */
   hypre_ParCSRMatrixSetRowStartsOwner(C, 0);
   hypre_ParCSRMatrixSetColStartsOwner(C, 0);
-#endif

   return C;
 }
@@ -40,6 +40,10 @@
 #include "sundials.hpp"
 #endif

+#ifdef MFEM_USE_EPIC
+#include "epic.hpp"
+#endif
+
 #ifdef MFEM_USE_HIOP
 #include "hiop.hpp"
 #endif
@@ -33,12 +33,7 @@ int SparseMatrix::SparseMatrixCount = 0;
 cusparseHandle_t SparseMatrix::handle = nullptr;
 size_t SparseMatrix::bufferSize = 0;
 void * SparseMatrix::dBuffer = nullptr;
-#  if CUSPARSE_VERSION >=  11400
-#    define MFEM_CUSPARSE_ALG CUSPARSE_SPMV_CSR_ALG1
-#  else
-#    define MFEM_CUSPARSE_ALG CUSPARSE_CSRMV_ALG1
-#  endif // CUSPARSE_VERSION >= 11400
-#endif // MFEM_USE_CUDA
+#endif

 void SparseMatrix::InitCuSparse()
 {
@@ -684,16 +679,25 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
         cusparseCreateMatDescr(&matA_descr);
         cusparseSetMatIndexBase(matA_descr, CUSPARSE_INDEX_BASE_ZERO);
         cusparseSetMatType(matA_descr, CUSPARSE_MATRIX_TYPE_GENERAL);
+
 #endif
+
         initBuffers = true;
      }
      // Allocate kernel space. Buffer is shared between different sparsemats
      size_t newBufferSize = 0;

+#if CUDA_VERSION >= 11020
      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
                              matA_descr,
                              vecX_descr, &beta, vecY_descr, CUDA_R_64F,
-                              MFEM_CUSPARSE_ALG, &newBufferSize);
+                              CUSPARSE_SPMV_CSR_ALG1, &newBufferSize);
+#elif CUDA_VERSION >= 10010
+      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
+                              matA_descr,
+                              vecX_descr, &beta, vecY_descr, CUDA_R_64F,
+                              CUSPARSE_CSRMV_ALG1, &newBufferSize);
+#endif

      // Check if we need to resize
      if (newBufferSize > bufferSize)
@@ -703,22 +707,30 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
         CuMemAlloc(&dBuffer, bufferSize);
      }

-#if CUDA_VERSION >= 10010
+#if CUDA_VERSION >= 11020
      // Update input/output vectors
      cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
      cusparseDnVecSetValues(vecY_descr, d_y);

      // Y = alpha A * X + beta * Y
      cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA_descr,
-                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, MFEM_CUSPARSE_ALG, dBuffer);
+                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, CUSPARSE_SPMV_CSR_ALG1, dBuffer);
+#elif CUDA_VERSION >= 10010
+      // Update input/output vectors
+      cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
+      cusparseDnVecSetValues(vecY_descr, d_y);
+
+      // Y = alpha A * X + beta * Y
+      cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA_descr,
+                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, CUSPARSE_CSRMV_ALG1, dBuffer);
 #else
      cusparseDcsrmv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
                     Height(), Width(), J.Capacity(),
                     &alpha, matA_descr,
                     const_cast<double *>(d_A), const_cast<int *>(d_I), const_cast<int *>(d_J),
                     const_cast<double *>(d_x), &beta, d_y);
-#endif // CUDA_VERSION >= 10010
-#endif // MFEM_USE_CUDA
+#endif
+#endif
   }
   else
   {
@@ -425,6 +425,26 @@ static SUNMatrix_ID MatGetID(SUNMatrix)
   return (SUNMATRIX_CUSTOM);
 }

+//BEGIN WORKAROUND CODE
+static SUNMatrix MatClone(SUNMatrix A)
+{
+   SUNMatrix B = SUNMatNewEmpty();
+   MFEM_VERIFY(B, "error in MatClone()");
+
+   B->content = A->content;
+
+   int flag = SUNMatCopyOps(A, B);
+   MFEM_VERIFY(flag == SUNMAT_SUCCESS, "error in MatClone()");
+
+   return B;
+}
+
+static int MatCopy(SUNMatrix, SUNMatrix)
+{
+   return 0;
+}
+// END WORKAROUN CODE
+
 static void MatDestroy(SUNMatrix A)
 {
   if (A->content) { A->content = NULL; }
@@ -1493,7 +1513,12 @@ void ARKStepSolver::UseMFEMMassLinearSolver(int tdep)
   MFEM_VERIFY(M, "error in SUNMatNewEmpty()");

   M->content      = this;
-   M->ops->getid   = SUNMatGetID;
+// BEGIN WORKAROUND CODE
+   // M->ops->getid   = SUNMatGetID;
+   M->ops->getid = MatGetID;
+   M->ops->clone = MatClone;
+   M->ops->copy = MatCopy;
+// END WORKAROUND CODE
   M->ops->matvec  = ARKStepSolver::MassMult1;
   M->ops->destroy = MatDestroy;

@@ -1537,6 +1562,12 @@ void ARKStepSolver::SetSStolerances(double reltol, double abstol)
   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSStolerances()");
 }

+void ARKStepSolver::SetMaxIter(int iterations)
+{
+  flag = ARKStepSetMaxNonlinIters(sundials_mem, iterations);
+  MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMaxNonlinIters()");
+}
+
 void ARKStepSolver::SetMaxStep(double dt_max)
 {
   flag = ARKStepSetMaxStep(sundials_mem, dt_max);
@@ -666,6 +666,9 @@ public:
   /// Set the scalar relative and scalar absolute tolerances.
   void SetSStolerances(double reltol, double abstol);

+   /// Set the maximum number of nonlinear iterations per RK stage
+   void SetMaxIter(int iterations);
+
   /// Set the maximum time step.
   void SetMaxStep(double dt_max);

@@ -82,11 +82,6 @@ public:
   Vector(double *data_, int size_)
   { data.Wrap(data_, size_, false); size = size_; }

-   /** @brief Create a Vector referencing a sub-vector of the Vector @a base
-       starting at the given offset, @a base_offset, and size @a size_. */
-   Vector(Vector &base, int base_offset, int size_)
-      : data(base.data, base_offset, size_), size(size_) { }
-
   /// Create a Vector of size @a size_ using MemoryType @a mt.
   Vector(int size_, MemoryType mt)
      : data(size_, mt), size(size_) { }
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -119,7 +119,7 @@ $(if $(word 2,$(SRC)),$(error Spaces in SRC = "$(SRC)" are not supported))
 MFEM_GIT_STRING = $(shell [ -d $(MFEM_DIR)/.git ] && git -C $(MFEM_DIR) \
   describe --all --long --abbrev=40 --dirty --always 2> /dev/null)

-EXAMPLE_SUBDIRS = amgx caliper ginkgo hiop petsc pumi sundials superlu
+EXAMPLE_SUBDIRS = amgx ginkgo hiop petsc pumi sundials epic superlu
 EXAMPLE_DIRS := examples $(addprefix examples/,$(EXAMPLE_SUBDIRS))
 EXAMPLE_TEST_DIRS := examples

@@ -272,7 +272,7 @@ ifeq ($(MFEM_USE_LEGACY_OPENMP),YES)
 endif

 # List of MFEM dependencies, that require the *_LIB variable to be non-empty
-MFEM_REQ_LIB_DEPS = SUPERLU MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS MESQUITE\
+MFEM_REQ_LIB_DEPS = SUPERLU MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS EPIC MESQUITE\
 SUITESPARSE STRUMPACK GINKGO GNUTLS NETCDF PETSC SLEPC MPFR PUMI HIOP GSLIB\
 OCCA CEED RAJA UMPIRE MKL_CPARDISO AMGX CALIPER

@@ -333,7 +333,7 @@ endif
 MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
 MFEM_USE_METIS MFEM_USE_METIS_5 MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB\
 MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP\
- MFEM_USE_LEGACY_OPENMP MFEM_USE_MEMALLOC MFEM_TIMER_TYPE MFEM_USE_SUNDIALS\
+ MFEM_USE_LEGACY_OPENMP MFEM_USE_MEMALLOC MFEM_TIMER_TYPE MFEM_USE_SUNDIALS MFEM_USE_EPIC\
 MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE MFEM_USE_GINKGO MFEM_USE_SUPERLU\
 MFEM_USE_STRUMPACK MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC\
 MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT\
@@ -650,6 +650,7 @@ status info:
 	$(info MFEM_USE_MEMALLOC      = $(MFEM_USE_MEMALLOC))
 	$(info MFEM_TIMER_TYPE        = $(MFEM_TIMER_TYPE))
 	$(info MFEM_USE_SUNDIALS      = $(MFEM_USE_SUNDIALS))
+	$(info MFEM_USE_EPIC          = $(MFEM_USE_EPIC))
 	$(info MFEM_USE_MESQUITE      = $(MFEM_USE_MESQUITE))
 	$(info MFEM_USE_SUITESPARSE   = $(MFEM_USE_SUITESPARSE))
 	$(info MFEM_USE_SUPERLU       = $(MFEM_USE_SUPERLU))
@@ -75,7 +75,7 @@ void Mesh::GetElementCenter(int i, Vector &center)

 double Mesh::GetElementSize(ElementTransformation *T, int type)
 {
-   DenseMatrix J(spaceDim,Dim);
+   DenseMatrix J(Dim);

   Geometry::Type geom = T->GetGeometryType();
   T->SetIntPoint(&Geometries.GetCenter(geom));
@@ -83,7 +83,7 @@ double Mesh::GetElementSize(ElementTransformation *T, int type)

   if (type == 0)
   {
-      return pow(fabs(J.Weight()), 1./Dim);
+      return pow(fabs(J.Det()), 1./Dim);
   }
   else if (type == 1)
   {
@@ -102,7 +102,7 @@ double Mesh::GetElementSize(int i, int type)

 double Mesh::GetElementSize(int i, const Vector &dir)
 {
-   DenseMatrix J(spaceDim,Dim);
+   DenseMatrix J(Dim);
   Vector d_hat(Dim);
   GetElementJacobian(i, J);
   J.MultTranspose(dir, d_hat);
@@ -3495,6 +3495,8 @@ void Mesh::SetMeshGen()
            mesh_geoms |= (1 << Geometry::TETRAHEDRON);
         case Element::TRIANGLE:
            mesh_geoms |= (1 << Geometry::TRIANGLE);
+         case Element::SEGMENT:
+            mesh_geoms |= (1 << Geometry::SEGMENT);
         case Element::POINT:
            mesh_geoms |= (1 << Geometry::POINT);
            meshgen |= 1;
@@ -3506,8 +3508,6 @@ void Mesh::SetMeshGen()
            mesh_geoms |= (1 << Geometry::SQUARE);
            mesh_geoms |= (1 << Geometry::SEGMENT);
            mesh_geoms |= (1 << Geometry::POINT);
-         case Element::SEGMENT:
-            mesh_geoms |= (1 << Geometry::SEGMENT);
            meshgen |= 2;
            break;

@@ -3974,12 +3974,6 @@ void Mesh::MakeRefined_(Mesh &orig_mesh, const Array<int> ref_factors,
      }
   }

-   if (Dim > 2)
-   {
-      GetElementToFaceTable(false);
-      GenerateFaces();
-   }
-
   // Add refined boundary elements
   for (int el = 0; el < orig_mesh.GetNBE(); el++)
   {
@@ -7646,8 +7640,6 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p,

   // Map from edge-index to vertex-index, needed for ReorientTetMesh() for
   // parallel meshes.
-   // Note: with the removal of ReorientTetMesh() this may no longer
-   // be needed.  Unfortunately, it's hard to be sure.
   Array<int> e2v;
   if (HasGeometry(Geometry::TETRAHEDRON))
   {
@@ -503,7 +503,7 @@ public:
   Mesh& operator=(Mesh &&mesh);

   /// Explicitly delete the copy assignment operator.
-   Mesh& operator=(const Mesh &mesh) = delete;
+   Mesh& operator=(Mesh &mesh) = delete;

   /** @name Named mesh constructors.

@@ -1227,7 +1227,7 @@ public:
       satisfy: v0 < min(v1, v2).

       @note Refinement does not work after a call to this method! */
-   MFEM_DEPRECATED virtual void ReorientTetMesh();
+   virtual void ReorientTetMesh();

   int *CartesianPartitioning(int nxyz[]);
   int *GeneratePartitioning(int nparts, int part_method = 1);
@@ -514,7 +514,8 @@ void Mesh::CreateVTKMesh(const Vector &points, const Array<int> &cell_data,
            }
         }
      }
-      // Generate faces and edges so that we can define FE space on the mesh
+      // Generate faces and edges so that we can define
+      // FE space on the mesh
      FinalizeTopology();

      FiniteElementCollection *fec;
@@ -2444,10 +2445,6 @@ void Mesh::ReadGmshMesh(std::istream &input, int &curved, int &read_gf)
            // initialize mesh_geoms so we can create Nodes FE space below
            this->SetMeshGen();

-            // Generate faces and edges so that we can define
-            // FE space on the mesh
-            this->FinalizeTopology();
-
            // Construct GridFunction for uniformly spaced high order coords
            FiniteElementCollection* nfec;
            FiniteElementSpace* nfes;
@@ -2669,7 +2666,6 @@ void Mesh::ReadGmshMesh(std::istream &input, int &curved, int &read_gf)
         // Convert nodes to discontinuous GridFunction (if they aren't already)
         if (mesh_order == 1)
         {
-            this->FinalizeTopology();
            this->SetMeshGen();
            this->SetCurvature(1, true, spaceDim, Ordering::byVDIM);
         }
@@ -2283,6 +2283,7 @@ void NCMesh::GetMeshComponents(Mesh &mesh) const
   // left uninitialized here; they will be initialized later by the Mesh from
   // Nodes -- here we just make sure mesh.vertices has the correct size.

+   mesh.elements.SetSize(NElements);
   mesh.elements.SetSize(0);

   mesh.boundary.SetSize(0);
@@ -4253,19 +4254,6 @@ void NCMesh::GetPointMatrix(Geometry::Type geom, const char* ref_path,
            pm = PointMatrix(mid12, mid20, mid01);
         }
      }
-      else if (geom == Geometry::SEGMENT)
-      {
-         Point mid01(pm(0), pm(1));
-
-         if (child == 0)
-         {
-            pm = PointMatrix(pm(0), mid01);
-         }
-         else if (child == 1)
-         {
-            pm = PointMatrix(mid01, pm(1));
-         }
-      }
   }

   // write the points to the matrix
@@ -131,7 +131,6 @@ public:
   int GetNVertices() const { return NVertices; }
   int GetNEdges() const { return NEdges; }
   int GetNFaces() const { return NFaces; }
-   virtual int GetNGhostElements() const { return 0; }

   /** Perform the given batch of refinements. Please note that in the presence
       of anisotropic splits additional refinements may be necessary to keep
@@ -161,6 +160,7 @@ public:
       derefinements may have to be skipped to preserve mesh consistency. */
   virtual void Derefine(const Array<int> &derefs);

+
   // master/slave lists

   /// Identifies a vertex/edge/face in both Mesh and NCMesh.
@@ -34,7 +34,6 @@ ParMesh::ParMesh(const ParMesh &pmesh, bool copy_nodes)
     group_sedge(pmesh.group_sedge),
     group_stria(pmesh.group_stria),
     group_squad(pmesh.group_squad),
-     face_nbr_el_to_face(NULL),
     glob_elem_offset(-1),
     glob_offset_sequence(-1),
     gtopo(pmesh.gtopo)
@@ -106,8 +105,7 @@ ParMesh& ParMesh::operator=(ParMesh &&mesh)

 ParMesh::ParMesh(MPI_Comm comm, Mesh &mesh, int *partitioning_,
                 int part_method)
-   : face_nbr_el_to_face(NULL)
-   , glob_elem_offset(-1)
+   : glob_elem_offset(-1)
   , glob_offset_sequence(-1)
   , gtopo(comm)
 {
@@ -854,7 +852,6 @@ ParMesh::ParMesh(const ParNCMesh &pncmesh)
   : MyComm(pncmesh.MyComm)
   , NRanks(pncmesh.NRanks)
   , MyRank(pncmesh.MyRank)
-   , face_nbr_el_to_face(NULL)
   , glob_elem_offset(-1)
   , glob_offset_sequence(-1)
   , gtopo(MyComm)
@@ -921,8 +918,7 @@ void ParMesh::FinalizeParTopo()
 }

 ParMesh::ParMesh(MPI_Comm comm, istream &input, bool refine)
-   : face_nbr_el_to_face(NULL)
-   , glob_elem_offset(-1)
+   : glob_elem_offset(-1)
   , glob_offset_sequence(-1)
   , gtopo(comm)
 {
@@ -1133,7 +1129,6 @@ void ParMesh::MakeRefined_(ParMesh &orig_mesh, int ref_factor, int ref_type)
   MyComm = orig_mesh.GetComm();
   NRanks = orig_mesh.GetNRanks();
   MyRank = orig_mesh.GetMyRank();
-   face_nbr_el_to_face = NULL;
   glob_elem_offset = -1;
   glob_offset_sequence = -1;
   gtopo = orig_mesh.gtopo;
@@ -2078,11 +2073,6 @@ void ParMesh::ExchangeFaceNbrData()

   ExchangeFaceNbrData(gr_sface, s2l_face);

-   if (Dim == 3)
-   {
-      GetFaceNbrElementToFaceTable();
-   }
-
   if (del_tables) { delete gr_sface; }

   if ( have_face_nbr_data ) { return; }
@@ -2151,8 +2141,6 @@ void ParMesh::ExchangeFaceNbrData(Table *gr_sface, int *s2l_face)
   el_marker = -1;
   vertex_marker = -1;

-   Array<int> fcs, cor;
-
   Table send_face_nbr_elemdata, send_face_nbr_facedata;

   send_face_nbr_elements.MakeI(num_face_nbrs);
@@ -2182,9 +2170,7 @@ void ParMesh::ExchangeFaceNbrData(Table *gr_sface, int *s2l_face)
                  send_face_nbr_vertices.AddAColumnInRow(fn);
               }

-            const int nf = elements[el]->GetNFaces();
-
-            send_face_nbr_elemdata.AddColumnsInRow(fn, nv + nf + 2);
+            send_face_nbr_elemdata.AddColumnsInRow(fn, nv + 2);
         }
      }
      send_face_nbr_facedata.AddColumnsInRow(fn, 2*num_sfaces);
@@ -2236,13 +2222,6 @@ void ParMesh::ExchangeFaceNbrData(Table *gr_sface, int *s2l_face)
            send_face_nbr_elemdata.AddConnection(
               fn, GetElementBaseGeometry(el));
            send_face_nbr_elemdata.AddConnections(fn, v, nv);
-
-            if (Dim == 3)
-            {
-               const int nf = elements[el]->GetNFaces();
-               GetElementFaces(el, fcs, cor);
-               send_face_nbr_elemdata.AddConnections(fn, cor, nf);
-            }
         }
         send_face_nbr_facedata.AddConnection(fn, el);
         int info = faces_info[lface].Elem1Inf;
@@ -2288,13 +2267,12 @@ void ParMesh::ExchangeFaceNbrData(Table *gr_sface, int *s2l_face)
      for (int el = 0; el < num_elems; el++)
      {
         const int nv = elements[elems[el]]->GetNVertices();
-         const int nf = (Dim == 3) ? elements[elems[el]]->GetNFaces() : 0;
         elemdata += 2; // skip the attribute and the geometry type
         for (int j = 0; j < nv; j++)
         {
            elemdata[j] = vertex_marker[elemdata[j]];
         }
-         elemdata += nv + nf;
+         elemdata += nv;

         el_marker[elems[el]] = el;
      }
@@ -2345,8 +2323,6 @@ void ParMesh::ExchangeFaceNbrData(Table *gr_sface, int *s2l_face)

   // convert the element data into face_nbr_elements
   face_nbr_elements.SetSize(face_nbr_elements_offset[num_face_nbrs]);
-   face_nbr_el_ori.Clear();
-   face_nbr_el_ori.SetSize(face_nbr_elements_offset[num_face_nbrs], 6);
   while (true)
   {
      int fn;
@@ -2374,20 +2350,9 @@ void ParMesh::ExchangeFaceNbrData(Table *gr_sface, int *s2l_face)
         }
         el->SetVertices(recv_elemdata);
         recv_elemdata += nv;
-         if (Dim == 3)
-         {
-            int nf = el->GetNFaces();
-            int * fn_ori = face_nbr_el_ori.GetRow(elem_off);
-            for (int j = 0; j < nf; j++)
-            {
-               fn_ori[j] = recv_elemdata[j];
-            }
-            recv_elemdata += nf;
-         }
         face_nbr_elements[elem_off++] = el;
      }
   }
-   face_nbr_el_ori.Finalize();

   MPI_Waitall(num_face_nbrs, send_requests, statuses);

@@ -2546,179 +2511,6 @@ void ParMesh::ExchangeFaceNbrNodes()
   }
 }

-STable3D *ParMesh::GetSharedFacesTable()
-{
-   STable3D *sfaces_tbl = new STable3D(face_nbr_vertices.Size());
-   for (int i = 0; i < face_nbr_elements.Size(); i++)
-   {
-      const int *v = face_nbr_elements[i]->GetVertices();
-      switch (face_nbr_elements[i]->GetType())
-      {
-         case Element::TETRAHEDRON:
-         {
-            for (int j = 0; j < 4; j++)
-            {
-               const int *fv = tet_t::FaceVert[j];
-               sfaces_tbl->Push(v[fv[0]], v[fv[1]], v[fv[2]]);
-            }
-            break;
-         }
-         case Element::WEDGE:
-         {
-            for (int j = 0; j < 2; j++)
-            {
-               const int *fv = pri_t::FaceVert[j];
-               sfaces_tbl->Push(v[fv[0]], v[fv[1]], v[fv[2]]);
-            }
-            for (int j = 2; j < 5; j++)
-            {
-               const int *fv = pri_t::FaceVert[j];
-               sfaces_tbl->Push4(v[fv[0]], v[fv[1]], v[fv[2]], v[fv[3]]);
-            }
-            break;
-         }
-         case Element::HEXAHEDRON:
-         {
-            // find the face by the vertices with the smallest 3 numbers
-            // z = 0, y = 0, x = 1, y = 1, x = 0, z = 1
-            for (int j = 0; j < 6; j++)
-            {
-               const int *fv = hex_t::FaceVert[j];
-               sfaces_tbl->Push4(v[fv[0]], v[fv[1]], v[fv[2]], v[fv[3]]);
-            }
-            break;
-         }
-         default:
-            MFEM_ABORT("Unexpected type of Element.");
-      }
-   }
-   return sfaces_tbl;
-}
-
-STable3D *ParMesh::GetFaceNbrElementToFaceTable(int ret_ftbl)
-{
-   int i, *v;
-   STable3D * faces_tbl = GetFacesTable();
-   STable3D * sfaces_tbl = GetSharedFacesTable();
-
-   if (face_nbr_el_to_face != NULL)
-   {
-      delete face_nbr_el_to_face;
-   }
-   face_nbr_el_to_face = new Table(face_nbr_elements.Size(), 6);
-   for (i = 0; i < face_nbr_elements.Size(); i++)
-   {
-      v = face_nbr_elements[i]->GetVertices();
-      switch (face_nbr_elements[i]->GetType())
-      {
-         case Element::TETRAHEDRON:
-         {
-            for (int j = 0; j < 4; j++)
-            {
-               const int *fv = tet_t::FaceVert[j];
-               int lf = faces_tbl->Index(v[fv[0]], v[fv[1]], v[fv[2]]);
-               if (lf < 0)
-               {
-                  lf = sfaces_tbl->Index(v[fv[0]], v[fv[1]], v[fv[2]]);
-                  if (lf >= 0)
-                  {
-                     lf += NumOfFaces;
-                  }
-               }
-               face_nbr_el_to_face->Push(i, lf);
-            }
-            break;
-         }
-         case Element::WEDGE:
-         {
-            for (int j = 0; j < 2; j++)
-            {
-               const int *fv = pri_t::FaceVert[j];
-               face_nbr_el_to_face->Push(
-                  i, faces_tbl->Index(v[fv[0]], v[fv[1]], v[fv[2]]));
-            }
-            for (int j = 2; j < 5; j++)
-            {
-               const int *fv = pri_t::FaceVert[j];
-               int k = 0;
-               int max = v[fv[0]];
-
-               if (max < v[fv[1]]) { max = v[fv[1]], k = 1; }
-               if (max < v[fv[2]]) { max = v[fv[2]], k = 2; }
-               if (max < v[fv[3]]) { k = 3; }
-
-               switch (k)
-               {
-                  case 0:
-                     face_nbr_el_to_face->Push(
-                        i, faces_tbl->Index(v[fv[1]],v[fv[2]],v[fv[3]]));
-                     break;
-                  case 1:
-                     face_nbr_el_to_face->Push(
-                        i, faces_tbl->Index(v[fv[0]],v[fv[2]],v[fv[3]]));
-                     break;
-                  case 2:
-                     face_nbr_el_to_face->Push(
-                        i, faces_tbl->Index(v[fv[0]],v[fv[1]],v[fv[3]]));
-                     break;
-                  case 3:
-                     face_nbr_el_to_face->Push(
-                        i, faces_tbl->Index(v[fv[0]],v[fv[1]],v[fv[2]]));
-                     break;
-               }
-            }
-            break;
-         }
-         case Element::HEXAHEDRON:
-         {
-            // find the face by the vertices with the smallest 3 numbers
-            // z = 0, y = 0, x = 1, y = 1, x = 0, z = 1
-            for (int j = 0; j < 6; j++)
-            {
-               const int *fv = hex_t::FaceVert[j];
-               int k = 0;
-               int max = v[fv[0]];
-
-               if (max < v[fv[1]]) { max = v[fv[1]], k = 1; }
-               if (max < v[fv[2]]) { max = v[fv[2]], k = 2; }
-               if (max < v[fv[3]]) { k = 3; }
-
-               switch (k)
-               {
-                  case 0:
-                     face_nbr_el_to_face->Push(
-                        i, faces_tbl->Index(v[fv[1]],v[fv[2]],v[fv[3]]));
-                     break;
-                  case 1:
-                     face_nbr_el_to_face->Push(
-                        i, faces_tbl->Index(v[fv[0]],v[fv[2]],v[fv[3]]));
-                     break;
-                  case 2:
-                     face_nbr_el_to_face->Push(
-                        i, faces_tbl->Index(v[fv[0]],v[fv[1]],v[fv[3]]));
-                     break;
-                  case 3:
-                     face_nbr_el_to_face->Push(
-                        i, faces_tbl->Index(v[fv[0]],v[fv[1]],v[fv[2]]));
-                     break;
-               }
-            }
-            break;
-         }
-         default:
-            MFEM_ABORT("Unexpected type of Element.");
-      }
-   }
-   face_nbr_el_to_face->Finalize();
-
-   if (ret_ftbl)
-   {
-      return faces_tbl;
-   }
-   delete faces_tbl;
-   return NULL;
-}
-
 int ParMesh::GetFaceNbrRank(int fn) const
 {
   if (Conforming())
@@ -2736,37 +2528,6 @@ int ParMesh::GetFaceNbrRank(int fn) const
   }
 }

-void
-ParMesh::GetFaceNbrElementFaces(int i, Array<int> &fcs, Array<int> &cor) const
-{
-   int n, j;
-   int el_nbr = i - GetNE();
-   if (face_nbr_el_to_face)
-   {
-      face_nbr_el_to_face->GetRow(el_nbr, fcs);
-   }
-   else
-   {
-      MFEM_ABORT("ParMesh::GetFaceNbrElementFaces(...) : "
-                 "face_nbr_el_to_face not generated.");
-   }
-   if (el_nbr < face_nbr_el_ori.Size())
-   {
-      const int * row = face_nbr_el_ori.GetRow(el_nbr);
-      n = fcs.Size();
-      cor.SetSize(n);
-      for (j=0; j<n; j++)
-      {
-         cor[j] = row[j];
-      }
-   }
-   else
-   {
-      MFEM_ABORT("ParMesh::GetFaceNbrElementFaces(...) : "
-                 "face_nbr_el_to_face not generated.");
-   }
-}
-
 Table *ParMesh::GetFaceToAllElementTable() const
 {
   const Array<int> *s2l_face;
@@ -75,9 +75,6 @@ protected:
   // sface ids: all triangles first, then all quads
   Array<int> sface_lface;

-   Table *face_nbr_el_to_face;
-   Table  face_nbr_el_ori; // orientations for each face (from nbr processor)
-
   IsoparametricTransformation FaceNbrTransformation;

   // glob_elem_offset + local element number defines a global element numbering
@@ -106,8 +103,6 @@ protected:
   bool DecodeFaceSplittings(HashTable<Hashed2> &v_to_v, const int *v,
                             const Array<unsigned> &codes, int &pos);

-   STable3D *GetFaceNbrElementToFaceTable(int ret_ftbl = 0);
-
   void GetFaceNbrElementTransformation(
      int i, IsoparametricTransformation *ElTr);

@@ -201,9 +196,6 @@ protected:
   void BuildSharedVertMapping(int nvert, const Table* vert_element,
                               const Array<int> &vert_global_local);

-   // Similar to Mesh::GetFacesTable()
-   STable3D *GetSharedFacesTable();
-
   /// Ensure that bdr_attributes and attributes agree across processors
   void DistributeAttributes(Array<int> &attr);

@@ -224,9 +216,8 @@ protected:

 public:
   /// Default constructor. Create an empty @a ParMesh.
-   ParMesh() : MyComm(0), NRanks(0), MyRank(-1), face_nbr_el_to_face(NULL),
-      glob_elem_offset(-1), glob_offset_sequence(-1),
-      have_face_nbr_data(false), pncmesh(NULL) { }
+   ParMesh() : MyComm(0), NRanks(0), MyRank(-1), glob_elem_offset(-1),
+      glob_offset_sequence(-1), have_face_nbr_data(false), pncmesh(NULL) { }

   /// Create a parallel mesh by partitioning a serial Mesh.
   /** The mesh is partitioned automatically or using external partitioning
@@ -350,9 +341,6 @@ public:
   int GetFaceNbrGroup(int fn) const { return face_nbr_group[fn]; }
   int GetFaceNbrRank(int fn) const;

-   /** Similar to Mesh::GetElementFaces */
-   void GetFaceNbrElementFaces(int i, Array<int> &fcs, Array<int> &cor) const;
-
   /** Similar to Mesh::GetFaceToElementTable with added face-neighbor elements
       with indices offset by the local number of elements. */
   Table *GetFaceToAllElementTable() const;
@@ -382,7 +370,7 @@ public:
   int GetSharedFace(int sface) const;

   /// See the remarks for the serial version in mesh.hpp
-   MFEM_DEPRECATED virtual void ReorientTetMesh();
+   virtual void ReorientTetMesh();

   /// Utility function: sum integers from all processors (Allreduce).
   virtual long ReduceInt(int value) const;
@@ -1339,37 +1339,6 @@ void ParNCMesh::LimitNCLevel(int max_nc_level)
   }
 }

-void ParNCMesh::GetFineToCoarsePartitioning(const Array<int> &derefs,
-                                            Array<int> &new_ranks) const
-{
-   new_ranks.SetSize(leaf_elements.Size()-GetNGhostElements());
-   for (int i = 0; i < leaf_elements.Size()-GetNGhostElements(); i++)
-   {
-      new_ranks[i] = elements[leaf_elements[i]].rank;
-   }
-
-   for (int i = 0; i < derefs.Size(); i++)
-   {
-      int row = derefs[i];
-      MFEM_VERIFY(row >= 0 && row < derefinements.Size(),
-                  "invalid derefinement number.");
-
-      const int* fine = derefinements.GetRow(row);
-      int size = derefinements.RowSize(row);
-
-      int coarse_rank = INT_MAX;
-      for (int j = 0; j < size; j++)
-      {
-         int fine_rank = elements[leaf_elements[fine[j]]].rank;
-         coarse_rank = std::min(coarse_rank, fine_rank);
-      }
-      for (int j = 0; j < size; j++)
-      {
-         new_ranks[fine[j]] = coarse_rank;
-      }
-   }
-}
-
 void ParNCMesh::Derefine(const Array<int> &derefs)
 {
   MFEM_VERIFY(Dim < 3 || Iso,
@@ -95,11 +95,6 @@ public:
       in sync. The interface is identical. */
   virtual void Derefine(const Array<int> &derefs);

-   /** Gets partitioning for the coarse mesh if the current fine mesh were to
-       be derefined. */
-   virtual void GetFineToCoarsePartitioning(const Array<int> &derefs,
-                                            Array<int> &new_ranks) const;
-
   /** Migrate leaf elements of the global refinement hierarchy (including ghost
       elements) so that each processor owns the same number of leaves (+-1).
       The default partitioning strategy is based on equal splitting of the
@@ -248,6 +243,7 @@ public:
       The debug mesh will have element attributes set to element rank + 1. */
   void GetDebugMesh(Mesh &debug_mesh) const;

+
 protected: // interface for ParMesh

   friend class ParMesh;
@@ -84,7 +84,8 @@ public:
       the PUMI and MFEM meshes. E.g.,
       PUMI_tet{v0,v1,v2,v3}  --->  MFEM_tet{v1,v0,v3,v2}
       * Note that change in the orientation can be caused by
-         fixing wrong boundary element orientations */
+         a) fixing wrong boundary element orientations
+         b) a call to ReorientTetMesh() which is required for Nedelec */
   int RotationPUMItoMFEM(apf::Mesh2* apf_mesh,
                          apf::MeshEntity* tet,
                          int elemId);
@@ -409,14 +409,18 @@ int main(int argc, char *argv[])
      ref_list.DeleteAll();
   }

-   // 10. Rebalance the mesh. Since the mesh was adaptively refined in a
+   // 10. Reorient the mesh. Must be done after refinement but before definition
+   //     of higher order Nedelec spaces
+   pmesh->ReorientTetMesh();
+
+   // 11. Rebalance the mesh. Since the mesh was adaptively refined in a
   //     non-uniform way it will be computationally unbalanced.
   if (pmesh->Nonconforming())
   {
      pmesh->Rebalance();
   }

-   // 11. Define the parallel finite element spaces. We use:
+   // 12. Define the parallel finite element spaces. We use:
   //
   //     H(curl) for electric field,
   //     H(div) for magnetic flux,
@@ -464,14 +468,13 @@ int main(int argc, char *argv[])
   int Vsize_rt = HDivFESpace.GetVSize();
   int Vsize_h1 = HGradFESpace.GetVSize();

-   // 12. Declare storage for field data.
-   //     The big BlockVector stores the fields as
-   //       0 Temperature
-   //       1 Temperature Flux
-   //       2 P field
-   //       3 E field
-   //       4 B field
-   //       5 Joule Heating
+   // the big BlockVector stores the fields as
+   //    0 Temperature
+   //    1 Temperature Flux
+   //    2 P field
+   //    3 E field
+   //    4 B field
+   //    5 Joule Heating

   Array<int> true_offset(7);
   true_offset[0] = 0;
@@ -35,7 +35,7 @@
 //    mpirun -np 2 pfindpts -m ../../data/inline-wedge.mesh -o 3
 //    mpirun -np 2 pfindpts -m ../../data/amr-quad.mesh -o 2
 //    mpirun -np 2 pfindpts -m ../../data/rt-2d-q3.mesh -o 3 -mo 4 -ft 2
-//    mpirun -np 2 pfindpts -m ../../data/inline-quad.mesh -ft 1 -no-vis -sr0
+

 #include "mfem.hpp"

@@ -74,7 +74,6 @@ int main (int argc, char *argv[])
   bool visualization    = true;
   int fieldtype         = 0;
   int ncomp             = 1;
-   bool search_on_rank_0 = false;

   // Parse command-line options.
   OptionsParser args(argc, argv);
@@ -95,9 +94,6 @@ int main (int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&search_on_rank_0, "-sr0", "--search-on-r0", "-no-sr0",
-                  "--no-search-on-r0",
-                  "Enable search only on rank 0 (disable to search points on all tasks).");
   args.Parse();
   if (!args.Good())
   {
@@ -215,7 +211,7 @@ int main (int argc, char *argv[])
   // Note that some points might be outside, if the mesh is not a box. Note
   // also that all tasks search the same points (not mandatory).
   const int pts_cnt_1D = 10;
-   int pts_cnt = pow(pts_cnt_1D, dim);
+   const int pts_cnt = pow(pts_cnt_1D, dim);
   Vector vxyz(pts_cnt * dim);
   if (dim == 2)
   {
@@ -241,12 +237,6 @@ int main (int argc, char *argv[])
      }
   }

-   if ( (myid != 0) && (search_on_rank_0) )
-   {
-      pts_cnt = 0;
-      vxyz.Destroy();
-   }
-
   // Find and Interpolate FE function values on the desired points.
   Vector interp_vals(pts_cnt*vec_dim);
   FindPointsGSLIB finder(MPI_COMM_WORLD);
@@ -256,37 +246,36 @@ int main (int argc, char *argv[])
   Array<unsigned int> task_id_out = finder.GetProc();
   Vector dist_p_out = finder.GetDist();

-   // Print the results for task 0 since either 1) all tasks have the
-   // same set of points or 2) only task 0 has any points.
-   if (myid == 0 )
+   int face_pts = 0, not_found = 0, found_loc = 0, found_away = 0;
+   double max_err = 0.0, max_dist = 0.0;
+   Vector pos(dim);
+   int npt = 0;
+   for (int j = 0; j < vec_dim; j++)
   {
-      int face_pts = 0, not_found = 0, found_loc = 0, found_away = 0;
-      double max_err = 0.0, max_dist = 0.0;
-      Vector pos(dim);
-      int npt = 0;
-      for (int j = 0; j < vec_dim; j++)
+      for (int i = 0; i < pts_cnt; i++)
      {
-         for (int i = 0; i < pts_cnt; i++)
+         if (j == 0)
         {
-            if (j == 0)
-            {
-               (task_id_out[i] == (unsigned)myid) ? found_loc++ : found_away++;
-            }
-
-            if (code_out[i] < 2)
-            {
-               for (int d = 0; d < dim; d++) { pos(d) = vxyz(d * pts_cnt + i); }
-               Vector exact_val(vec_dim);
-               F_exact(pos, exact_val);
-               max_err  = std::max(max_err, fabs(exact_val(j) - interp_vals(npt)));
-               max_dist = std::max(max_dist, dist_p_out(i));
-               if (code_out[i] == 1 && j == 0) { face_pts++; }
-            }
-            else { if (j == 0) { not_found++; } }
-            npt++;
+            (task_id_out[i] == (unsigned)myid) ? found_loc++ : found_away++;
         }
-      }

+         if (code_out[i] < 2)
+         {
+            for (int d = 0; d < dim; d++) { pos(d) = vxyz(d * pts_cnt + i); }
+            Vector exact_val(vec_dim);
+            F_exact(pos, exact_val);
+            max_err  = std::max(max_err, fabs(exact_val(j) - interp_vals(npt)));
+            max_dist = std::max(max_dist, dist_p_out(i));
+            if (code_out[i] == 1 && j == 0) { face_pts++; }
+         }
+         else { if (j == 0) { not_found++; } }
+         npt++;
+      }
+   }
+
+   // Print the results for task 0 since all tasks have the same set of points.
+   if (myid == 0)
+   {
      cout << setprecision(16)
           << "Searched unique points: " << pts_cnt
           << "\nFound on local mesh:  " << found_loc
@@ -39,11 +39,6 @@
 //   Adapted analytic shape+orientation:
 //     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 85 -tid 4 -ni 100 -bnd -qt 1 -qo 8 -fd
 //
-//   Adapted analytic shape and/or size with hr-adaptivity:
-//     mesh-optimizer -m square01.mesh -o 2 -tid 9  -ni 50 -li 20 -hmid 55 -mid 7 -hr
-//     mesh-optimizer -m square01.mesh -o 2 -tid 10 -ni 50 -li 20 -hmid 55 -mid 7 -hr
-//     mesh-optimizer -m square01.mesh -o 2 -tid 11 -ni 50 -li 20 -hmid 58 -mid 7 -hr
-//
 //   Adapted discrete size:
 //     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 80 -tid 5 -ni 50 -qo 4 -nor
 //   Adapted discrete size 3D with PA:
@@ -128,8 +123,6 @@ int main(int argc, char *argv[])
   int max_lin_iter      = 100;
   bool move_bnd         = true;
   int combomet          = 0;
-   bool hradaptivity     = false;
-   int h_metric_id       = -1;
   bool normalization    = false;
   bool visualization    = true;
   int verbosity_level   = 0;
@@ -138,8 +131,6 @@ int main(int argc, char *argv[])
   bool exactaction      = false;
   const char *devopt    = "cpu";
   bool pa               = false;
-   int n_hr_iter         = 5;
-   int n_h_iter          = 1;

   // 1. Parse command-line options.
   OptionsParser args(argc, argv);
@@ -230,12 +221,6 @@ int main(int argc, char *argv[])
                  "0: Use single metric\n\t"
                  "1: Shape + space-dependent size given analytically\n\t"
                  "2: Shape + adapted size given discretely; shared target");
-   args.AddOption(&hradaptivity, "-hr", "--hr-adaptivity", "-no-hr",
-                  "--no-hr-adaptivity",
-                  "Enable hr-adaptivity.");
-   args.AddOption(&h_metric_id, "-hmid", "--h-metric",
-                  "Same options as metric_id. Used to determine refinement"
-                  " type for each element if h-adaptivity is enabled.");
   args.AddOption(&normalization, "-nor", "--normalization", "-no-nor",
                  "--no-normalization",
                  "Make all terms in the optimization functional unitless.");
@@ -256,11 +241,6 @@ int main(int argc, char *argv[])
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&n_hr_iter, "-nhr", "--n_hr_iter",
-                  "Number of hr-adaptivity iterations.");
-   args.AddOption(&n_h_iter, "-nh", "--n_h_iter",
-                  "Number of h-adaptivity iterations per r-adaptivity"
-                  "iteration.");
   args.Parse();
   if (!args.Good())
   {
@@ -269,13 +249,6 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   if (h_metric_id < 0) { h_metric_id = metric_id; }
-
-   if (hradaptivity)
-   {
-      MFEM_VERIFY(strcmp(devopt,"cpu")==0, "HR-adaptivity is currently only"
-                  " supported on cpus.");
-   }
   Device device(devopt);
   device.Print();

@@ -288,9 +261,7 @@ int main(int argc, char *argv[])
   else { cout << "(NONE)"; }
   cout << endl;

-   if (hradaptivity) { mesh->EnsureNCMesh(); }
-
-   // 3. Define a finite element space on the mesh-> Here we use vector finite
+   // 3. Define a finite element space on the mesh. Here we use vector finite
   //    elements which are tensor products of quadratic finite elements. The
   //    number of components in the vector finite element space is specified by
   //    the last parameter of the FiniteElementSpace constructor.
@@ -421,41 +392,9 @@ int main(int argc, char *argv[])
         cout << "Unknown metric_id: " << metric_id << endl;
         return 3;
   }
-   TMOP_QualityMetric *h_metric = NULL;
-   if (hradaptivity)
-   {
-      switch (h_metric_id)
-      {
-         case 1: h_metric = new TMOP_Metric_001; break;
-         case 2: h_metric = new TMOP_Metric_002; break;
-         case 7: h_metric = new TMOP_Metric_007; break;
-         case 9: h_metric = new TMOP_Metric_009; break;
-         case 55: h_metric = new TMOP_Metric_055; break;
-         case 56: h_metric = new TMOP_Metric_056; break;
-         case 58: h_metric = new TMOP_Metric_058; break;
-         case 77: h_metric = new TMOP_Metric_077; break;
-         case 315: h_metric = new TMOP_Metric_315; break;
-         case 316: h_metric = new TMOP_Metric_316; break;
-         case 321: h_metric = new TMOP_Metric_321; break;
-         default: cout << "Metric_id not supported for h-adaptivity: " << h_metric_id <<
-                          endl;
-            return 3;
-      }
-   }
-
-   if (metric_id < 300 || h_metric_id < 300)
-   {
-      MFEM_VERIFY(dim == 2, "Incompatible metric for 3D meshes");
-   }
-   if (metric_id >= 300 || h_metric_id >= 300)
-   {
-      MFEM_VERIFY(dim == 3, "Incompatible metric for 2D meshes");
-   }
-
   TargetConstructor::TargetType target_t;
   TargetConstructor *target_c = NULL;
   HessianCoefficient *adapt_coeff = NULL;
-   HRHessianCoefficient *hr_adapt_coeff = NULL;
   H1_FECollection ind_fec(mesh_poly_deg, dim);
   FiniteElementSpace ind_fes(mesh, &ind_fec);
   FiniteElementSpace ind_fesv(mesh, &ind_fec, dim);
@@ -665,18 +604,6 @@ int main(int argc, char *argv[])
         target_c = tc;
         break;
      }
-      // Targets used for hr-adaptivity tests.
-      case 9:  // size target in an annular region.
-      case 10: // size+aspect-ratio in an annular region.
-      case 11: // size+aspect-ratio target for a rotate sine wave
-      {
-         target_t = TargetConstructor::GIVEN_FULL;
-         AnalyticAdaptTC *tc = new AnalyticAdaptTC(target_t);
-         hr_adapt_coeff = new HRHessianCoefficient(dim, target_id - 9);
-         tc->SetAnalyticTargetSpec(NULL, NULL, hr_adapt_coeff);
-         target_c = tc;
-         break;
-      }
      default: cout << "Unknown target_id: " << target_id << endl; return 3;
   }
   if (target_c == NULL)
@@ -684,8 +611,7 @@ int main(int argc, char *argv[])
      target_c = new TargetConstructor(target_t);
   }
   target_c->SetNodes(x0);
-   TMOP_Integrator *he_nlf_integ = new TMOP_Integrator(metric, target_c,
-                                                       h_metric);
+   TMOP_Integrator *he_nlf_integ = new TMOP_Integrator(metric, target_c);

   // Finite differences for computations of derivatives.
   if (fdscheme)
@@ -795,10 +721,10 @@ int main(int argc, char *argv[])
            TargetConstructor::IDEAL_SHAPE_EQUAL_SIZE);
         target_c2->SetVolumeScale(0.01);
         target_c2->SetNodes(x0);
-         he_nlf_integ2 = new TMOP_Integrator(metric2, target_c2, h_metric);
+         he_nlf_integ2 = new TMOP_Integrator(metric2, target_c2);
         he_nlf_integ2->SetCoefficient(coeff2);
      }
-      else { he_nlf_integ2 = new TMOP_Integrator(metric2, target_c, h_metric); }
+      else { he_nlf_integ2 = new TMOP_Integrator(metric2, target_c); }
      he_nlf_integ2->SetIntegrationRules(*irules, quad_order);
      if (fdscheme) { he_nlf_integ2->EnableFiniteDifferences(x); }
      he_nlf_integ2->SetExactActionFlag(exactaction);
@@ -852,9 +778,7 @@ int main(int argc, char *argv[])
      tauval -= 0.01 * h0.Min();
   }

-   // For HR tests, the energy is normalized by the number of elements.
-   const double init_energy = a.GetGridFunctionEnergy(x) /
-                              (hradaptivity ? mesh->GetNE() : 1);
+   const double init_energy = a.GetGridFunctionEnergy(x);

   // Visualize the starting mesh and metric values.
   // Note that for combinations of metrics, this only shows the first metric.
@@ -982,25 +906,13 @@ int main(int argc, char *argv[])
      solver.SetAdaptiveLinRtol(solver_art_type, 0.5, 0.9);
   }
   solver.SetPrintLevel(verbosity_level >= 1 ? 1 : -1);
-
-   // hr-adaptivity solver.
-   // If hr-adaptivity is disabled, r-adaptivity is done once using the
-   // TMOPNewtonSolver.
-   // Otherwise, "hr_iter" iterations of r-adaptivity are done followed by
-   // "h_per_r_iter" iterations of h-adaptivity after each r-adaptivity.
-   // The solver terminates if an h-adaptivity iteration does not modify
-   // any element in the mesh.
-   TMOPHRSolver hr_solver(*mesh, a, solver,
-                          x, move_bnd, hradaptivity,
-                          mesh_poly_deg, h_metric_id,
-                          n_hr_iter, n_h_iter);
-   hr_solver.AddGridFunctionForUpdate(&x0);
-   if (adapt_lim_const > 0.)
+   solver.SetOperator(a);
+   solver.Mult(b, x.GetTrueVector());
+   x.SetFromTrueVector();
+   if (solver.GetConverged() == false)
   {
-      hr_solver.AddGridFunctionForUpdate(&zeta_0);
-      hr_solver.AddFESpaceForUpdate(&ind_fes);
+      cout << "Nonlinear solver: rtol = " << solver_rtol << " not achieved.\n";
   }
-   hr_solver.Mult();

   // 15. Save the optimized mesh to a file. This output can be viewed later
   //     using GLVis: "glvis -m optimized.mesh".
@@ -1010,15 +922,14 @@ int main(int argc, char *argv[])
      mesh->Print(mesh_ofs);
   }

-   const double fin_energy = a.GetGridFunctionEnergy(x) /
-                             (hradaptivity ? mesh->GetNE() : 1);
+   // 16. Compute the amount of energy decrease.
+   const double fin_energy = a.GetGridFunctionEnergy(x);
   double metric_part = fin_energy;
   if (lim_const > 0.0 || adapt_lim_const > 0.0)
   {
      lim_coeff.constant = 0.0;
      coef_zeta.constant = 0.0;
-      metric_part = a.GetGridFunctionEnergy(x) /
-                    (hradaptivity ? mesh->GetNE() : 1);
+      metric_part = a.GetGridFunctionEnergy(x);
      lim_coeff.constant = lim_const;
      coef_zeta.constant = adapt_lim_const;
   }
@@ -1031,7 +942,7 @@ int main(int argc, char *argv[])
   cout << "The strain energy decreased by: " << setprecision(12)
        << (init_energy - fin_energy) * 100.0 / init_energy << " %." << endl;

-   // 16. Visualize the final mesh and metric values.
+   // 17. Visualize the final mesh and metric values.
   if (visualization)
   {
      char title[] = "Final metric values";
@@ -1045,13 +956,13 @@ int main(int argc, char *argv[])
                             600, 600, 300, 300);
   }

-   // 17. Visualize the mesh displacement.
+   // 18. Visualize the mesh displacement.
   if (visualization)
   {
+      x0 -= x;
      osockstream sock(19916, "localhost");
      sock << "solution\n";
      mesh->Print(sock);
-      x0 -= x;
      x0.Save(sock);
      sock.send();
      sock << "window_title 'Displacements'\n"
@@ -1060,6 +971,7 @@ int main(int argc, char *argv[])
           << "keys jRmclA" << endl;
   }

+   // 19. Free the used memory.
   delete S;
   delete S_prec;
   delete target_c2;
@@ -1067,9 +979,7 @@ int main(int argc, char *argv[])
   delete coeff1;
   delete adapt_evaluator;
   delete target_c;
-   delete hr_adapt_coeff;
   delete adapt_coeff;
-   delete h_metric;
   delete metric;
   delete fespace;
   delete fec;
@@ -90,6 +90,8 @@ double discrete_aspr_2d(const Vector &x)
   double th = 22.5*M_PI/180.;
   double xn =  cos(th)*xc + sin(th)*yc;
   double yn = -sin(th)*xc + cos(th)*yc;
+   // double th2 = (th > 45.*M_PI/180) ? M_PI/2 - th : th;
+   // double stretch = 1/cos(th2);
   xc = xn; yc = yn;

   double tfac = 20;
@@ -221,134 +223,6 @@ public:
   }
 };

-class HRHessianCoefficient : public TMOPMatrixCoefficient
-{
-private:
-   int dim;
-   // 0 - size target in an annular region,
-   // 1 - size+aspect-ratio in an annular region,
-   // 2 - size+aspect-ratio target for a rotate sine wave.
-   int hr_target_type;
-
-public:
-   HRHessianCoefficient(int dim_, int hr_target_type_ = 0)
-      : TMOPMatrixCoefficient(dim_), dim(dim_),
-        hr_target_type(hr_target_type_) { }
-
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
-                     const IntegrationPoint &ip)
-   {
-      Vector pos(3);
-      T.Transform(ip, pos);
-      if (hr_target_type == 0) // size only circle
-      {
-         double small = 0.001, big = 0.01;
-         if (dim == 3) { small = 0.005, big = 0.1; }
-         const double xc = pos(0) - 0.5, yc = pos(1) - 0.5;
-         double zc;
-         if (dim == 3) { zc = pos(2) - 0.5; }
-         double r = sqrt(xc*xc + yc*yc);
-         if (dim == 3) { r = sqrt(xc*xc + yc*yc + zc*zc); }
-         double r1 = 0.15; double r2 = 0.35; double sf=30.0;
-
-         const double tan1 = std::tanh(sf*(r-r1)),
-                      tan2 = std::tanh(sf*(r-r2));
-
-         double ind = (tan1 - tan2);
-         if (ind > 1.0) {ind = 1.;}
-         if (ind < 0.0) {ind = 0.;}
-         double val = ind * small + (1.0 - ind) * big;
-         K = 0.0;
-         K(0, 0) = 1.0;
-         K(0, 1) = 0.0;
-         K(1, 0) = 0.0;
-         K(1, 1) = 1.0;
-         K(0, 0) *= pow(val,0.5);
-         K(1, 1) *= pow(val,0.5);
-         if (dim == 3) { K(2, 2) = pow(val,0.5); }
-      }
-      else if (hr_target_type == 1) // circle with size and AR
-      {
-         const double small = 0.001, big = 0.01;
-         const double xc = pos(0)-0.5, yc = pos(1)-0.5;
-         const double rv = xc*xc + yc*yc;
-         double r = 0;
-         if (rv>0.) {r = sqrt(rv);}
-
-         double r1 = 0.2; double r2 = 0.3; double sf=30.0;
-         const double szfac = 1;
-         const double asfac = 4;
-         const double eps2 = szfac/asfac;
-         const double eps1 = szfac;
-
-         double tan1 = std::tanh(sf*(r-r1)+1),
-                tan2 = std::tanh(sf*(r-r2)-1);
-         double wgt = 0.5*(tan1-tan2);
-
-         tan1 = std::tanh(sf*(r-r1)),
-         tan2 = std::tanh(sf*(r-r2));
-
-         double ind = (tan1 - tan2);
-         if (ind > 1.0) {ind = 1.;}
-         if (ind < 0.0) {ind = 0.;}
-         double szval = ind * small + (1.0 - ind) * big;
-
-         double th = std::atan2(yc,xc)*180./M_PI;
-         if (wgt > 1) { wgt = 1; }
-         if (wgt < 0) { wgt = 0; }
-
-         double maxval = eps2 + eps1*(1-wgt)*(1-wgt);
-         double minval = eps1;
-         double avgval = 0.5*(maxval+minval);
-         double ampval = 0.5*(maxval-minval);
-         double val1 = avgval + ampval*sin(2.*th*M_PI/180.+90*M_PI/180.);
-         double val2 = avgval + ampval*sin(2.*th*M_PI/180.-90*M_PI/180.);
-
-         K(0,1) = 0.0;
-         K(1,0) = 0.0;
-         K(0,0) = val1;
-         K(1,1) = val2;
-
-         K(0,0) *= pow(szval,0.5);
-         K(1,1) *= pow(szval,0.5);
-      }
-      else if (hr_target_type == 2) // sharp rotated sine wave
-      {
-         double xc = pos(0)-0.5, yc = pos(1)-0.5;
-         double th = 15.5*M_PI/180.;
-         double xn =  cos(th)*xc + sin(th)*yc;
-         double yn = -sin(th)*xc + cos(th)*yc;
-         double th2 = (th > 45.*M_PI/180) ? M_PI/2 - th : th;
-         double stretch = 1/cos(th2);
-         xc = xn/stretch;
-         yc = yn;
-         double tfac = 20;
-         double s1 = 3;
-         double s2 = 2;
-         double yl1 = -0.025;
-         double yl2 =  0.025;
-         double wgt = std::tanh((tfac*(yc-yl1) + s2*std::sin(s1*M_PI*xc)) + 1) -
-                      std::tanh((tfac*(yc-yl2) + s2*std::sin(s1*M_PI*xc)) - 1);
-         if (wgt > 1) { wgt = 1; }
-         if (wgt < 0) { wgt = 0; }
-
-         const double eps2 = 25;
-         const double eps1 = 1;
-         K(1,1) = eps1/eps2 + eps1*(1-wgt)*(1-wgt);
-         K(0,0) = eps1;
-         K(0,1) = 0.0;
-         K(1,0) = 0.0;
-      }
-      else { MFEM_ABORT("Unsupported option / wrong input."); }
-   }
-
-   virtual void EvalGrad(DenseMatrix &K, ElementTransformation &T,
-                         const IntegrationPoint &ip, int comp)
-   {
-      K = 0.;
-   }
-};
-
 // Additional IntegrationRules that can be used with the --quad-type option.
 IntegrationRules IntRulesLo(0, Quadrature1D::GaussLobatto);
 IntegrationRules IntRulesCU(0, Quadrature1D::ClosedUniform);
@@ -378,13 +252,13 @@ double adapt_lim_fun(const Vector &x)

 void DiffuseField(GridFunction &field, int smooth_steps)
 {
-   // Setup the Laplacian operator
+   //Setup the Laplacian operator
   BilinearForm *Lap = new BilinearForm(field.FESpace());
   Lap->AddDomainIntegrator(new DiffusionIntegrator());
   Lap->Assemble();
   Lap->Finalize();

-   // Setup the smoothing operator
+   //Setup the smoothing operator
   DSmoother *S = new DSmoother(0,1.0,smooth_steps);
   S->iterative_mode = true;
   S->SetOperator(Lap->SpMat());
@@ -400,7 +274,7 @@ void DiffuseField(GridFunction &field, int smooth_steps)
 #ifdef MFEM_USE_MPI
 void DiffuseField(ParGridFunction &field, int smooth_steps)
 {
-   // Setup the Laplacian operator
+   //Setup the Laplacian operator
   ParBilinearForm *Lap = new ParBilinearForm(field.ParFESpace());
   Lap->AddDomainIntegrator(new DiffusionIntegrator());
   Lap->Assemble();
@@ -39,11 +39,6 @@
 //   Adapted analytic shape+orientation:
 //     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 85 -tid 4 -ni 100 -bnd -qt 1 -qo 8 -fd
 //
-//   Adapted analytic shape and/or size with hr-adaptivity:
-//     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -tid 9  -ni 50 -li 20 -hmid 55 -mid 7 -hr
-//     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -tid 10 -ni 50 -li 20 -hmid 55 -mid 7 -hr
-//     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -tid 11 -ni 50 -li 20 -hmid 58 -mid 7 -hr
-//
 //   Adapted discrete size:
 //     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 80 -tid 5 -ni 50 -qo 4 -nor
 //   Adapted discrete size 3D with PA:
@@ -135,8 +130,6 @@ int main (int argc, char *argv[])
   int max_lin_iter      = 100;
   bool move_bnd         = true;
   int combomet          = 0;
-   bool hradaptivity     = false;
-   int h_metric_id       = -1;
   bool normalization    = false;
   bool visualization    = true;
   int verbosity_level   = 0;
@@ -145,8 +138,6 @@ int main (int argc, char *argv[])
   bool exactaction      = false;
   const char *devopt    = "cpu";
   bool pa               = false;
-   int n_hr_iter         = 5;
-   int n_h_iter          = 1;

   // 2. Parse command-line options.
   OptionsParser args(argc, argv);
@@ -239,12 +230,6 @@ int main (int argc, char *argv[])
                  "0: Use single metric\n\t"
                  "1: Shape + space-dependent size given analytically\n\t"
                  "2: Shape + adapted size given discretely; shared target");
-   args.AddOption(&hradaptivity, "-hr", "--hr-adaptivity", "-no-hr",
-                  "--no-hr-adaptivity",
-                  "Enable hr-adaptivity.");
-   args.AddOption(&h_metric_id, "-hmid", "--h-metric",
-                  "Same options as metric_id. Used to determine refinement"
-                  " type for each element if h-adaptivity is enabled.");
   args.AddOption(&normalization, "-nor", "--normalization", "-no-nor",
                  "--no-normalization",
                  "Make all terms in the optimization functional unitless.");
@@ -265,11 +250,6 @@ int main (int argc, char *argv[])
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&n_hr_iter, "-nhr", "--n_hr_iter",
-                  "Number of hr-adaptivity iterations.");
-   args.AddOption(&n_h_iter, "-nh", "--n_h_iter",
-                  "Number of h-adaptivity iterations per r-adaptivity"
-                  "iteration.");
   args.Parse();
   if (!args.Good())
   {
@@ -277,13 +257,7 @@ int main (int argc, char *argv[])
      return 1;
   }
   if (myid == 0) { args.PrintOptions(cout); }
-   if (h_metric_id < 0) { h_metric_id = metric_id; }

-   if (hradaptivity)
-   {
-      MFEM_VERIFY(strcmp(devopt,"cpu")==0, "HR-adaptivity is currently only"
-                  " supported on cpus.");
-   }
   Device device(devopt);
   if (myid == 0) { device.Print();}

@@ -302,7 +276,6 @@ int main (int argc, char *argv[])
      cout << endl;
   }

-   if (hradaptivity) { mesh->EnsureNCMesh(); }
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);

   delete mesh;
@@ -449,41 +422,9 @@ int main (int argc, char *argv[])
         if (myid == 0) { cout << "Unknown metric_id: " << metric_id << endl; }
         return 3;
   }
-   TMOP_QualityMetric *h_metric = NULL;
-   if (hradaptivity)
-   {
-      switch (h_metric_id)
-      {
-         case 1: h_metric = new TMOP_Metric_001; break;
-         case 2: h_metric = new TMOP_Metric_002; break;
-         case 7: h_metric = new TMOP_Metric_007; break;
-         case 9: h_metric = new TMOP_Metric_009; break;
-         case 55: h_metric = new TMOP_Metric_055; break;
-         case 56: h_metric = new TMOP_Metric_056; break;
-         case 58: h_metric = new TMOP_Metric_058; break;
-         case 77: h_metric = new TMOP_Metric_077; break;
-         case 315: h_metric = new TMOP_Metric_315; break;
-         case 316: h_metric = new TMOP_Metric_316; break;
-         case 321: h_metric = new TMOP_Metric_321; break;
-         default: cout << "Metric_id not supported for h-adaptivity: " << h_metric_id <<
-                          endl;
-            return 3;
-      }
-   }
-
-   if (metric_id < 300 || h_metric_id < 300)
-   {
-      MFEM_VERIFY(dim == 2, "Incompatible metric for 3D meshes");
-   }
-   if (metric_id >= 300 || h_metric_id >= 300)
-   {
-      MFEM_VERIFY(dim == 3, "Incompatible metric for 2D meshes");
-   }
-
   TargetConstructor::TargetType target_t;
   TargetConstructor *target_c = NULL;
   HessianCoefficient *adapt_coeff = NULL;
-   HRHessianCoefficient *hr_adapt_coeff = NULL;
   H1_FECollection ind_fec(mesh_poly_deg, dim);
   ParFiniteElementSpace ind_fes(pmesh, &ind_fec);
   ParFiniteElementSpace ind_fesv(pmesh, &ind_fec, dim);
@@ -698,18 +639,6 @@ int main (int argc, char *argv[])
         target_c = tc;
         break;
      }
-      // Targets used for hr-adaptivity tests.
-      case 9:  // size target in an annular region.
-      case 10: // size+aspect-ratio in an annular region.
-      case 11: // size+aspect-ratio target for a rotate sine wave
-      {
-         target_t = TargetConstructor::GIVEN_FULL;
-         AnalyticAdaptTC *tc = new AnalyticAdaptTC(target_t);
-         hr_adapt_coeff = new HRHessianCoefficient(dim, target_id - 9);
-         tc->SetAnalyticTargetSpec(NULL, NULL, hr_adapt_coeff);
-         target_c = tc;
-         break;
-      }
      default:
         if (myid == 0) { cout << "Unknown target_id: " << target_id << endl; }
         return 3;
@@ -720,13 +649,13 @@ int main (int argc, char *argv[])
      target_c = new TargetConstructor(target_t, MPI_COMM_WORLD);
   }
   target_c->SetNodes(x0);
-   TMOP_Integrator *he_nlf_integ = new TMOP_Integrator(metric, target_c,
-                                                       h_metric);
+   TMOP_Integrator *he_nlf_integ= new TMOP_Integrator(metric, target_c);

   // Finite differences for computations of derivatives.
   if (fdscheme)
   {
      MFEM_VERIFY(pa == false, "PA for finite differences is not implemented.");
+
      he_nlf_integ->EnableFiniteDifferences(x);
   }
   he_nlf_integ->SetExactActionFlag(exactaction);
@@ -824,8 +753,7 @@ int main (int argc, char *argv[])
      he_nlf_integ->SetCoefficient(*coeff1);

      // Second metric.
-      if (dim == 2) { metric2 = new TMOP_Metric_077; }
-      else          { metric2 = new TMOP_Metric_315; }
+      metric2 = new TMOP_Metric_077;
      TMOP_Integrator *he_nlf_integ2 = NULL;
      if (combomet == 1)
      {
@@ -833,10 +761,10 @@ int main (int argc, char *argv[])
            TargetConstructor::IDEAL_SHAPE_EQUAL_SIZE, MPI_COMM_WORLD);
         target_c2->SetVolumeScale(0.01);
         target_c2->SetNodes(x0);
-         he_nlf_integ2 = new TMOP_Integrator(metric2, target_c2, h_metric);
+         he_nlf_integ2 = new TMOP_Integrator(metric2, target_c2);
         he_nlf_integ2->SetCoefficient(coeff2);
      }
-      else { he_nlf_integ2 = new TMOP_Integrator(metric2, target_c, h_metric); }
+      else { he_nlf_integ2 = new TMOP_Integrator(metric2, target_c); }
      he_nlf_integ2->SetIntegrationRules(*irules, quad_order);
      if (fdscheme) { he_nlf_integ2->EnableFiniteDifferences(x); }
      he_nlf_integ2->SetExactActionFlag(exactaction);
@@ -849,10 +777,7 @@ int main (int argc, char *argv[])

      a.AddDomainIntegrator(combo);
   }
-   else
-   {
-      a.AddDomainIntegrator(he_nlf_integ);
-   }
+   else { a.AddDomainIntegrator(he_nlf_integ); }

   if (pa) { a.Setup(); }

@@ -896,9 +821,7 @@ int main (int argc, char *argv[])
      tauval -= 0.01 * h0min_all;
   }

-   // For HR tests, the energy is normalized by the number of elements.
-   const double init_energy = a.GetParGridFunctionEnergy(x) /
-                              (hradaptivity ? pmesh->GetGlobalNE() : 1);
+   const double init_energy = a.GetParGridFunctionEnergy(x);

   // Visualize the starting mesh and metric values.
   // Note that for combinations of metrics, this only shows the first metric.
@@ -1029,25 +952,13 @@ int main (int argc, char *argv[])
      solver.SetAdaptiveLinRtol(solver_art_type, 0.5, 0.9);
   }
   solver.SetPrintLevel(verbosity_level >= 1 ? 1 : -1);
-
-   // hr-adaptivity solver.
-   // If hr-adaptivity is disabled, r-adaptivity is done once using the
-   // TMOPNewtonSolver.
-   // Otherwise, "hr_iter" iterations of r-adaptivity are done followed by
-   // "h_per_r_iter" iterations of h-adaptivity after each r-adaptivity.
-   // The solver terminates if an h-adaptivity iteration does not modify
-   // any element in the mesh.
-   TMOPHRSolver hr_solver(*pmesh, a, solver,
-                          x, move_bnd, hradaptivity,
-                          mesh_poly_deg, h_metric_id,
-                          n_hr_iter, n_h_iter);
-   hr_solver.AddGridFunctionForUpdate(&x0);
-   if (adapt_lim_const > 0.)
+   solver.SetOperator(a);
+   solver.Mult(b, x.GetTrueVector());
+   x.SetFromTrueVector();
+   if (myid == 0 && solver.GetConverged() == false)
   {
-      hr_solver.AddGridFunctionForUpdate(&zeta_0);
-      hr_solver.AddFESpaceForUpdate(&ind_fes);
+      cout << "Nonlinear solver: rtol = " << solver_rtol << " not achieved.\n";
   }
-   hr_solver.Mult();

   // 16. Save the optimized mesh to a file. This output can be viewed later
   //     using GLVis: "glvis -m optimized -np num_mpi_tasks".
@@ -1060,15 +971,13 @@ int main (int argc, char *argv[])
   }

   // 17. Compute the amount of energy decrease.
-   const double fin_energy = a.GetParGridFunctionEnergy(x) /
-                             (hradaptivity ? pmesh->GetGlobalNE() : 1);
+   const double fin_energy = a.GetParGridFunctionEnergy(x);
   double metric_part = fin_energy;
   if (lim_const > 0.0 || adapt_lim_const > 0.0)
   {
      lim_coeff.constant = 0.0;
      coef_zeta.constant = 0.0;
-      metric_part = a.GetParGridFunctionEnergy(x) /
-                    (hradaptivity ? pmesh->GetGlobalNE() : 1);
+      metric_part = a.GetParGridFunctionEnergy(x);
      lim_coeff.constant = lim_const;
      coef_zeta.constant = adapt_lim_const;
   }
@@ -1127,9 +1036,7 @@ int main (int argc, char *argv[])
   delete coeff1;
   delete adapt_evaluator;
   delete target_c;
-   delete hr_adapt_coeff;
   delete adapt_coeff;
-   delete h_metric;
   delete metric;
   delete pfespace;
   delete fec;
@@ -56,10 +56,8 @@ then
        prefix_opt="--prefix=${prefix}"
    fi

-    echo "Fetching uberenv."
    tests/gitlab/get_mfem_uberenv || ( echo "Error fetching Uberenv" && exit 1 );

-    echo "Installing dependencies."
    python tests/uberenv/uberenv.py --spec="${spec}" "${upstream_opt}" "${prefix_opt}"

 fi
@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset

 uberenv_url="https://github.com/mfem/mfem-uberenv.git"
-uberenv_ref="222e3743377d8c903e044f80ae44624e1668feb4"
+uberenv_ref="dbe35eb4dbeaabc7184ad110fcb98377092baeb0"

 [[ ! -d tests/uberenv ]] && git clone ${uberenv_url} tests/uberenv
 cd tests/uberenv
@@ -4003,6 +4003,11 @@ TEST_CASE("3D Bilinear Weak Curl Integrators",
      Mesh mesh =
         Mesh::MakeCartesian3D(n, n, n, (Element::Type)type, 2.0, 3.0, 5.0);

+      if (type == Element::TETRAHEDRON)
+      {
+         mesh.ReorientTetMesh();
+      }
+
      SECTION("Operators on ND for element type " + std::to_string(type))
      {
         ND_FECollection    fec_nd(order, dim);
@@ -4462,6 +4467,11 @@ TEST_CASE("3D Bilinear Weak Curl Cross Integrators",
      Mesh mesh =
         Mesh::MakeCartesian3D(n, n, n, (Element::Type)type, 2.0, 3.0, 5.0);

+      if (type == Element::TETRAHEDRON)
+      {
+         mesh.ReorientTetMesh();
+      }
+
      SECTION("Operators on ND for element type " + std::to_string(type))
      {
         ND_FECollection    fec_nd(order, dim);
@@ -5310,9 +5320,15 @@ TEST_CASE("3D Bilinear Curl Curl Integrators",
   for (int type = (int)Element::TETRAHEDRON;
        type <= (int)Element::HEXAHEDRON; type++)
   {
+      type++;
      Mesh mesh =
         Mesh::MakeCartesian3D(n, n, n, (Element::Type)type, 2.0, 3.0, 5.0);

+      if (type == Element::TETRAHEDRON)
+      {
+         mesh.ReorientTetMesh();
+      }
+
      SECTION("Operators on ND for element type " + std::to_string(type))
      {
         ND_FECollection    fec_nd(order, dim);
@@ -5419,6 +5435,11 @@ TEST_CASE("3D Bilinear Mixed Curl Curl Integrators",
      Mesh mesh =
         Mesh::MakeCartesian3D(n, n, n, (Element::Type)type, 2.0, 3.0, 5.0);

+      if (type == Element::TETRAHEDRON)
+      {
+         mesh.ReorientTetMesh();
+      }
+
      SECTION("Operators on ND for element type " + std::to_string(type))
      {
         ND_FECollection    fec_nd(order, dim);
@@ -5576,6 +5597,11 @@ TEST_CASE("3D Bilinear Mixed Cross Curl Curl Integrators",
      Mesh mesh =
         Mesh::MakeCartesian3D(n, n, n, (Element::Type)type, 2.0, 3.0, 5.0);

+      if (type == Element::TETRAHEDRON)
+      {
+         mesh.ReorientTetMesh();
+      }
+
      SECTION("Operators on ND for element type " + std::to_string(type))
      {
         ND_FECollection    fec_nd(order, dim);
@@ -5648,6 +5674,11 @@ TEST_CASE("3D Bilinear Mixed Cross Grad Curl Integrators",
      Mesh mesh =
         Mesh::MakeCartesian3D(n, n, n, (Element::Type)type, 2.0, 3.0, 5.0);

+      if (type == Element::TETRAHEDRON)
+      {
+         mesh.ReorientTetMesh();
+      }
+
      SECTION("Operators on H1 for element type " + std::to_string(type))
      {
         H1_FECollection    fec_h1(order, dim);
@@ -72,6 +72,10 @@ TEST_CASE("Build Dof To Arrays",
      {
         mesh->UniformRefinement();
      }
+      if (dim == 3)
+      {
+         mesh->ReorientTetMesh();
+      }

      for (int bt = (int)BasisType::H1; bt <= (int)BasisType::L2; bt++)
      {
@@ -173,6 +177,10 @@ TEST_CASE("Build Dof To Arrays (Parallel)",
      }
      ParMesh pmesh(MPI_COMM_WORLD, *mesh);
      delete mesh;
+      if (dim == 3)
+      {
+         pmesh.ReorientTetMesh();
+      }

      for (int bt = (int)BasisType::H1; bt <= (int)BasisType::L2; bt++)
      {
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Valentin Dallerit	360b9e8596	Merge branch 'fusion-ldrd-dev' into epic-dev	2022-03-30 12:35:54 -07:00
Chris Vogl	260f9e2275	Merge remote-tracking branch 'origin/master' into fusion-ldrd-dev	2021-08-02 16:00:08 -07:00
Valentin Dallerit	449b49f8cc	Fix typo in cmake file	2021-07-06 21:18:33 -07:00
Valentin Dallerit	bead5f3004	Update example exponential integrator	2021-07-01 16:41:56 -07:00
Valentin Dallerit	aba97e995f	Add examples for EPIC	2021-07-01 16:41:54 -07:00
Valentin Dallerit	8ec8701d5d	Add support for the EPIC library (exponential integrator)	2021-07-01 16:40:23 -07:00
Milan Holec	87624c6d6c	Merge branch 'master' into fusion-ldrd-dev	2021-07-01 14:18:11 -07:00
Chris Vogl	1c988f4d45	Merge branch 'master' into fusion-ldrd-dev	2021-06-09 08:43:41 -07:00
Chris Vogl	c46b3e69db	Reverting `ffc0a9ee` that added MakePeriodic functions to Mesh class... will use new features in master	2021-06-09 08:43:31 -07:00
Chris Vogl	84d954b44b	added some functionality to SUNDIALS interface, as well as workaround code for IMEX	2021-06-08 19:23:26 -07:00
Chris Vogl	b61db555d5	Adding MakePeriodic functions to Mesh class	2021-06-08 13:18:46 -07:00