add margin to WeightRevDiff test

bug-fix in BuildConformingProlongation

.gitignore

@@ -259,6 +259,8 @@ miniapps/performance/sol.*
 
 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
+miniapps/shifted/extrapolate
+miniapps/shifted/ParaViewExtrapolate
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf

CHANGELOG

@@ -21,6 +21,9 @@ Version 4.3.1 (development)
 - More explicit and consistent formating of the output of iterative solvers
   with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
 
+- Added a miniapp for PDE-based extrapolation of finite element functions. See
+  miniapps/shifted/extrapolate.cpp.
+
 - Added support for automatic differentiation. Users can select between native
   implementation and external library implementation during configuration. One
   parallel and two serial examples are implemented in the miniapps/autodiff/
@@ -95,6 +98,9 @@ Version 4.3.1 (development)
 - The HPC versions of ex1 and ex1p (in miniapps/performance) now support
   runtime selection of either 2D or 3D meshes.
 
+- Added ParaView visualization of `QuadratureFunction` fields, through both
+  `QuadratureFunction::SaveVTU` and `ParaViewDataCollection::RegisterQField`.
+
 
 Version 4.3, released on July 29, 2021
 ======================================
@@ -135,6 +141,12 @@ Discretization improvements
 
 - Added support for nonscalar coefficient with VectorDiffusionIntegrator.
 
+- Added support for Partial Assembly with Discontinuous Galerkin methods on
+  nonconforming meshes.
+
+- Added a simpler interface to request face information: see
+  `Mesh::FaceInformation` and `Mesh::GetFaceInformation`.
+
 Linear and nonlinear solvers
 ----------------------------
 - Added support for AMG preconditioners on GPUs based on the hypre library

examples/ex5.cpp

@@ -197,7 +197,7 @@ int main(int argc, char *argv[])
       SparseMatrix &M(mVarf->SpMat());
       SparseMatrix &B(bVarf->SpMat());
       B *= -1.;
-      if (Device::IsEnabled()) { B.BuildTranspose(); }
+      B.EnsureMultTranspose();
       Bt = new TransposeOperator(&B);
 
       darcyOp.SetBlock(0,0, &M);

examples/ex9.cpp

@@ -27,6 +27,7 @@
 //    ex9 -pa -m ../data/periodic-cube.mesh -d cuda
 //    ex9 -ea -m ../data/periodic-cube.mesh -d cuda
 //    ex9 -fa -m ../data/periodic-cube.mesh -d cuda
+//    ex9 -pa -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9 -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and

examples/ex9p.cpp

@@ -28,6 +28,7 @@
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh -d cuda
 //    mpirun -np 4 ex9p -ea -m ../data/periodic-cube.mesh -d cuda
 //    mpirun -np 4 ex9p -fa -m ../data/periodic-cube.mesh -d cuda
+//    mpirun -np 4 ex9p -pa -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9 -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and

examples/petsc/ex2p.cpp

@@ -31,6 +31,10 @@
 //               also illustrated. The example also shows how to form a linear
 //               system using a PETSc matrix and solve with a PETSc solver.
 //
+//               The example also show how to use the non-overlapping feature of
+//               the ParBilinearForm class to obtain the linear operator in
+//               a format suitable for the BDDC preconditioner in PETSc.
+//
 //               We recommend viewing Example 1 before viewing this example.
 
 #include "mfem.hpp"
@@ -61,10 +65,15 @@ int main(int argc, char *argv[])
    bool use_petsc = true;
    const char *petscrc_file = "";
    bool use_nonoverlapping = false;
+   int ser_ref_levels = -1, par_ref_levels = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
    args.AddOption(&amg_elast, "-elast", "--amg-for-elasticity", "-sys",
@@ -131,8 +140,8 @@ int main(int argc, char *argv[])
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 1,000 elements.
    {
-      int ref_levels =
-         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels = ser_ref_levels >= 0 ? ser_ref_levels :
+                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
@@ -145,7 +154,6 @@ int main(int argc, char *argv[])
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
-      int par_ref_levels = 1;
       for (int l = 0; l < par_ref_levels; l++)
       {
          pmesh->UniformRefinement();
@@ -296,12 +304,20 @@ int main(int argc, char *argv[])
       PetscPreconditioner *prec = NULL;
       if (use_nonoverlapping)
       {
+         // Compute dofs belonging to the natural boundary
+         Array<int> nat_tdof_list, nat_bdr(pmesh->bdr_attributes.Max());
+         nat_bdr = 1;
+         nat_bdr[0] = 0;
+         fespace->GetEssentialTrueDofs(nat_bdr, nat_tdof_list);
+
          // Auxiliary class for BDDC customization
          PetscBDDCSolverParams opts;
          // Inform the solver about the finite element space
          opts.SetSpace(fespace);
          // Inform the solver about essential dofs
          opts.SetEssBdrDofs(&ess_tdof_list);
+         // Inform the solver about natural dofs
+         opts.SetNatBdrDofs(&nat_tdof_list);
          // Create a BDDC solver with parameters
          prec = new PetscBDDCSolver(A,opts);
          pcg->SetPreconditioner(*prec);

examples/petsc/makefile

@@ -77,6 +77,7 @@ EX1_ARGS_P            := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_e
 EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly --device cuda --petscopts rc_ex1p_cuda
 EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
+EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
 EX3_ARGS              := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
 EX4_ARGS              := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
 EX4_HYB_ARGS          := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
@@ -107,6 +108,7 @@ ifeq ($(MFEM_USE_CUDA),YES)
 endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_BDDC))
 ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p

examples/petsc/rc_ex2p_bddc

@@ -0,0 +1,25 @@
+# Sample options for BDDC
+
+-ksp_converged_reason
+-ksp_view
+-pc_type bddc
+
+# Turn on diagnostic for errors
+#-pc_bddc_check_level 1
+
+# This is an H1 problem, local problems may be singular
+# Turn on automatic corner selection
+-pc_bddc_corner_selection
+
+# Advanced customization
+
+# Deluxe scaling
+-pc_bddc_use_deluxe_scaling
+
+# Adaptive primal space (requires PETSc configured with MUMPS or PARDISO support)
+#-pc_bddc_adaptive_threshold 1.2 # tolerance for eigenvalue selection
+#-pc_bddc_adaptive_userdefined # preserve RBMs
+#-pc_bddc_monolithic # treat all displacements components at once -> smaller primal spaces, larger eigenvalue problems
+
+# Select solver for coarse problem
+# -pc_bddc_coarse_pc_type cholesky

fem/bilininteg_dgtrace_pa.cpp

@@ -182,22 +182,26 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       auto C = Reshape(vel.HostWrite(), dim, nq, nf);
       Vector Vq(dim);
       int f_ind = 0;
-      for (int f = 0; f < fes.GetNF(); ++f)
+      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
       {
-         int e1, e2;
-         int inf1, inf2;
-         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-         int face_id = inf1 / 64;
-         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-             (type==FaceType::Boundary && e2<0 && inf2<0) )
+         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
+         if (face.IsNonconformingCoarse())
          {
+            // We skip nonconforming coarse faces as they are treated
+            // by the corresponding nonconforming fine faces.
+            continue;
+         }
+         else if ( face.IsOfFaceType(type) )
+         {
+            const int mask = FaceElementTransformations::HAVE_ELEM1 |
+                             FaceElementTransformations::HAVE_LOC1;
             FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
+               *fes.GetMesh()->GetFaceElementTransformations(f, mask);
             for (int q = 0; q < nq; ++q)
             {
                // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face_id, quad1D, q);
+               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                      quad1D, q);
                T.SetAllIntPoints(&ir->IntPoint(q));
                const IntegrationPoint &eip1 = T.GetElement1IntPoint();
                u->Eval(Vq, *T.Elem1, eip1);
@@ -242,29 +246,31 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       auto n = Reshape(geom->normal.HostRead(), nq, dim, nf);
       auto C = Reshape(r.HostWrite(), nq, nf);
       int f_ind = 0;
-      for (int f = 0; f < fes.GetNF(); ++f)
+      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
       {
-         int e1, e2;
-         int inf1, inf2;
-         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-         int face_id = inf1 / 64;
-         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-             (type==FaceType::Boundary && e2<0 && inf2<0) )
+         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
+         if (face.IsNonconformingCoarse())
+         {
+            // We skip nonconforming coarse faces as they are treated
+            // by the corresponding nonconforming fine faces.
+            continue;
+         }
+         else if ( face.IsOfFaceType(type) )
          {
             FaceElementTransformations &T =
                *fes.GetMesh()->GetFaceElementTransformations(f);
             for (int q = 0; q < nq; ++q)
             {
                // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face_id, quad1D, q);
+               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                      quad1D, q);
 
                T.SetAllIntPoints(&ir->IntPoint(q));
                const IntegrationPoint &eip1 = T.GetElement1IntPoint();
                const IntegrationPoint &eip2 = T.GetElement2IntPoint();
                double r;
 
-               if (inf2 < 0)
+               if ( face.IsBoundary() )
                {
                   r = rho->Eval(*T.Elem1, eip1);
                }

fem/bilininteg_mass_pa.cpp

@@ -41,6 +41,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
       ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
       return;
    }
+   int map_type = el.GetMapType();
    dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
    nq = ir->GetNPoints();
@@ -93,6 +94,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
       const int NE = ne;
       const int Q1D = quad1D;
       const bool const_c = coeff.Size() == 1;
+      const bool by_val = map_type == FiniteElement::VALUE;
       const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
       const auto J = Reshape(geom->J.Read(), Q1D,Q1D,2,2,NE);
       const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
@@ -110,7 +112,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
                const double J22 = J(qx,qy,1,1,e);
                const double detJ = (J11*J22)-(J21*J12);
                const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-               v(qx,qy,e) =  W(qx,qy) * coeff * detJ;
+               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
             }
          }
       });
@@ -120,6 +122,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
       const int NE = ne;
       const int Q1D = quad1D;
       const bool const_c = coeff.Size() == 1;
+      const bool by_val = map_type == FiniteElement::VALUE;
       const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
       const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
       const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
@@ -146,7 +149,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
                   /* */               J21 * (J12 * J33 - J32 * J13) +
                   /* */               J31 * (J12 * J23 - J22 * J13);
                   const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * detJ;
+                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * (by_val ? detJ : 1.0/detJ);
                }
             }
          }

fem/ceed/algebraic.cpp

@@ -667,7 +667,7 @@ AlgebraicSpaceHierarchy::AlgebraicSpaceHierarchy(FiniteElementSpace &fes)
       const SparseMatrix *R = fespaces[ilevel+1]->GetRestrictionMatrix();
       if (R)
       {
-         R->BuildTranspose();
+         R->EnsureMultTranspose();
          R_tr[ilevel] = new TransposeOperator(*R);
       }
       else

fem/coefficient.cpp

@@ -14,6 +14,7 @@
 #include "fem.hpp"
 
 #include <cmath>
+#include <cstddef>
 #include <limits>
 
 namespace mfem
@@ -48,6 +49,15 @@ ElementTransformation *RefinedToCoarse(
    return coarse_T;
 }
 
+void Coefficient::EvalRevDiff(const double Q_bar,
+                              ElementTransformation &T,
+                              const IntegrationPoint &ip,
+                              DenseMatrix &PointMat_bar)
+{
+   MFEM_ABORT("Coefficient::EvalRevDiff\n"
+              "\tEvalRevDiff not implemented for this coefficient!\n");
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -119,6 +129,34 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
    }
 }
 
+void FunctionCoefficient::EvalRevDiff(const double Q_bar,
+                                      ElementTransformation &T,
+                                      const IntegrationPoint &ip,
+                                      DenseMatrix &PointMat_bar)
+{
+   int space_dim = T.GetSpaceDim();
+   double x[3] = {};
+   Vector transip(x, space_dim);
+   T.Transform(ip, transip);
+
+   double x_bar[3] = {};
+   Vector transip_bar(x_bar, space_dim);
+   if (Function)
+   {
+      MFEM_ASSERT(FunctionRevDiff, "EvalRevDiff: reverse-mode differentiated "
+                  "version of Function must be provided");
+      FunctionRevDiff(transip, Q_bar, transip_bar);
+   }
+   else
+   {
+      MFEM_ASSERT(TDFunctionRevDiff, "EvalRevDiff: reverse-mode differentiated"
+                  " version of TDFunction must be provided");
+      TDFunctionRevDiff(transip, GetTime(), Q_bar, transip_bar);
+   }
+   static_cast<IsoparametricTransformation &>(T).TransformRevDiff(
+      ip, transip_bar, PointMat_bar);
+}
+
 double GridFunctionCoefficient::Eval (ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
@@ -189,6 +227,15 @@ void RestrictedCoefficient::SetTime(double t)
    this->Coefficient::SetTime(t);
 }
 
+void VectorCoefficient::EvalRevDiff(const Vector &V_bar,
+                                    ElementTransformation &T,
+                                    const IntegrationPoint &ip,
+                                    DenseMatrix &PointMat_bar)
+{
+   MFEM_ABORT("VectorCoefficient::EvalRevDiff\n"
+              "\tEvalRevDiff not implemented for this coefficient!\n");
+}
+
 void VectorCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
                              const IntegrationRule &ir)
 {
@@ -283,6 +330,35 @@ void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
    }
 }
 
+void VectorFunctionCoefficient::EvalRevDiff(const Vector &V_bar,
+                                            ElementTransformation &T,
+                                            const IntegrationPoint &ip,
+                                            DenseMatrix &PointMat_bar)
+{
+   MFEM_ASSERT( Q == NULL, "EvalRevDiff: not implemented for use with Q.")
+
+   double x[3];
+   Vector transip(x, vdim);
+   double x_bar[3];
+   Vector transip_bar(x_bar, vdim);
+   T.Transform(ip, transip);
+   transip_bar = 0.0;
+   if (Function)
+   {
+      MFEM_ASSERT(FunctionRevDiff, "EvalRevDiff: reverse-mode differentiated "
+                  "version of Function must be provided");
+      FunctionRevDiff(transip, V_bar, transip_bar);
+   }
+   else
+   {
+      MFEM_ASSERT(TDFunctionRevDiff, "EvalRevDiff: reverse-mode differentiated"
+                  " version of TDFunction must be provided");
+      TDFunctionRevDiff(transip, GetTime(), V_bar, transip_bar);
+   }
+   static_cast<IsoparametricTransformation &>(T).TransformRevDiff(
+      ip, transip_bar, PointMat_bar);
+}
+
 VectorArrayCoefficient::VectorArrayCoefficient (int dim)
    : VectorCoefficient(dim), Coeff(dim), ownCoeff(dim)
 {
@@ -797,6 +873,25 @@ void ProductCoefficient::SetTime(double t)
    this->Coefficient::SetTime(t);
 }
 
+void ProductCoefficient::EvalRevDiff(const double Q_bar,
+                                     ElementTransformation &T,
+                                     const IntegrationPoint &ip,
+                                     DenseMatrix &PointMat_bar)
+{
+   if (a == nullptr)
+   {
+      b->EvalRevDiff(Q_bar * aConst, T, ip, PointMat_bar);
+   }
+   else
+   {
+      double a_val = a->Eval(T, ip);
+      double b_val = b->Eval(T, ip);
+
+      a->EvalRevDiff(Q_bar * b_val, T, ip, PointMat_bar);
+      b->EvalRevDiff(Q_bar * a_val, T, ip, PointMat_bar);
+   }
+}
+
 void RatioCoefficient::SetTime(double t)
 {
    if (a) { a->SetTime(t); }
@@ -967,6 +1062,35 @@ void ScalarVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
    V *= sa;
 }
 
+void ScalarVectorProductCoefficient::EvalRevDiff(
+   const Vector &V_bar,
+   ElementTransformation &T,
+   const IntegrationPoint &ip,
+   DenseMatrix &PointMat_bar)
+{
+#ifdef MFEM_THREAD_SAFE
+   Vector W(V_bar.Size());
+   Vector W_bar(V_bar.Size());
+#else
+   W.SetSize(V_bar.Size());
+   W_bar.SetSize(V_bar.Size());
+#endif
+
+   double sa = (a == nullptr) ? aConst : a->Eval(T, ip);
+   b->Eval(W, T, ip);
+   W *= sa;
+
+   /// reverse pass
+   W_bar = 0.0;
+   add(W_bar, sa, V_bar, W_bar);
+   b->EvalRevDiff(W_bar, T, ip, PointMat_bar);
+   if (a != nullptr)
+   {
+      const double sa_bar = V_bar * W;
+      a->EvalRevDiff(sa_bar, T, ip, PointMat_bar);
+   }
+}
+
 NormalizedVectorCoefficient::NormalizedVectorCoefficient(VectorCoefficient &A,
                                                          double tol_)
    : VectorCoefficient(A.GetVDim()), a(&A), tol(tol_)

fem/coefficient.hpp

@@ -58,6 +58,21 @@ public:
    virtual double Eval(ElementTransformation &T,
                        const IntegrationPoint &ip) = 0;
 
+   /** @brief Reverse-mode differentiation of Eval w.r.t. the mesh node
+       locations in the element described by @a T, accumulating the result in
+       @a PointMat_bar */
+   /** @param[in] Q_bar - derivative of some output w.r.t. result of Eval */
+   /** @param[in] T - an element transformation */
+   /** @param[in] ip - defines location in reference space */
+   /** @param[inout] PointMat_bar - derivative of output w.r.t. mesh nodes */
+   /** @note When this method is called, the caller must make sure that the
+       IntegrationPoint associated with @a T is the same as @a ip. This can be
+       achieved by calling T.SetIntPoint(&ip). */
+   virtual void EvalRevDiff(const double Q_bar,
+                            ElementTransformation &T,
+                            const IntegrationPoint &ip,
+                            DenseMatrix &PointMat_bar);
+
    /** @brief Evaluate the coefficient in the element described by @a T at the
        point @a ip at time @a t. */
    /** @note When this method is called, the caller must make sure that the
@@ -211,7 +226,15 @@ class FunctionCoefficient : public Coefficient
 {
 protected:
    std::function<double(const Vector &)> Function;
+   std::function<void(const Vector &,
+                      const double,
+                      Vector &)> FunctionRevDiff;
+
    std::function<double(const Vector &, double)> TDFunction;
+   std::function<void(const Vector &,
+                      double,
+                      const double,
+                      Vector &)> TDFunctionRevDiff;
 
 public:
    /// Define a time-independent coefficient from a std function
@@ -226,6 +249,23 @@ public:
       : TDFunction(std::move(TDF))
    { }
 
+   /// Construct time-independent coefficient that can be differentiated
+   FunctionCoefficient(std::function<double(const Vector &)> F,
+                       std::function<void(const Vector &,
+                                          const double,
+                                          Vector &)> dF)
+      : Function(F), FunctionRevDiff(dF)
+   { }
+
+   /// Construct time-dependent coefficient that can be differentiated
+   FunctionCoefficient(std::function<double(const Vector &, double)> TDF,
+                       std::function<void(const Vector &,
+                                          double,
+                                          const double,
+                                          Vector &)> dTDF)
+      : TDFunction(TDF), TDFunctionRevDiff(dTDF)
+   { }
+
    /// (DEPRECATED) Define a time-independent coefficient from a C-function
    /** @deprecated Use the method where the C-function, @a f, uses a const
        Vector argument instead of Vector. */
@@ -247,6 +287,11 @@ public:
    /// Evaluate the coefficient at @a ip.
    virtual double Eval(ElementTransformation &T,
                        const IntegrationPoint &ip);
+
+   virtual void EvalRevDiff(const double Q_bar,
+                            ElementTransformation &T,
+                            const IntegrationPoint &ip,
+                            DenseMatrix &PointMat_bar);
 };
 
 class GridFunction;
@@ -456,6 +501,20 @@ public:
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip) = 0;
 
+   /** @brief Reverse-mode differentiation of Eval w.r.t. the mesh node
+       locations in the element described by @a T, accumulating the result in
+       @a PointMat_bar */
+   /** @param[in] V_bar - derivative of some output with respect to `V` */
+   /** @param[in] T - an element transformation */
+   /** @param[in] ip - defines location in reference space */
+   /** @param[inout] PointMat_bar - derivative of output w.r.t. mesh nodes */
+   /** @note When this method is called, the caller must make sure that the
+       IntegrationPoint associated with @a T is the same as @a ip. This can be
+       achieved by calling T.SetIntPoint(&ip). */
+   virtual void EvalRevDiff(const Vector &V_bar, ElementTransformation &T,
+                            const IntegrationPoint &ip,
+                            DenseMatrix &PointMat_bar);
+
    /** @brief Evaluate the vector coefficient in the element described by @a T
        at all points of @a ir, storing the result in @a M. */
    /** The dimensions of @a M are GetVDim() by ir.GetNPoints() and they must be
@@ -490,6 +549,10 @@ public:
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip) { V = vec; }
 
+   virtual void EvalRevDiff(const Vector &V_bar, ElementTransformation &T,
+                            const IntegrationPoint &ip,
+                            DenseMatrix &PointMat_bar) { }
+
    /// Return a reference to the constant vector in this class.
    const Vector& GetVec() { return vec; }
 };
@@ -582,8 +645,17 @@ class VectorFunctionCoefficient : public VectorCoefficient
 {
 private:
    std::function<void(const Vector &, Vector &)> Function;
+   std::function<void(const Vector &,
+                      const Vector &,
+                      Vector &)> FunctionRevDiff;
+
    std::function<void(const Vector &, double, Vector &)> TDFunction;
+   std::function<void(const Vector &,
+                      double,
+                      const Vector &,
+                      Vector &)> TDFunctionRevDiff;
    Coefficient *Q;
+   // Coefficient *dQ;
 
 public:
    /// Define a time-independent vector coefficient from a std function
@@ -606,11 +678,37 @@ public:
       : VectorCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
    { }
 
+   /// Construct time-independent vector coefficient that can be differentiated
+   VectorFunctionCoefficient(int dim,
+                             std::function<void(const Vector &,
+                                                Vector &)> F,
+                             std::function<void(const Vector &,
+                                                const Vector &,
+                                                Vector &)> dF)
+      : VectorCoefficient(dim), Function(std::move(F)),
+        FunctionRevDiff(std::move(dF)), Q(NULL)
+   { }
+
+   /// Construct time-dependent vector coefficient that can be differentiated
+   VectorFunctionCoefficient(int dim,
+                             std::function<void(const Vector &,
+                                                double,
+                                                Vector &)> TDF,
+                             std::function<void(const Vector &,
+                                                double, const Vector &, Vector &)> dTDF)
+      : VectorCoefficient(dim), TDFunction(std::move(TDF)),
+        TDFunctionRevDiff(std::move(dTDF)), Q(NULL)
+   { }
+
    using VectorCoefficient::Eval;
    /// Evaluate the vector coefficient at @a ip.
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
 
+   virtual void EvalRevDiff(const Vector &V_bar, ElementTransformation &T,
+                            const IntegrationPoint &ip,
+                            DenseMatrix &PointMat_bar);
+
    virtual ~VectorFunctionCoefficient() { }
 };
 
@@ -1388,6 +1486,11 @@ public:
    virtual double Eval(ElementTransformation &T,
                        const IntegrationPoint &ip)
    { return ((a == NULL ) ? aConst : a->Eval(T, ip) ) * b->Eval(T, ip); }
+
+   void EvalRevDiff(const double Q_bar,
+                    ElementTransformation &T,
+                    const IntegrationPoint &ip,
+                    DenseMatrix &PointMat_bar);
 };
 
 /** @brief Scalar coefficient defined as the ratio of two scalars where one or
@@ -1657,6 +1760,9 @@ private:
    double aConst;
    Coefficient * a;
    VectorCoefficient * b;
+#ifndef MFEM_THREAD_SAFE
+   Vector W, W_bar;
+#endif
 
 public:
    /// Constructor with constant and vector coefficient.  Result is A * B.
@@ -1687,6 +1793,11 @@ public:
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
    using VectorCoefficient::Eval;
+
+   virtual void EvalRevDiff(const Vector &V_bar, ElementTransformation &T,
+                            const IntegrationPoint &ip,
+                            DenseMatrix &PointMat_bar);
+
 };
 
 /// Vector coefficient defined as a normalized vector field (returns v/|v|)

fem/datacollection.cpp

@@ -11,6 +11,7 @@
 
 #include "fem.hpp"
 #include "../mesh/nurbs.hpp"
+#include "../mesh/vtk.hpp"
 #include "../general/binaryio.hpp"
 #include "../general/text.hpp"
 #include "picojson.h"
@@ -787,53 +788,44 @@ void ParaViewDataCollection::Load(int )
 
 std::string ParaViewDataCollection::GenerateCollectionPath()
 {
-   std::string out = "";
-   out = prefix_path + DataCollection::GetCollectionName();
-   return out;
+   return prefix_path + DataCollection::GetCollectionName();
 }
 
 std::string ParaViewDataCollection::GeneratePVTUPath()
 {
-   std::string out = "Cycle" + to_padded_string(cycle,pad_digits_cycle);
-   return out;
+   return "Cycle" + to_padded_string(cycle,pad_digits_cycle);
 }
 
 std::string ParaViewDataCollection::GenerateVTUPath()
 {
-   std::string out = GeneratePVTUPath();
-   return out;
+   return GeneratePVTUPath();
 }
 
 std::string ParaViewDataCollection::GeneratePVDFileName()
 {
-   std::string out = GetCollectionName()+".pvd";
-   return out;
+   return GetCollectionName() + ".pvd";
 }
 
-std::string ParaViewDataCollection::GeneratePVTUFileName()
+std::string ParaViewDataCollection::GeneratePVTUFileName(
+   const std::string &prefix)
 {
-   std::string out = "data.pvtu";
-   return out;
+   return prefix + ".pvtu";
 }
 
-std::string ParaViewDataCollection::GenerateVTUFileName()
+std::string ParaViewDataCollection::GenerateVTUFileName(
+   const std::string &prefix, int rank)
 {
-   std::string out = "proc" + to_padded_string(myid,pad_digits_rank)+".vtu";
-   return out;
-}
-std::string ParaViewDataCollection::GenerateVTUFileName(int crank)
-{
-   std::string out = "proc" + to_padded_string(crank,pad_digits_rank)+".vtu";
-   return out;
+   return prefix + to_padded_string(rank, pad_digits_rank) + ".vtu";
 }
 
 void ParaViewDataCollection::Save()
 {
    // add a new collection to the PDV file
 
+   std::string col_path = GenerateCollectionPath();
    // check if the directories are created
    {
-      std::string path = GenerateCollectionPath()+"/"+GenerateVTUPath();
+      std::string path = col_path + "/" + GenerateVTUPath();
       int err = create_directory(path, mesh, myid);
       if (err)
       {
@@ -850,8 +842,7 @@ void ParaViewDataCollection::Save()
 
    if (myid == 0 && !pvd_stream.is_open())
    {
-      std::string dpath=GenerateCollectionPath();
-      std::string pvdname=dpath+"/"+GeneratePVDFileName();
+      std::string pvdname = col_path + "/" + GeneratePVDFileName();
 
       bool write_header = true;
       std::ifstream pvd_in;
@@ -915,80 +906,87 @@ void ParaViewDataCollection::Save()
       }
    }
 
-   // define the vtu file
+   std::string vtu_prefix = col_path + "/" + GenerateVTUPath() + "/";
+
+   // Save the local part of the mesh and grid functions fields to the local
+   // VTU file
    {
-      std::string fname = GenerateCollectionPath()+"/"+GenerateVTUPath()+"/"
-                          +GenerateVTUFileName();
-      std::fstream out(fname, std::ios::out);
+      std::ofstream out(vtu_prefix + GenerateVTUFileName("proc", myid));
       out.precision(precision);
-      SaveDataVTU(out,levels_of_detail);
-      out.close();
+      SaveDataVTU(out, levels_of_detail);
    }
 
-   // define the pvtu file only on process 0
-   if (myid==0)
+   // Save the local part of the quadrature function fields
+   for (const auto &qfield : q_field_map)
    {
-      std::string fname = GenerateCollectionPath()+"/"+GeneratePVTUPath()+"/"
-                          +GeneratePVTUFileName();
-      std::fstream out(fname, std::ios::out);
+      const std::string &field_name = qfield.first;
+      std::ofstream out(vtu_prefix + GenerateVTUFileName(field_name, myid));
+      qfield.second->SaveVTU(out, pv_data_format, compression);
+   }
 
-      out << "<?xml version=\"1.0\"?>\n";
-      out << "<VTKFile type=\"PUnstructuredGrid\"";
-      out << " version =\"0.1\" byte_order=\"" << VTKByteOrder() << "\">\n";
-      out << "<PUnstructuredGrid GhostLevel=\"0\">\n";
-
-      out << "<PPoints>\n";
-      out << "\t<PDataArray type=\"" << GetDataTypeString() << "\" ";
-      out << " Name=\"Points\" NumberOfComponents=\"3\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PPoints>\n";
-
-      out << "<PCells>\n";
-      out << "\t<PDataArray type=\"Int32\" ";
-      out << " Name=\"connectivity\" NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "\t<PDataArray type=\"Int32\" ";
-      out << " Name=\"offsets\"      NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "\t<PDataArray type=\"UInt8\" ";
-      out << " Name=\"types\"        NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PCells>\n";
-
-      out << "<PPointData>\n";
-      for (FieldMapIterator it=field_map.begin(); it!=field_map.end(); ++it)
+   // MPI rank 0 also creates a "PVTU" file that points to all of the separately
+   // written VTU files.
+   // This file path is then appended to the PVD file.
+   if (myid == 0)
+   {
+      // Create the main PVTU file
       {
-         int vec_dim=it->second->VectorDim();
-         out << "<PDataArray type=\"" << GetDataTypeString()
-             << "\" Name=\"" << it->first
-             << "\" NumberOfComponents=\"" << vec_dim << "\" "
-             << "format=\"" << GetDataFormatString() << "\" />\n";
+         std::ofstream pvtu_out(vtu_prefix + GeneratePVTUFileName("data"));
+         WritePVTUHeader(pvtu_out);
+
+         // Grid function fields
+         pvtu_out << "<PPointData>\n";
+         for (auto &field_it : field_map)
+         {
+            int vec_dim = field_it.second->VectorDim();
+            pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
+                     << "\" Name=\"" << field_it.first
+                     << "\" NumberOfComponents=\"" << vec_dim << "\" "
+                     << "format=\"" << GetDataFormatString() << "\" />\n";
+         }
+         pvtu_out << "</PPointData>\n";
+         // Element attributes
+         pvtu_out << "<PCellData>\n";
+         pvtu_out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
+                  << "\" NumberOfComponents=\"1\""
+                  << " format=\"" << GetDataFormatString() << "\"/>\n";
+         pvtu_out << "</PCellData>\n";
+
+         WritePVTUFooter(pvtu_out, "proc");
       }
-      out << "</PPointData>\n";
 
-      // CELL DATA
-      out << "<PCellData>\n";
-      out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
-          << "\" NumberOfComponents=\"1\""
-          << " format=\"" << GetDataFormatString() << "\"/>\n";
-      out << "</PCellData>\n";
+      // Add the latest PVTU to the PVD
+      pvd_stream << "<DataSet timestep=\"" << GetTime()
+                 << "\" group=\"\" part=\"" << 0 << "\" file=\""
+                 << GeneratePVTUPath() + "/" + GeneratePVTUFileName("data")
+                 << "\" name=\"mesh\"/>\n";
 
-      for (int ii=0; ii<num_procs; ii++)
+      // Create PVTU files for each quadrature field and add them to the PVD
+      // file
+      for (auto &q_field : q_field_map)
       {
-         // this one is generated without the path
-         std::string nfname=GenerateVTUFileName(ii);
-         out << "<Piece Source=\"" << nfname << "\"/>\n";
-      }
-      out << "</PUnstructuredGrid>\n";
-      out << "</VTKFile>\n";
-      out.close();
+         const std::string &q_field_name = q_field.first;
+         std::string q_fname = GeneratePVTUPath() + "/"
+                               + GeneratePVTUFileName(q_field_name);
 
-      fname = GeneratePVTUPath()+"/"+GeneratePVTUFileName();
-      // add the pvtu file to the pvd_stream
-      pvd_stream << "<DataSet timestep=\"" << GetTime();  // GetCycle();
-      pvd_stream << "\" group=\"\" part=\"" << 0 << "\" file=\"";
-      pvd_stream << fname << "\"/>\n";
+         std::ofstream pvtu_out(col_path + "/" + q_fname);
+         WritePVTUHeader(pvtu_out);
+         int vec_dim = q_field.second->GetVDim();
+         pvtu_out << "<PPointData>\n";
+         pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
+                  << "\" Name=\"" << q_field_name
+                  << "\" NumberOfComponents=\"" << vec_dim << "\" "
+                  << "format=\"" << GetDataFormatString() << "\" />\n";
+         pvtu_out << "</PPointData>\n";
+         WritePVTUFooter(pvtu_out, q_field_name);
+
+         pvd_stream << "<DataSet timestep=\"" << GetTime()
+                    << "\" group=\"\" part=\"" << 0 << "\" file=\""
+                    << q_fname << "\" name=\"" << q_field_name << "\"/>\n";
+      }
       pvd_stream.flush();
+      // Move the insertion point before the closing collection tag, so that
+      // the PVD file is valid even when writing incrementally.
       std::fstream::pos_type pos = pvd_stream.tellp();
       pvd_stream << "</Collection>\n";
       pvd_stream << "</VTKFile>" << std::endl;
@@ -996,6 +994,44 @@ void ParaViewDataCollection::Save()
    }
 }
 
+void ParaViewDataCollection::WritePVTUHeader(std::ostream &out)
+{
+   out << "<?xml version=\"1.0\"?>\n";
+   out << "<VTKFile type=\"PUnstructuredGrid\"";
+   out << " version =\"0.1\" byte_order=\"" << VTKByteOrder() << "\">\n";
+   out << "<PUnstructuredGrid GhostLevel=\"0\">\n";
+
+   out << "<PPoints>\n";
+   out << "\t<PDataArray type=\"" << GetDataTypeString() << "\" ";
+   out << " Name=\"Points\" NumberOfComponents=\"3\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "</PPoints>\n";
+
+   out << "<PCells>\n";
+   out << "\t<PDataArray type=\"Int32\" ";
+   out << " Name=\"connectivity\" NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "\t<PDataArray type=\"Int32\" ";
+   out << " Name=\"offsets\"      NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "\t<PDataArray type=\"UInt8\" ";
+   out << " Name=\"types\"        NumberOfComponents=\"1\""
+       << " format=\"" << GetDataFormatString() << "\"/>\n";
+   out << "</PCells>\n";
+}
+
+void ParaViewDataCollection::WritePVTUFooter(std::ostream &out,
+                                             const std::string &vtu_prefix)
+{
+   for (int ii=0; ii<num_procs; ii++)
+   {
+      std::string vtu_filename = GenerateVTUFileName(vtu_prefix, ii);
+      out << "<Piece Source=\"" << vtu_filename << "\"/>\n";
+   }
+   out << "</PUnstructuredGrid>\n";
+   out << "</VTKFile>\n";
+}
+
 void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
 {
    out << "<VTKFile type=\"UnstructuredGrid\"";
@@ -1015,16 +1051,6 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
    {
       SaveGFieldVTU(out,ref,it);
    }
-   // iterate over all quadrature functions
-   // if the Quadrature functions are dumped as cell data
-   // the cycle should be moved before the grid functions
-   // and the PrintVTU CellData section should be open in the mesh dump
-   for (QFieldMapIterator it=q_field_map.begin(); it!=q_field_map.end(); ++it)
-   {
-      // save the quadrature functions
-      // this one is not implemented yet
-      SaveQFieldVTU(out,ref,it);
-   }
    out << "</PointData>\n";
    // close the mesh
    out << "</Piece>\n"; // close the piece open in the PrintVTU method
@@ -1032,27 +1058,21 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
    out << "</VTKFile>" << std::endl;
 }
 
-void ParaViewDataCollection::SaveQFieldVTU(std::ostream &out, int ref,
-                                           const QFieldMapIterator& it )
-{
-   MFEM_WARNING("SaveQFieldVTU is not currently implemented - field name:"<<it->second);
-}
-
 void ParaViewDataCollection::SaveGFieldVTU(std::ostream &out, int ref_,
-                                           const FieldMapIterator& it)
+                                           const FieldMapIterator &it)
 {
    RefinedGeometry *RefG;
    Vector val;
    DenseMatrix vval, pmat;
    std::vector<char> buf;
    int vec_dim = it->second->VectorDim();
+   out << "<DataArray type=\"" << GetDataTypeString()
+       << "\" Name=\"" << it->first;
+   out << "\" NumberOfComponents=\"" << vec_dim << "\""
+       << " format=\"" << GetDataFormatString() << "\" >" << '\n';
    if (vec_dim == 1)
    {
       // scalar data
-      out << "<DataArray type=\"" << GetDataTypeString()
-          << "\" Name=\"" << it->first;
-      out << "\" NumberOfComponents=\"1\" format=\""
-          << GetDataFormatString() << "\" >\n";
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          RefG = GlobGeometryRefiner.Refine(
@@ -1060,51 +1080,23 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &out, int ref_,
          it->second->GetValues(i, RefG->RefPts, val, pmat);
          for (int j = 0; j < val.Size(); j++)
          {
-            if (pv_data_format == VTKFormat::ASCII)
-            {
-               out << ZeroSubnormal(val(j)) << '\n';
-            }
-            else if (pv_data_format == VTKFormat::BINARY)
-            {
-               bin_io::AppendBytes(buf, val(j));
-            }
-            else
-            {
-               bin_io::AppendBytes<float>(buf, float(val(j)));
-            }
+            WriteBinaryOrASCII(out, buf, val(j), "\n", pv_data_format);
          }
       }
    }
    else
    {
       // vector data
-      out << "<DataArray type=\"" << GetDataTypeString()
-          << "\" Name=\"" << it->first;
-      out << "\" NumberOfComponents=\"" << vec_dim << "\""
-          << " format=\"" << GetDataFormatString() << "\" >" << '\n';
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          RefG = GlobGeometryRefiner.Refine(
                    mesh->GetElementBaseGeometry(i), ref_, 1);
-
          it->second->GetVectorValues(i, RefG->RefPts, vval, pmat);
-
          for (int jj = 0; jj < vval.Width(); jj++)
          {
             for (int ii = 0; ii < vval.Height(); ii++)
             {
-               if (pv_data_format == VTKFormat::ASCII)
-               {
-                  out << ZeroSubnormal(vval(ii,jj)) << ' ';
-               }
-               else if (pv_data_format == VTKFormat::BINARY)
-               {
-                  bin_io::AppendBytes(buf, vval(ii,jj));
-               }
-               else
-               {
-                  bin_io::AppendBytes<float>(buf, float(vval(ii,jj)));
-               }
+               WriteBinaryOrASCII(out, buf, vval(ii,jj), " ", pv_data_format);
             }
             if (pv_data_format == VTKFormat::ASCII) { out << '\n'; }
          }

fem/datacollection.hpp

@@ -491,19 +491,20 @@ private:
    bool restart_mode;
 
 protected:
+   void WritePVTUHeader(std::ostream &out);
+   void WritePVTUFooter(std::ostream &out, const std::string &vtu_prefix);
    void SaveDataVTU(std::ostream &out, int ref);
    void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it);
-   void SaveQFieldVTU(std::ostream &out, int ref, const QFieldMapIterator& it);
    const char *GetDataFormatString() const;
    const char *GetDataTypeString() const;
 
-   std::string  GenerateCollectionPath();
-   std::string  GenerateVTUFileName();
-   std::string  GenerateVTUFileName(int rank);
-   std::string  GenerateVTUPath();
-   std::string  GeneratePVDFileName();
-   std::string  GeneratePVTUFileName();
-   std::string  GeneratePVTUPath();
+   std::string GenerateCollectionPath();
+   std::string GenerateVTUFileName(const std::string &prefix, int rank);
+   std::string GenerateVTUPath();
+   std::string GeneratePVDFileName();
+   std::string GeneratePVTUFileName(const std::string &prefix);
+   std::string GeneratePVTUPath();
+
 
 public:
    /// Constructor. The collection name is used when saving the data.

fem/eltrans.cpp

@@ -532,6 +532,82 @@ void IsoparametricTransformation::Transform (const DenseMatrix &matrix,
    }
 }
 
+void IsoparametricTransformation::TransformRevDiff(const IntegrationPoint &ip,
+                                                   const Vector &x_bar,
+                                                   DenseMatrix &PointMat_bar)
+{
+   MFEM_ASSERT((PointMat_bar.Width() == PointMat.Width()) &&
+               (PointMat_bar.Height() == PointMat.Height()),
+               "PointMat_bar shape != PointMat shape");
+   shape.SetSize(FElem->GetDof());
+   FElem->CalcShape(ip, shape);
+   AddMultVWt(x_bar, shape, PointMat_bar);
+}
+
+void IsoparametricTransformation::JacobianRevDiff(const DenseMatrix &dFdx_bar,
+                                                  DenseMatrix &PointMat_bar)
+{
+   MFEM_ASSERT((PointMat_bar.Width() == PointMat.Width()) &&
+               (PointMat_bar.Height() == PointMat.Height()),
+               "PointMat_bar shape != PointMat shape");
+
+   dshape.SetSize(FElem->GetDof(), FElem->GetDim());
+   if (dshape.Width() > 0)
+   {
+      // The math here can be found in Giles' report "An extended collection of
+      // matrix derivative results for forward and reverse mode algorithmic
+      // differentiation"
+      FElem->CalcDShape(*IntPoint, dshape);
+      AddMultABt(dFdx_bar, dshape, PointMat_bar);
+   }
+}
+
+void IsoparametricTransformation::AdjugateJacobianRevDiff(
+   const DenseMatrix &adjJ_bar, DenseMatrix &PointMat_bar)
+{
+   Jacobian(); // Recompute the Jacobian, if necessary
+   double dFdx_bar_buffer[9];
+   DenseMatrix dFdx_bar(dFdx_bar_buffer, dFdx.Height(), dFdx.Width());
+   if (dFdx.Width() > 0)
+   {
+      CalcAdjugateRevDiff(dFdx, adjJ_bar, dFdx_bar);
+   }
+   JacobianRevDiff(dFdx_bar, PointMat_bar);
+}
+
+void IsoparametricTransformation::InverseJacobianRevDiff(
+   const DenseMatrix &invJ_bar, DenseMatrix &PointMat_bar)
+{
+   Jacobian(); // Recompute the Jacobian, if necessary
+   double dFdx_bar_buffer[9];
+   DenseMatrix dFdx_bar(dFdx_bar_buffer, dFdx.Height(), dFdx.Width());
+   if (dFdx.Width() > 0)
+   {
+      CalcInverseRevDiff(dFdx, invJ_bar, dFdx_bar);
+   }
+   JacobianRevDiff(dFdx_bar, PointMat_bar);
+}
+
+void IsoparametricTransformation::WeightRevDiff(DenseMatrix &PointMat_bar)
+{
+   Jacobian(); // Recompute the Jacobian, if necessary
+   double dFdx_bar_buffer[9];
+   DenseMatrix dFdx_bar(dFdx_bar_buffer, dFdx.Height(), dFdx.Width());
+   dFdx.WeightRevDiff(dFdx_bar);
+   JacobianRevDiff(dFdx_bar, PointMat_bar);
+}
+
+void IsoparametricTransformation::WeightRevDiff(double weight_bar,
+                                                DenseMatrix &PointMat_bar)
+{
+   Jacobian(); // Recompute the Jacobian, if necessary
+   double dFdx_bar_buffer[9];
+   DenseMatrix dFdx_bar(dFdx_bar_buffer, dFdx.Height(), dFdx.Width());
+   dFdx.WeightRevDiff(dFdx_bar);
+   dFdx_bar *= weight_bar;
+   JacobianRevDiff(dFdx_bar, PointMat_bar);
+}
+
 void IntegrationPointTransformation::Transform (const IntegrationPoint &ip1,
                                                 IntegrationPoint &ip2)
 {

fem/eltrans.hpp

@@ -446,6 +446,58 @@ public:
       return inv_tr.Transform(v, ip);
    }
 
+   /// @brief Reverse-mode differentiation of Transform() w.r.t. PointMat
+   /// @param[in] ip - specifies the location in reference space
+   /// @param[in] x_bar - derivative of some output w.r.t. x
+   /// @param[out] PointMat_bar - derivative of output w.r.t. PointMat
+   /// @note PointMat_bar must have the same shape as PointMat
+   /// @warning This routine does not initialize PointMat_bar, and instead
+   /// accumulates (with += or -=) contributions to its derivative.
+   void TransformRevDiff(const IntegrationPoint &ip, const Vector &x_bar,
+                         DenseMatrix &PointMat_bar);
+
+   /// @brief Reverse-mode differentiation of Jacobian() w.r.t. PointMat
+   /// @param[in] dFdx_bar - derivative of functional w.r.t. Jacobian
+   /// @param[out] PointMat_bar - derivative w.r.t. PointMat
+   /// @note PointMat_bar must have the same shape as PointMat
+   /// @warning This routine does not initialize PointMat_bar, and instead
+   /// accumulates (with += or -=) contributions to its derivative.
+   void JacobianRevDiff(const DenseMatrix &dFdx_bar,
+                        DenseMatrix &PointMat_bar);
+
+   /// @brief Reverse-mode differentiation of AdjugateJacobian() w.r.t. PointMat
+   /// @param[in] adjJ_bar - derivative of functional w.r.t. Adjugate
+   /// @param[out] PointMat_bar - derivative w.r.t. PointMat
+   /// @note PointMat_bar must have the same shape as PointMat
+   /// @warning This routine does not initialize PointMat_bar, and instead
+   /// accumulates (with += or -=) contributions to its derivative.
+   void AdjugateJacobianRevDiff(const DenseMatrix &adjJ_bar,
+                                DenseMatrix &PointMat_bar);
+
+   /// @brief Reverse-mode differentiation of InverseJacobian() w.r.t PointMat
+   /// @param[in] invJ_bar - derivative of functional w.r.t. Inverse
+   /// @param[out] PointMat_bar - derivative w.r.t. PointMat
+   /// @note PointMat_bar must have the same shape as PointMat
+   /// @warning This routine does not initialize PointMat_bar, and instead
+   /// accumulates (with += or -=) contributions to its derivative.
+   void InverseJacobianRevDiff(const DenseMatrix &adjJ_bar,
+                               DenseMatrix &PointMat_bar);
+
+   /// @brief Reverse-mode differentiation of Weight()
+   /// @param[out] PointMat_bar - derivative of functional w.r.t. PointMat
+   /// @note PointMat_bar must have the same shape as PointMat
+   /// @warning This routine does not initialize PointMat_bar, and instead
+   /// accumulates (with += or -=) contributions to its derivative.
+   void WeightRevDiff(DenseMatrix &PointMat_bar);
+
+   /// @brief Reverse-mode differentiation of Weight()
+   /// @param[in] weight_bar - derivative of functional w.r.t Weight
+   /// @param[out] PointMat_bar - derivative of functional w.r.t. PointMat
+   /// @note PointMat_bar must have the same shape as PointMat
+   /// @warning This routine does not initialize PointMat_bar, and instead
+   /// accumulates (with += or -=) contributions to its derivative.
+   void WeightRevDiff(double weight_bar, DenseMatrix &PointMat_bar);
+
    virtual ~IsoparametricTransformation() { }
 
    MFEM_DEPRECATED void FinalizeTransformation() {}

fem/fe/fe_base.cpp

@@ -30,6 +30,7 @@ FiniteElement::FiniteElement(int D, Geometry::Type G, int Do, int O, int F)
    deriv_map_type = VALUE;
    for (int i = 0; i < Geometry::MaxDim; i++) { orders[i] = -1; }
 #ifndef MFEM_THREAD_SAFE
+   shape.SetSize(dof);
    vshape.SetSize(dof, dim);
 #endif
 }
@@ -46,6 +47,13 @@ void FiniteElement::CalcVShape (
    MFEM_ABORT("method is not implemented for this class");
 }
 
+void FiniteElement::CalcVShapeRevDiff(ElementTransformation &Trans,
+                                      const DenseMatrix &shape_bar,
+                                      DenseMatrix &PointMat_bar) const
+{
+   MFEM_ABORT("method is not implemented for this class");
+}
+
 void FiniteElement::CalcDivShape (
    const IntegrationPoint &ip, Vector &divshape) const
 {
@@ -90,6 +98,60 @@ void FiniteElement::CalcPhysCurlShape(ElementTransformation &Trans,
    }
 }
 
+void FiniteElement::CalcPhysCurlShapeRevDiff(ElementTransformation &Trans,
+                                             const DenseMatrix &curlshape_bar,
+                                             DenseMatrix &PointMat_bar) const
+{
+   switch (dim)
+   {
+      case 3:
+      {
+#ifdef MFEM_THREAD_SAFE
+         DenseMatrix vshape(dof, dim);
+#endif
+         DenseMatrix vshapedxt(dof, dim);
+         DenseMatrix vshapedxt_bar(dof, dim);
+
+         CalcCurlShape(Trans.GetIntPoint(), vshape);
+         const auto &jac = Trans.Jacobian();
+         MultABt(vshape, jac, vshapedxt);
+
+         const double weight = Trans.Weight();
+         // curl_shape *= 1.0 / weight;
+
+         /// start reverse pass
+         auto &isotrans = dynamic_cast<IsoparametricTransformation&>(Trans);
+
+         /// curl_shape = vshapedxt / weight;
+         double weight_bar = 0.0;
+         for (int j = 0; j < curlshape_bar.Width(); ++j)
+         {
+            for (int i = 0; i < curlshape_bar.Height(); ++i)
+            {
+               weight_bar -= curlshape_bar(i,j) * vshapedxt(i,j) / pow(weight, 2);
+            }
+         }
+         vshapedxt_bar = curlshape_bar; vshapedxt_bar *= (1.0 / weight);
+
+         /// double weight = Trans.Weight();
+         isotrans.WeightRevDiff(weight_bar, PointMat_bar);
+
+         /// const auto &jac = Trans.Jacobian();
+         /// MultABt(vshape, jac, vshapedxt);
+         double jac_bar_buffer[9];
+         DenseMatrix jac_bar(jac_bar_buffer, jac.Width(), jac.Height());
+         MultAtB(vshapedxt_bar, vshape, jac_bar);
+         isotrans.JacobianRevDiff(jac_bar, PointMat_bar);
+         break;
+      }
+      case 2:
+         MFEM_ABORT("CalcPhysCurlShapeRevDiff not implemented!\n");
+         break;
+      default:
+         MFEM_ABORT("Invalid dimension, Dim = " << dim);
+   }
+}
+
 void FiniteElement::GetFaceDofs(int face, int **dofs, int *ndofs) const
 {
    MFEM_ABORT("method is not overloaded");
@@ -138,6 +200,16 @@ void FiniteElement::ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
    mfem_error ("FiniteElement::ProjectFromNodes() (vector) is not overloaded!");
 }
 
+void FiniteElement::ProjectRevDiff (
+   const Vector &P_bar,
+   VectorCoefficient &vc,
+   ElementTransformation &Trans,
+   DenseMatrix &PointMat_bar) const
+{
+   mfem_error ("FiniteElement::ProjectRevDiff (...) (vector) is not "
+               "overloaded !");
+}
+
 void FiniteElement::ProjectMatrixCoefficient(
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
 {
@@ -186,6 +258,26 @@ void FiniteElement::CalcPhysShape(ElementTransformation &Trans,
    }
 }
 
+void FiniteElement::CalcPhysShapeRevDiff(ElementTransformation &Trans,
+                                         const Vector &shape_bar,
+                                         DenseMatrix &PointMat_bar) const
+{
+   if (map_type == INTEGRAL)
+   {
+#ifdef MFEM_THREAD_SAFE
+      Vector shape(dof);
+#endif
+      CalcShape(Trans.GetIntPoint(), shape);
+      // shape /= Trans.Weight();
+      auto weight = Trans.Weight();
+      auto weight_bar = -(shape_bar * shape) / pow(weight, 2);
+
+      // cast the ElementTransformation
+      auto &isotrans = dynamic_cast<IsoparametricTransformation &>(Trans);
+      isotrans.WeightRevDiff(weight_bar, PointMat_bar);
+   }
+}
+
 void FiniteElement::CalcPhysDShape(ElementTransformation &Trans,
                                    DenseMatrix &dshape) const
 {
@@ -894,6 +986,46 @@ void VectorFiniteElement::CalcVShape_RT (
    shape *= (1.0 / Trans.Weight());
 }
 
+void VectorFiniteElement::CalcVShape_RTRevDiff(ElementTransformation &Trans,
+                                               const DenseMatrix &vshape_bar,
+                                               DenseMatrix &PointMat_bar) const
+{
+   MFEM_ASSERT(map_type == H_DIV, "");
+#ifdef MFEM_THREAD_SAFE
+   DenseMatrix vshape(dof, dim);
+   DenseMatrix vshapedxt(dof, dim);
+   DenseMatrix vshapedxt_bar(dof, dim);
+#else
+   vshapedxt.SetSize(dof, dim);
+   vshapedxt_bar.SetSize(dof, dim);
+#endif
+   CalcVShape(Trans.GetIntPoint(), vshape);
+   const auto &jac = Trans.Jacobian();
+   MultABt(vshape, jac, vshapedxt);
+
+   const double weight = Trans.Weight();
+   // shape *= (1.0 / weight);
+
+   /// start reverse pass
+   auto &isotrans = dynamic_cast<IsoparametricTransformation&>(Trans);
+
+   double weight_bar = 0.0;
+   for (int j = 0; j < vshape_bar.Width(); ++j)
+   {
+      for (int i = 0; i < vshape_bar.Height(); ++i)
+      {
+         weight_bar -= vshape_bar(i,j) * vshapedxt(i,j) / pow(weight,2);
+      }
+   }
+   isotrans.WeightRevDiff(weight_bar, PointMat_bar);
+
+   vshapedxt_bar = vshape_bar; vshapedxt_bar *= (1.0 / weight);
+   double jac_bar_buffer[9];
+   DenseMatrix jac_bar(jac_bar_buffer, jac.Width(), jac.Height());
+   MultAtB(vshapedxt_bar, vshape, jac_bar);
+   isotrans.JacobianRevDiff(jac_bar, PointMat_bar);
+}
+
 void VectorFiniteElement::CalcVShape_ND (
    ElementTransformation &Trans, DenseMatrix &shape) const
 {
@@ -905,6 +1037,46 @@ void VectorFiniteElement::CalcVShape_ND (
    Mult(vshape, Trans.InverseJacobian(), shape);
 }
 
+void VectorFiniteElement::CalcVShape_NDRevDiff(ElementTransformation &Trans,
+                                               const DenseMatrix &vshape_bar,
+                                               DenseMatrix &PointMat_bar) const
+{
+   MFEM_ASSERT(map_type == H_CURL, "");
+#ifdef MFEM_THREAD_SAFE
+   DenseMatrix vshape(dof, dim);
+   DenseMatrix vshapedxt(dof, dim);
+   DenseMatrix vshapedxt_bar(dof, dim);
+#else
+   vshapedxt.SetSize(dof, dim);
+   vshapedxt_bar.SetSize(dof, dim);
+#endif
+   CalcVShape(Trans.GetIntPoint(), vshape);
+   const auto &adjJ = Trans.AdjugateJacobian();
+   Mult(vshape, adjJ, vshapedxt);
+
+   const double weight = Trans.Weight();
+   // shape *= (1.0 / weight);
+
+   /// start reverse pass
+   auto &isotrans = dynamic_cast<IsoparametricTransformation&>(Trans);
+
+   double weight_bar = 0.0;
+   for (int j = 0; j < vshape_bar.Width(); ++j)
+   {
+      for (int i = 0; i < vshape_bar.Height(); ++i)
+      {
+         weight_bar -= vshape_bar(i,j) * vshapedxt(i,j) / pow(weight, 2);
+      }
+   }
+   isotrans.WeightRevDiff(weight_bar, PointMat_bar);
+
+   vshapedxt_bar = vshape_bar; vshapedxt_bar *= (1.0 / weight);
+   double adjJ_bar_buffer[9];
+   DenseMatrix adjJ_bar(adjJ_bar_buffer, adjJ.Width(), adjJ.Height());
+   MultAtB(vshape, vshapedxt_bar, adjJ_bar);
+   isotrans.AdjugateJacobianRevDiff(adjJ_bar, PointMat_bar);
+}
+
 void VectorFiniteElement::Project_RT(
    const double *nk, const Array<int> &d2n,
    VectorCoefficient &vc, ElementTransformation &Trans, Vector &dofs) const
@@ -942,6 +1114,47 @@ void VectorFiniteElement::Project_RT(
    }
 }
 
+void VectorFiniteElement::Project_RTRevDiff(
+   const Vector &P_bar,
+   const double *nk, const Array<int> &d2n,
+   VectorCoefficient &vc, ElementTransformation &Trans,
+   DenseMatrix &PointMat_bar) const
+{
+   double vk[Geometry::MaxDim];
+   const int sdim = Trans.GetSpaceDim();
+   MFEM_ASSERT(vc.GetVDim() == sdim, "");
+   Vector xk(vk, sdim);
+   MFEM_ASSERT(dim == sdim, "VectorFiniteElement::Project_RTRevDiff\n"
+               "\tOnly implemented if space dim == reference dim!\n");
+
+   DenseMatrix temp_bar(PointMat_bar.Height(), PointMat_bar.Width());
+
+   IsoparametricTransformation &isotrans =
+      dynamic_cast<IsoparametricTransformation&>(Trans);
+
+   for (int k = 0; k < dof; k++)
+   {
+      temp_bar = 0.0;
+      isotrans.SetIntPoint(&Nodes.IntPoint(k));
+      vc.Eval(xk, isotrans, Nodes.IntPoint(k));
+      // dof_k = nk^t adj(J) xk
+      const Vector nk_vec(const_cast<double*>(nk + d2n[k]*dim), sdim);
+      double adjJ_bar_buffer[Geometry::MaxDim*Geometry::MaxDim];
+      DenseMatrix adjJ_bar(adjJ_bar_buffer, dim, dim);
+      MultVWt(nk_vec, xk, adjJ_bar);
+      isotrans.AdjugateJacobianRevDiff(adjJ_bar, temp_bar);
+
+      double V_bar_buffer[Geometry::MaxDim];
+      Vector V_bar(V_bar_buffer, sdim);
+      isotrans.AdjugateJacobian().MultTranspose(nk_vec, V_bar);
+      vc.EvalRevDiff(V_bar, isotrans,
+                     Nodes.IntPoint(k), temp_bar);
+
+      temp_bar *= P_bar(k);
+      PointMat_bar += temp_bar;
+   }
+}
+
 void VectorFiniteElement::ProjectMatrixCoefficient_RT(
    const double *nk, const Array<int> &d2n,
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
@@ -1159,6 +1372,48 @@ void VectorFiniteElement::Project_ND(
    }
 }
 
+void VectorFiniteElement::Project_NDRevDiff(
+   const Vector &P_bar,
+   const double *tk, const Array<int> &d2t,
+   VectorCoefficient &vc, ElementTransformation &Trans,
+   DenseMatrix &PointMat_bar) const
+{
+   double vk[Geometry::MaxDim];
+   const int sdim = Trans.GetSpaceDim();
+   MFEM_ASSERT(vc.GetVDim() == sdim, "");
+   Vector xk(vk, sdim);
+   MFEM_ASSERT(dim == sdim, "VectorFiniteElement::Project_NDRevDiff\n"
+               "\tOnly implemented if space dim == reference dim!\n");
+
+   DenseMatrix temp_bar(PointMat_bar.Height(), PointMat_bar.Width());
+
+   IsoparametricTransformation &isotrans =
+      dynamic_cast<IsoparametricTransformation&>(Trans);
+
+   for (int k = 0; k < dof; k++)
+   {
+      temp_bar = 0.0;
+      isotrans.SetIntPoint(&Nodes.IntPoint(k));
+      vc.Eval(xk, isotrans, Nodes.IntPoint(k));
+      // dof_k = nk^t J xk
+      const Vector tk_vec(const_cast<double*>(tk + d2t[k]*dim), sdim);
+
+      double J_bar_buffer[Geometry::MaxDim*Geometry::MaxDim];
+      DenseMatrix J_bar(J_bar_buffer, dim, dim);
+      MultVWt(xk, tk_vec, J_bar);
+      isotrans.JacobianRevDiff(J_bar, temp_bar);
+
+      double V_bar_buffer[Geometry::MaxDim];
+      Vector V_bar(V_bar_buffer, sdim);
+      isotrans.Jacobian().Mult(tk_vec, V_bar);
+      vc.EvalRevDiff(V_bar, isotrans,
+                     Nodes.IntPoint(k), temp_bar);
+
+      temp_bar *= P_bar(k);
+      PointMat_bar += temp_bar;
+   }
+}
+
 void VectorFiniteElement::ProjectMatrixCoefficient_ND(
    const double *tk, const Array<int> &d2t,
    MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
@@ -1538,7 +1793,7 @@ void VectorFiniteElement::LocalRestriction_ND(
 
 
 Poly_1D::Basis::Basis(const int p, const double *nodes, EvalType etype)
-   : etype(etype), auxiliary_basis(NULL)
+   : etype(etype), auxiliary_basis(NULL), scale_integrated(false)
 {
    switch (etype)
    {
@@ -1838,11 +2093,29 @@ void Poly_1D::Basis::EvalIntegrated(const Vector &d_aux, Vector &u) const
    MFEM_VERIFY(etype == Integrated,
                "EvalIntegrated is only valid for Integrated basis type");
    int p = d_aux.Size() - 1;
+   // See Gerritsma, M. (2010).  "Edge functions for spectral element methods",
+   // in Lecture Notes in Computational Science and Engineering, 199--207.
    u[0] = -d_aux[0];
    for (int j=1; j<p; ++j)
    {
       u[j] = u[j-1] - d_aux[j];
    }
+   // If scale_integrated is true, the degrees of freedom represent mean values,
+   // otherwise they represent subcell integrals. Generally, scale_integrated
+   // should be true for MapType::VALUE, and false for other map types.
+   if (scale_integrated)
+   {
+      Vector &aux_nodes = auxiliary_basis->x;
+      for (int j=0; j<aux_nodes.Size()-1; ++j)
+      {
+         u[j] *= aux_nodes[j+1] - aux_nodes[j];
+      }
+   }
+}
+
+void Poly_1D::Basis::ScaleIntegrated(bool scale_integrated_)
+{
+   scale_integrated = scale_integrated_;
 }
 
 Poly_1D::Basis::~Basis()
@@ -2379,6 +2652,18 @@ NodalTensorFiniteElement::NodalTensorFiniteElement(const int dims,
    lex_ordering = dof_map;
 }
 
+void NodalTensorFiniteElement::SetMapType(const int map_type)
+{
+   ScalarFiniteElement::SetMapType(map_type);
+   // If we are using the "integrated" basis, the basis functions should be
+   // scaled for MapType::VALUE, and not scaled for MapType::INTEGRAL. This
+   // ensures spectral equivalence of the mass matrix with its low-order-refined
+   // counterpart (cf. LORDiscretization)
+   if (basis1d.IsIntegratedType())
+   {
+      basis1d.ScaleIntegrated(map_type == VALUE);
+   }
+}
 
 VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                                                      const int d,

fem/fe/fe_base.hpp

@@ -245,6 +245,7 @@ protected:
    mutable int orders[Geometry::MaxDim]; ///< Anisotropic orders
    IntegrationRule Nodes;
 #ifndef MFEM_THREAD_SAFE
+   mutable Vector shape;
    mutable DenseMatrix vshape; // Dof x Dim
 #endif
    /// Container for all DofToQuad objects created by the FiniteElement.
@@ -362,6 +363,10 @@ public:
    /** The size (#dof) of the result Vector @a shape must be set in advance. */
    void CalcPhysShape(ElementTransformation &Trans, Vector &shape) const;
 
+   void CalcPhysShapeRevDiff(ElementTransformation &Trans,
+                             const Vector &shape_bar,
+                             DenseMatrix &PointMat_bar) const;
+
    /** @brief Evaluate the gradients of all shape functions of a scalar finite
        element in reference space at the given point @a ip. */
    /** Each row of the result DenseMatrix @a dshape contains the derivatives of
@@ -400,6 +405,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const;
 
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const;
+
    /// Equivalent to the CalcVShape() method with the same arguments.
    void CalcPhysVShape(ElementTransformation &Trans, DenseMatrix &shape) const
    { CalcVShape(Trans, shape); }
@@ -435,6 +444,10 @@ public:
    void CalcPhysCurlShape(ElementTransformation &Trans,
                           DenseMatrix &curl_shape) const;
 
+   void CalcPhysCurlShapeRevDiff(ElementTransformation &Trans,
+                                 const DenseMatrix &curlshape_bar,
+                                 DenseMatrix &PointMat_bar) const;
+
    /** @brief Get the dofs associated with the given @a face.
        @a *dofs is set to an internal array of the local dofc on the
        face, while *ndofs is set to the number of dofs on that face.
@@ -519,6 +532,14 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const;
 
+   /** Given a vector coefficient and a transformation, compute the derivative of
+          its projection (approximation) in the local finite dimensional space
+          w.r.t. the mesh nodes (VectorFiniteElements) */
+   virtual void ProjectRevDiff(const Vector &P_bar,
+                               VectorCoefficient &vc,
+                               ElementTransformation &Trans,
+                               DenseMatrix &PointMat_bar) const;
+
    /** @brief Given a vector of values at the finite element nodes and a
        transformation, compute its projection (approximation) in the local
        finite dimensional space in terms of the degrees of freedom. Valid for
@@ -657,7 +678,7 @@ public:
    /** @brief Set the FiniteElement::MapType of the element to either VALUE or
        INTEGRAL. Also sets the FiniteElement::DerivType to GRAD if the
        FiniteElement::MapType is VALUE. */
-   void SetMapType(int M)
+   virtual void SetMapType(int M)
    {
       MFEM_VERIFY(M == VALUE || M == INTEGRAL, "unknown MapType");
       map_type = M;
@@ -791,15 +812,24 @@ protected:
 #ifndef MFEM_THREAD_SAFE
    mutable DenseMatrix J, Jinv;
    mutable DenseMatrix curlshape, curlshape_J;
+   mutable DenseMatrix vshapedxt, vshapedxt_bar;
 #endif
    void SetDerivMembers();
 
    void CalcVShape_RT(ElementTransformation &Trans,
                       DenseMatrix &shape) const;
 
+   void CalcVShape_RTRevDiff(ElementTransformation &Trans,
+                             const DenseMatrix &shape_bar,
+                             DenseMatrix &PointMat_bar) const;
+
    void CalcVShape_ND(ElementTransformation &Trans,
                       DenseMatrix &shape) const;
 
+   void CalcVShape_NDRevDiff(ElementTransformation &Trans,
+                             const DenseMatrix &shape_bar,
+                             DenseMatrix &PointMat_bar) const;
+
    /** @brief Project a vector coefficient onto the RT basis functions
        @param nk    Face normal vectors for this element type
        @param d2n   Offset into nk for each degree of freedom
@@ -845,6 +875,23 @@ protected:
                    const FiniteElement &fe, ElementTransformation &Trans,
                    DenseMatrix &I) const;
 
+   /** Reverse-mode differentiation of Project_ND w.r.t. the mesh node
+       locations in the element described by @a T
+       @param[in] P_bar - derivative of function with respect to the projection
+       @param[in] nk - Face normal vectors for this element type
+       @param[in] d2n - Offset into nk for each degree of freedom
+       @param[in] vc - VectorCoefficient being projected
+       @param[in] Trans - an element transformation
+       @param[out] PointMat_bar - derivative of projected degrees of freedom w.r.t.
+                              mesh nodes
+       @warning - only implemented for the same space and reference dimension
+   */
+   void Project_RTRevDiff(const Vector &P_bar,
+                          const double *nk, const Array<int> &d2n,
+                          VectorCoefficient &vc,
+                          ElementTransformation &Trans,
+                          DenseMatrix &PointMat_bar) const;
+
    // rotated gradient in 2D
    void ProjectGrad_RT(const double *nk, const Array<int> &d2n,
                        const FiniteElement &fe, ElementTransformation &Trans,
@@ -883,6 +930,22 @@ protected:
                    Vector &vc, ElementTransformation &Trans,
                    Vector &dofs) const;
 
+   /** Reverse-mode differentiation of Project_ND w.r.t. the mesh node
+       locations in the element described by @a T
+       @param[in] P_bar - derivative of output with respect to the projection
+       @param[in] tk - Edge tangent vectors for this element type
+       @param[in] d2t - Offset into tk for each degree of freedom
+       @param[in] vc - Vector coefficient being projected
+       @param[in] Trans - Transformation from reference to physical coordinates
+       @param[out] PointMat_bar - derivative of some output w.r.t. mesh nodes
+       @warning - only implemented for the same space and reference dimension
+   */
+   void Project_NDRevDiff(const Vector &P_bar,
+                          const double *tk, const Array<int> &d2t,
+                          VectorCoefficient &vc,
+                          ElementTransformation &Trans,
+                          DenseMatrix &PointMat_bar) const;
+
    /// Project the rows of the matrix coefficient in an ND space
    void ProjectMatrixCoefficient_ND(
       const double *tk, const Array<int> &d2t,
@@ -955,41 +1018,68 @@ public:
 };
 
 
-/// Class for computing 1D special polynomials and their associated basis
+/// @brief Class for computing 1D special polynomials and their associated basis
 /// functions
 class Poly_1D
 {
 public:
+   /// One-dimensional basis evaluation type
    enum EvalType
    {
-      ChangeOfBasis = 0, // Use change of basis, O(p^2) Evals
-      Barycentric   = 1, // Use barycentric Lagrangian interpolation, O(p) Evals
-      Positive      = 2, // Fast evaluation of Bernstein polynomials
-      Integrated    = 3, // Integrated indicator functions (cf. Gerritsma)
-      NumEvalTypes  = 4  // Keep count of the number of eval types
+      ChangeOfBasis = 0, ///< Use change of basis, O(p^2) Evals
+      Barycentric   = 1, ///< Use barycentric Lagrangian interpolation, O(p) Evals
+      Positive      = 2, ///< Fast evaluation of Bernstein polynomials
+      Integrated    = 3, ///< Integrated indicator functions (cf. Gerritsma)
+      NumEvalTypes  = 4  ///< Keep count of the number of eval types
    };
 
+   /// @brief Class for evaluating 1D nodal, positive (Bernstein), or integrated
+   /// (Gerritsma) bases.
    class Basis
    {
    private:
-      int etype;
+      EvalType etype; ///< Determines how the basis functions should be evaluated.
       DenseMatrixInverse Ai;
       mutable Vector x, w;
-      // The following data members are used for "integrated basis type", which
-      // is defined in terms of nodal basis of one degree higher.
+      /// The following data members are used for "integrated basis type", which
+      /// is defined in terms of nodal basis of one degree higher.
+      ///@{
       mutable Vector u_aux, d_aux, d2_aux;
-      Basis *auxiliary_basis; // Non-NULL only for etype == Integrated
+      ///@}
+      /// @brief An auxiliary nodal basis used to evaluate the integrated basis.
+      /// This member variable is NULL whenever etype != Integrated.
+      Basis *auxiliary_basis;
+      /// Should the integrated basis functions be scaled? See ScaleIntegrated.
+      bool scale_integrated;
 
    public:
-      /// Create a nodal or positive (Bernstein) basis
+      /// Create a nodal or positive (Bernstein) basis of degree @a p
       Basis(const int p, const double *nodes, EvalType etype = Barycentric);
+      /// Evaluate the basis functions at point @a x in [0,1]
       void Eval(const double x, Vector &u) const;
+      /// @brief Evaluate the basis functions and their derivatives at point @a
+      /// x in [0,1]
       void Eval(const double x, Vector &u, Vector &d) const;
+      /// @brief Evaluate the basis functions and their first two derivatives at
+      /// point @a x in [0,1]
       void Eval(const double x, Vector &u, Vector &d, Vector &d2) const;
-      /// Evaluate the "integrated" basis, which is given by the negative
-      /// partial sum of the corresponding closed basis derivatives. The closed
-      /// basis derivatives are given by @a d, and the result is stored in @a i.
+      /// @brief Evaluate the "integrated" basis type using pre-computed closed
+      /// basis derivatives.
+      ///
+      /// This basis is given by the negative partial sum of the corresponding
+      /// closed basis derivatives. The closed basis derivatives are given by @a
+      /// d, and the result is stored in @a i.
       void EvalIntegrated(const Vector &d, Vector &i) const;
+      /// @brief Set whether the "integrated" basis should be scaled by the
+      /// subcell sizes. Has no effect for non-integrated bases.
+      ///
+      /// Generally, this should be true for mfem::FiniteElement::MapType VALUE
+      /// and false for all other map types. If this option is enabled, the
+      /// basis functions will be scaled by the widths of the subintervals, so
+      /// that the basis functions represent mean values. Otherwise, the basis
+      /// functions represent integrated values.
+      void ScaleIntegrated(bool scale_integrated_);
+      /// Returns true if the basis is "integrated", false otherwise.
       bool IsIntegratedType() const { return etype == Integrated; }
       ~Basis();
    };
@@ -1193,6 +1283,8 @@ public:
              ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
    }
 
+   virtual void SetMapType(const int map_type_);
+
    virtual void GetTransferMatrix(const FiniteElement &fe,
                                   ElementTransformation &Trans,
                                   DenseMatrix &I) const

fem/fe/fe_nd.hpp

@@ -42,6 +42,11 @@ public:
                            DenseMatrix &shape) const
    { CalcVShape_ND(Trans, shape); }
 
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_NDRevDiff(Trans, shape_bar, PointMat_bar); }
+
    virtual void CalcCurlShape(const IntegrationPoint &ip,
                               DenseMatrix &curl_shape) const;
 
@@ -120,6 +125,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const
    { CalcVShape_ND(Trans, shape); }
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_NDRevDiff(Trans, shape_bar, PointMat_bar); }
    virtual void CalcCurlShape(const IntegrationPoint &ip,
                               DenseMatrix &curl_shape) const;
    virtual void GetLocalInterpolation(ElementTransformation &Trans,
@@ -139,6 +148,11 @@ public:
       if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
       else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    }
+   virtual void ProjectRevDiff(const Vector &P_bar,
+                               VectorCoefficient &vc,
+                               ElementTransformation &Trans,
+                               DenseMatrix &dofs_bar) const
+   { Project_NDRevDiff(P_bar, tk, dof2tk, vc, Trans, dofs_bar); }
    virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
                                  Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
@@ -182,6 +196,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const
    { CalcVShape_ND(Trans, shape); }
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_NDRevDiff(Trans, shape_bar, PointMat_bar); }
    virtual void CalcCurlShape(const IntegrationPoint &ip,
                               DenseMatrix &curl_shape) const;
    virtual void GetLocalInterpolation(ElementTransformation &Trans,
@@ -198,6 +216,11 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   virtual void ProjectRevDiff(const Vector &P_bar,
+                               VectorCoefficient &vc,
+                               ElementTransformation &Trans,
+                               DenseMatrix &dofs_bar) const
+   { Project_NDRevDiff(P_bar, tk, dof2tk, vc, Trans, dofs_bar); }
    virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
                                  Vector &dofs) const
    { Project_ND(tk, dof2tk, vc, Trans, dofs); }
@@ -241,6 +264,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const
    { CalcVShape_ND(Trans, shape); }
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_NDRevDiff(Trans, shape_bar, PointMat_bar); }
    virtual void CalcCurlShape(const IntegrationPoint &ip,
                               DenseMatrix &curl_shape) const;
    virtual void GetLocalInterpolation(ElementTransformation &Trans,
@@ -292,6 +319,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const
    { CalcVShape_ND(Trans, shape); }
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_NDRevDiff(Trans, shape_bar, PointMat_bar); }
    // virtual void CalcCurlShape(const IntegrationPoint &ip,
    //                            DenseMatrix &curl_shape) const;
    virtual void GetLocalInterpolation(ElementTransformation &Trans,

fem/fe/fe_rt.hpp

@@ -41,6 +41,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const
    { CalcVShape_RT(Trans, shape); }
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_RTRevDiff(Trans, shape_bar, PointMat_bar); }
    virtual void CalcDivShape(const IntegrationPoint &ip,
                              Vector &divshape) const;
    virtual void GetLocalInterpolation(ElementTransformation &Trans,
@@ -60,6 +64,11 @@ public:
       if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
       else { Project_RT(nk, dof2nk, vc, Trans, dofs); }
    }
+   virtual void ProjectRevDiff(const Vector &P_bar,
+                               VectorCoefficient &vc,
+                               ElementTransformation &Trans,
+                               DenseMatrix &dofs_bar) const
+   { Project_RTRevDiff(P_bar, nk, dof2nk, vc, Trans, dofs_bar); }
    virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
                                  Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
@@ -110,6 +119,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const
    { CalcVShape_RT(Trans, shape); }
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_RTRevDiff(Trans, shape_bar, PointMat_bar); }
    virtual void CalcDivShape(const IntegrationPoint &ip,
                              Vector &divshape) const;
    virtual void GetLocalInterpolation(ElementTransformation &Trans,
@@ -172,6 +185,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const
    { CalcVShape_RT(Trans, shape); }
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_RTRevDiff(Trans, shape_bar, PointMat_bar); }
    virtual void CalcDivShape(const IntegrationPoint &ip,
                              Vector &divshape) const;
    virtual void GetLocalInterpolation(ElementTransformation &Trans,
@@ -232,6 +249,10 @@ public:
    virtual void CalcVShape(ElementTransformation &Trans,
                            DenseMatrix &shape) const
    { CalcVShape_RT(Trans, shape); }
+   virtual void CalcVShapeRevDiff(ElementTransformation &Trans,
+                                  const DenseMatrix &shape_bar,
+                                  DenseMatrix &PointMat_bar) const
+   { CalcVShape_RTRevDiff(Trans, shape_bar, PointMat_bar); }
    virtual void CalcDivShape(const IntegrationPoint &ip,
                              Vector &divshape) const;
    virtual void GetLocalInterpolation(ElementTransformation &Trans,
@@ -248,6 +269,11 @@ public:
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }
+   virtual void ProjectRevDiff(const Vector &P_bar,
+                               VectorCoefficient &vc,
+                               ElementTransformation &Trans,
+                               DenseMatrix &dofs_bar) const
+   { Project_RTRevDiff(P_bar, nk, dof2nk, vc, Trans, dofs_bar); }
    virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
                                  Vector &dofs) const
    { Project_RT(nk, dof2nk, vc, Trans, dofs); }

fem/fespace.cpp

@@ -935,6 +935,12 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    if (cP_is_set) { return; }
    cP_is_set = true;
 
+   if (FEColl()->GetContType() == FiniteElementCollection::DISCONTINUOUS)
+   {
+      cP = cR = cR_hp = NULL; // will be treated as identities
+      return;
+   }
+
    Array<int> master_dofs, slave_dofs, highest_dofs;
 
    IsoparametricTransformation T;
@@ -1055,6 +1061,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
             // get lowest order variant DOFs and FE
             int p = GetEntityDofs(entity, i, master_dofs, geom, 0);
             const auto *master_fe = fec->GetFE(geom, p);
+            if (!master_fe) { break; }
 
             // constrain all higher order DOFs: interpolate lowest order function
             for (int variant = 1; ; variant++)
@@ -1193,7 +1200,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
       if (cR_hp) { MakeVDimMatrix(*cR_hp); }
    }
 
-   if (Device::IsEnabled()) { cP->BuildTranspose(); }
+   cP->EnsureMultTranspose();
 }
 
 void FiniteElementSpace::MakeVDimMatrix(SparseMatrix &mat) const
@@ -1301,7 +1308,14 @@ const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
       FaceRestriction *res;
       if (is_dg_space)
       {
-         res = new L2FaceRestriction(*this, e_ordering, type, m);
+         if (Conforming())
+         {
+            res = new L2FaceRestriction(*this, e_ordering, type, m);
+         }
+         else
+         {
+            res = new NCL2FaceRestriction(*this, e_ordering, type, m);
+         }
       }
       else
       {

fem/gridfunc.cpp

@@ -3948,6 +3948,129 @@ std::ostream &operator<<(std::ostream &out, const QuadratureFunction &qf)
    return out;
 }
 
+void QuadratureFunction::SaveVTU(std::ostream &out, VTKFormat format,
+                                 int compression_level) const
+{
+   out << R"(<VTKFile type="UnstructuredGrid" version="0.1")";
+   if (compression_level != 0)
+   {
+      out << R"( compressor="vtkZLibDataCompressor")";
+   }
+   out << " byte_order=\"" << VTKByteOrder() << "\">\n";
+   out << "<UnstructuredGrid>\n";
+
+   const char *fmt_str = (format == VTKFormat::ASCII) ? "ascii" : "binary";
+   const char *type_str = (format != VTKFormat::BINARY32) ? "Float64" : "Float32";
+   std::vector<char> buf;
+
+   int np = qspace->GetSize();
+   int ne = qspace->GetNE();
+   int sdim = qspace->GetMesh()->SpaceDimension();
+
+   // For quadrature functions, each point is a vertex cell, so number of cells
+   // is equal to number of points
+   out << "<Piece NumberOfPoints=\"" << np
+       << "\" NumberOfCells=\"" << np << "\">\n";
+
+   // print out the points
+   out << "<Points>\n";
+   out << "<DataArray type=\"" << type_str
+       << "\" NumberOfComponents=\"3\" format=\"" << fmt_str << "\">\n";
+
+   Vector pt(sdim);
+   for (int i = 0; i < ne; i++)
+   {
+      ElementTransformation &T = *qspace->GetMesh()->GetElementTransformation(i);
+      const IntegrationRule &ir = GetElementIntRule(i);
+      for (int j = 0; j < ir.Size(); j++)
+      {
+         T.Transform(ir[j], pt);
+         WriteBinaryOrASCII(out, buf, pt[0], " ", format);
+         if (sdim > 1) { WriteBinaryOrASCII(out, buf, pt[1], " ", format); }
+         else { WriteBinaryOrASCII(out, buf, 0.0, " ", format); }
+         if (sdim > 2) { WriteBinaryOrASCII(out, buf, pt[2], "", format); }
+         else { WriteBinaryOrASCII(out, buf, 0.0, "", format); }
+         if (format == VTKFormat::ASCII) { out << '\n'; }
+      }
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</Points>\n";
+
+   // Write cells (each cell is just a vertex)
+   out << "<Cells>\n";
+   // Connectivity
+   out << R"(<DataArray type="Int32" Name="connectivity" format=")"
+       << fmt_str << "\">\n";
+
+   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(out, buf, i, "\n", format); }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   // Offsets
+   out << R"(<DataArray type="Int32" Name="offsets" format=")"
+       << fmt_str << "\">\n";
+   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(out, buf, i, "\n", format); }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   // Types
+   out << R"(<DataArray type="UInt8" Name="types" format=")"
+       << fmt_str << "\">\n";
+   for (int i = 0; i < np; i++)
+   {
+      uint8_t vtk_cell_type = VTKGeometry::POINT;
+      WriteBinaryOrASCII(out, buf, vtk_cell_type, "\n", format);
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</Cells>\n";
+
+   out << "<PointData>\n";
+   out << "<DataArray type=\"" << type_str << "\" Name=\"u\" format=\""
+       << fmt_str << "\" NumberOfComponents=\"" << vdim << "\">\n";
+   for (int i = 0; i < ne; i++)
+   {
+      DenseMatrix vals;
+      GetElementValues(i, vals);
+      for (int j = 0; j < vals.Size(); ++j)
+      {
+         for (int vd = 0; vd < vdim; ++vd)
+         {
+            WriteBinaryOrASCII(out, buf, vals(vd, j), " ", format);
+         }
+         if (format == VTKFormat::ASCII) { out << '\n'; }
+      }
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(out, buf, compression_level);
+   }
+   out << "</DataArray>\n";
+   out << "</PointData>\n";
+
+   out << "</Piece>\n";
+   out << "</UnstructuredGrid>\n";
+   out << "</VTKFile>" << std::endl;
+}
+
+void QuadratureFunction::SaveVTU(const std::string &filename, VTKFormat format,
+                                 int compression_level) const
+{
+   std::ofstream f(filename + ".vtu");
+   SaveVTU(f, format, compression_level);
+}
+
 
 double ZZErrorEstimator(BilinearFormIntegrator &blfi,
                         GridFunction &u,

fem/gridfunc.hpp

@@ -902,6 +902,22 @@ public:
 
    /// Write the QuadratureFunction to the stream @a out.
    void Save(std::ostream &out) const;
+
+   /// @brief Write the QuadratureFunction to @a out in VTU (ParaView) format.
+   ///
+   /// The data will be uncompressed if @a compression_level is zero, or if the
+   /// format is VTKFormat::ASCII. Otherwise, zlib compression will be used for
+   /// binary data.
+   void SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
+                int compression_level=0) const;
+
+   /// @brief Save the QuadratureFunction to a VTU (ParaView) file.
+   ///
+   /// The extension ".vtu" will be appended to @a filename.
+   /// @sa SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
+   ///             int compression_level=0)
+   void SaveVTU(const std::string &filename, VTKFormat format=VTKFormat::ASCII,
+                int compression_level=0) const;
 };
 
 /// Overload operator<< for std::ostream and QuadratureFunction.

fem/hybridization.cpp

@@ -776,7 +776,7 @@ void Hybridization::MultAfInv(const Vector &b, const Vector &lambda, Vector &bf,
          if (vdof_marker[vdof]) { el_vals(j) = 0.0; }
          else { vdof_marker[vdof] = true; }
       }
-      bf_i.SetDataAndSize(&bf[hat_offsets[i]], vdofs.Size());
+      bf_i.MakeRef(bf, hat_offsets[i], vdofs.Size());
       if (mode == 1)
       {
          el_vals -= bf_i;
@@ -821,7 +821,15 @@ void Hybridization::ReduceRHS(const Vector &b, Vector &b_r) const
    else
    {
       Vector bl(pC ? pC->Height() : Ct->Width());
-      pC ? pC->Mult(bf, bl) : Ct->MultTranspose(bf, bl);
+      if (pC)
+      {
+         pC->Mult(bf, bl);
+      }
+      else
+      {
+         Ct->EnsureMultTranspose();
+         Ct->MultTranspose(bf, bl);
+      }
       b_r.SetSize(pH.Ptr()->Height());
       (P_pc ? P_pc : c_pfes->GetProlongationMatrix())->MultTranspose(bl, b_r);
    }

fem/pbilinearform.cpp

@@ -373,6 +373,7 @@ void ParBilinearForm::FormLinearSystem(
       P.MultTranspose(b, true_B);
       R.Mult(x, true_X);
       p_mat.EliminateBC(p_mat_e, ess_tdof_list, true_X, true_B);
+      R.EnsureMultTranspose();
       R.MultTranspose(true_B, b);
       hybridization->ReduceRHS(true_B, B);
       X.SetSize(B.Size());

fem/pfespace.cpp

@@ -556,11 +556,25 @@ const FaceRestriction *ParFiniteElementSpace::GetFaceRestriction(
       FaceRestriction *res;
       if (is_dg_space)
       {
-         res = new ParL2FaceRestriction(*this, e_ordering, type, m);
+         if (Conforming())
+         {
+            res = new ParL2FaceRestriction(*this, e_ordering, type, m);
+         }
+         else
+         {
+            res = new ParNCL2FaceRestriction(*this, e_ordering, type, m);
+         }
       }
       else
       {
-         res = new H1FaceRestriction(*this, e_ordering, type);
+         if (Conforming())
+         {
+            res = new H1FaceRestriction(*this, e_ordering, type);
+         }
+         else
+         {
+            res = new ParNCH1FaceRestriction(*this, e_ordering, type);
+         }
       }
       L2F[key] = res;
       return res;

fem/prestriction.hpp

@@ -23,34 +23,348 @@ namespace mfem
 
 class ParFiniteElementSpace;
 
-/// Operator that extracts Face degrees of freedom in parallel.
-/** Objects of this type are typically created and owned by FiniteElementSpace
-    objects, see FiniteElementSpace::GetFaceRestriction(). */
-class ParL2FaceRestriction : public L2FaceRestriction
+/// Operator that extracts Face degrees of freedom for NCMesh in parallel.
+/** Objects of this type are typically created and owned by
+    ParFiniteElementSpace objects, see
+    ParFiniteElementSpace::GetFaceRestriction(). */
+class ParNCH1FaceRestriction : public H1FaceRestriction
 {
+protected:
+   const FaceType type;
+   InterpolationManager interpolations;
+   mutable Vector x_interp;
+
 public:
-   ParL2FaceRestriction(const ParFiniteElementSpace&, ElementDofOrdering,
+   /** @brief Constructs an ParNCH1FaceRestriction.
+
+       @param[in] fes      The ParFiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs */
+   ParNCH1FaceRestriction(const ParFiniteElementSpace &fes,
+                          ElementDofOrdering ordering,
+                          FaceType type);
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering.
+       @param[in,out] y The L-vector degrees of freedom. */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
+       for the gathering: E-vector to L-vector, and the interpolators from
+       coarse to fine face for master non-comforming faces.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+};
+
+/// Operator that extracts Face degrees of freedom in parallel.
+/** Objects of this type are typically created and owned by
+    ParFiniteElementSpace objects, see
+    ParFiniteElementSpace::GetFaceRestriction(). */
+class ParL2FaceRestriction : virtual public L2FaceRestriction
+{
+protected:
+   /** @brief Constructs an ParL2FaceRestriction.
+
+       @param[in] fes      The ParFiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2
+       @param[in] build    Request the ParL2FaceRestriction to compute the
+                           scatter/gather indices. False should only be used
+                           when inheriting from ParL2FaceRestriction. */
+   ParL2FaceRestriction(const ParFiniteElementSpace& fes,
+                        ElementDofOrdering ordering,
+                        FaceType type,
+                        L2FaceValues m,
+                        bool build);
+
+public:
+   /** @brief Constructs an ParL2FaceRestriction.
+
+       @param[in] fes      The ParFiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2 */
+   ParL2FaceRestriction(const ParFiniteElementSpace& fes,
+                        ElementDofOrdering ordering,
                         FaceType type,
                         L2FaceValues m = L2FaceValues::DoubleValued);
-   void Mult(const Vector &x, Vector &y) const;
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
    /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this L2FaceRestriction. */
-   virtual void FillI(SparseMatrix &mat, const bool keep_nbr_block = false) const;
+       given by this ParL2FaceRestriction.
+
+       @param[in,out] mat The sparse matrix for which we want to initialize the
+                          row offsets.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows. */
+   void FillI(SparseMatrix &mat,
+              const bool keep_nbr_block = false) const override;
+
    /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this L2FaceRestriction. @a mat contains the interior dofs
+       given by this ParL2FaceRestriction. @a mat contains the interior dofs
        contribution, the @a face_mat contains the shared dofs contribution.*/
-   virtual void FillI(SparseMatrix &mat, SparseMatrix &face_mat) const;
+   void FillI(SparseMatrix &mat,
+              SparseMatrix &face_mat) const;
+
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
-       pattern given by this L2FaceRestriction, and the values of ea_data.
+       pattern given by this ParL2FaceRestriction, and the values of ea_data.
        @a mat contains the interior dofs contribution, the @a face_mat contains
        the shared dofs contribution.*/
-   virtual void FillJAndData(const Vector &ea_data,
-                             SparseMatrix &mat,
-                             SparseMatrix &face_mat) const;
+   void FillJAndData(const Vector &ea_data,
+                     SparseMatrix &mat,
+                     SparseMatrix &face_mat) const;
 
-   virtual void FillJAndData(const Vector &ea_data,
-                             SparseMatrix &mat,
-                             const bool keep_nbr_block = false) const;
+   /** @brief Fill the J and Data arrays of the SparseMatrix corresponding to
+       the sparsity pattern given by this ParL2FaceRestriction, and the values of
+       fea_data.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf.
+                           On each face the first local matrix corresponds to
+                           the contribution of elem1 on elem2, and the second to
+                           the contribution of elem2 on elem1.
+       @param[in,out] mat The sparse matrix that is getting filled.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows. */
+   void FillJAndData(const Vector &fea_data,
+                     SparseMatrix &mat,
+                     const bool keep_nbr_block = false) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
+       for the gathering: E-vector to L-vector.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+public:
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::DoubleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x 2 x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void DoubleValuedConformingMult(const Vector& x, Vector& y) const override;
+};
+
+/// Operator that extracts Face degrees of freedom for NCMesh in parallel.
+/** Objects of this type are typically created and owned by
+    ParFiniteElementSpace objects, see
+    ParFiniteElementSpace::GetFaceRestriction(). */
+class ParNCL2FaceRestriction
+   : public NCL2FaceRestriction, public ParL2FaceRestriction
+{
+public:
+   /** @brief Constructs an ParNCL2FaceRestriction.
+
+       @param[in] fes      The ParFiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2 */
+   ParNCL2FaceRestriction(const ParFiniteElementSpace& fes,
+                          ElementDofOrdering ordering,
+                          FaceType type,
+                          L2FaceValues m = L2FaceValues::DoubleValued);
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
+
+   /** @brief Fill the I array of SparseMatrix corresponding to the sparsity
+       pattern given by this ParNCL2FaceRestriction.
+
+       @param[in,out] mat The sparse matrix for which we want to initialize the
+                          row offsets.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows.
+
+       @warning This method is not implemented yet. */
+   void FillI(SparseMatrix &mat,
+              const bool keep_nbr_block = false) const override;
+
+   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
+       given by this ParNCL2FaceRestriction. @a mat contains the interior dofs
+       contribution, the @a face_mat contains the shared dofs contribution.
+
+       @warning This method is not implemented yet. */
+   void FillI(SparseMatrix &mat,
+              SparseMatrix &face_mat) const;
+
+   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
+       pattern given by this ParNCL2FaceRestriction, and the values of ea_data.
+       @a mat contains the interior dofs contribution, the @a face_mat contains
+       the shared dofs contribution.
+
+       @warning This method is not implemented yet. */
+   void FillJAndData(const Vector &fea_data,
+                     SparseMatrix &mat,
+                     SparseMatrix &face_mat) const;
+
+   /** @brief Fill the J and Data arrays of the SparseMatrix corresponding to
+       the sparsity pattern given by this ParNCL2FaceRestriction, and the values
+       of ea_data.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf.
+                           On each face the first local matrix corresponds to
+                           the contribution of elem1 on elem2, and the second to
+                           the contribution of elem2 on elem1.
+       @param[in,out] mat The sparse matrix that is getting filled.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows.
+
+       @warning This method is not implemented yet. */
+   void FillJAndData(const Vector &fea_data,
+                     SparseMatrix &mat,
+                     const bool keep_nbr_block = false) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
+       for the gathering: E-vector to L-vector, and the interpolators from
+       coarse to fine face for master non-comforming faces.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+public:
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with nonconforming faces and when:
+       L2FaceValues m == L2FaceValues::SingleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void SingleValuedNonconformingMult(const Vector& x, Vector& y) const;
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with nonconforming faces and when:
+       L2FaceValues m == L2FaceValues::DoubleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     (face_dofs x vdim x 2 x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void DoubleValuedNonconformingMult(const Vector& x, Vector& y) const override;
 };
 
 }

fem/quadinterpolator_face.cpp

@@ -21,18 +21,21 @@ namespace mfem
 static void GetSigns(const FiniteElementSpace &fes, const FaceType type,
                      Array<bool> &signs)
 {
-   const int dim = fes.GetMesh()->SpaceDimension();
-   int e1, e2;
-   int inf1, inf2;
+   const Mesh &mesh = *fes.GetMesh();
+   const int dim = mesh.SpaceDimension();
    int face_id;
    int f_ind = 0;
-   for (int f = 0; f < fes.GetNF(); ++f)
+   for (int f = 0; f < mesh.GetNumFacesWithGhost(); ++f)
    {
-      fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-      fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-      face_id = inf1 / 64;
-      if ( (type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-           (type==FaceType::Boundary && e2<0 && inf2<0) )
+      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
+      face_id = face.element[0].local_face_id;
+      if (face.IsNonconformingCoarse())
+      {
+         // We skip nonconforming coarse-fine faces as they are treated
+         // by the corresponding nonconforming fine-coarse faces.
+         continue;
+      }
+      else if ( face.IsOfFaceType(type) )
       {
          if (dim==2)
          {

fem/restriction.hpp

@@ -41,7 +41,7 @@ protected:
    const int nedofs;
    Array<int> offsets;
    Array<int> indices;
-   Array<int> gatherMap;
+   Array<int> gather_map;
 
 public:
    ElementRestriction(const FiniteElementSpace&, ElementDofOrdering);
@@ -172,125 +172,719 @@ class H1FaceRestriction : public FaceRestriction
 {
 protected:
    const FiniteElementSpace &fes;
-   const int nf;
+   const int nf; // Number of faces of the requested type
    const int vdim;
    const bool byvdim;
-   const int ndofs;
-   const int dof;
-   const int nfdofs;
-   Array<int> scatter_indices;
-   Array<int> offsets;
-   Array<int> gather_indices;
+   const int face_dofs; // Number of dofs on each face
+   const int elem_dofs; // Number of dofs in each element
+   const int nfdofs; // Total number of face E-vector dofs
+   const int ndofs; // Total number of dofs
+   Array<int> scatter_indices; // Scattering indices for element 1 on each face
+   Array<int> gather_offsets; // offsets for the gathering indices of each dof
+   Array<int> gather_indices; // gathering indices for each dof
 
-public:
-   /** @brief Constructor for a H1FaceRestriction.
+   /** @brief Construct an H1FaceRestriction.
 
-       @param[in] fes The FiniteElementSpace on which this H1FaceRestriction
-                      operates.
-       @param[in] ordering The requested output ordering of the
-                           H1FaceRestriction, either Native or Lexicographic.
-       @param[in] type The requested type of faces on which this operator
-                       extracts the degrees of freedom, either Interior or
-                       Boundary.
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific element ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] build    Request the NCL2FaceRestriction to compute the
+                           scatter/gather indices. False should only be used
+                           when inheriting from H1FaceRestriction.
    */
+   H1FaceRestriction(const FiniteElementSpace& fes,
+                     const ElementDofOrdering ordering,
+                     const FaceType type,
+                     bool build);
+public:
+   /** @brief Construct an H1FaceRestriction.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific element ordering
+       @param[in] type     Request internal or boundary faces dofs */
    H1FaceRestriction(const FiniteElementSpace& fes,
                      const ElementDofOrdering ordering,
                      const FaceType type);
 
-   /** @brief Extract the face degrees of freedom from @a x into @a y.
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
 
-       @param[in]  x The L-vector of degrees of freedom.
-       @param[out] y The degrees of freedom on the face, corresponding to a face
-                     E-vector.
-   */
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
    void Mult(const Vector &x, Vector &y) const override;
 
-   /** @brief Add the face degrees of freedom @a x to the element degrees of
-       freedom @a y.
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
 
-       @param[in]     x The face degrees of freedom on the face.
-       @param[in,out] y The L-vector of degrees of freedom to which we add the
-                        face degrees of freedom.
-   */
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
    void AddMultTranspose(const Vector &x, Vector &y) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
+       for the gathering: E-vector to L-vector.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+protected:
+   mutable Array<int> face_map; // Used in the computation of GetFaceDofs
+
+   /** @brief Verify that H1FaceRestriction is build from an H1 FESpace.
+
+       @param[in] ordering The FESpace element ordering.
+   */
+   void CheckFESpace(const ElementDofOrdering ordering);
+
+   /** @brief Set the scattering indices of elem1, and increment the offsets for
+       the face described by the @a face.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+       @param[in] ordering Request a specific element ordering.
+    */
+   void SetFaceDofsScatterIndices(const Mesh::FaceInformation &face,
+                                  const int face_index,
+                                  const ElementDofOrdering ordering);
+
+   /** @brief Set the gathering indices of elem1 for the interior face described
+       by the @a face.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+       @param[in] ordering Request a specific element ordering.
+    */
+   void SetFaceDofsGatherIndices(const Mesh::FaceInformation &face,
+                                 const int face_index,
+                                 const ElementDofOrdering ordering);
 };
 
-/// Operator that extracts Face degrees of freedom on L2 FiniteElementSpaces.
+/// Operator that extracts Face degrees of freedom for L2 spaces.
 /** Objects of this type are typically created and owned by FiniteElementSpace
     objects, see FiniteElementSpace::GetFaceRestriction(). */
 class L2FaceRestriction : public FaceRestriction
 {
 protected:
    const FiniteElementSpace &fes;
-   const int nf;
-   const int ne;
-   const int vdim;
+   const int nf; // Number of faces of the requested type
+   const int ne; // Number of elements
+   const int vdim; // vdim
    const bool byvdim;
-   const int ndofs;
-   const int dof;
-   const int elemDofs;
+   const int face_dofs; // Number of dofs on each face
+   const int elem_dofs; // Number of dofs in each element
+   const int nfdofs; // Total number of dofs on the faces
+   const int ndofs; // Total number of dofs
+   const FaceType type;
    const L2FaceValues m;
-   const int nfdofs;
-   Array<int> scatter_indices1;
-   Array<int> scatter_indices2;
-   Array<int> offsets;
-   Array<int> gather_indices;
+   Array<int> scatter_indices1; // Scattering indices for element 1 on each face
+   Array<int> scatter_indices2; // Scattering indices for element 2 on each face
+   Array<int> gather_offsets; // offsets for the gathering indices of each dof
+   Array<int> gather_indices; // gathering indices for each dof
 
-   L2FaceRestriction(const FiniteElementSpace&,
-                     const FaceType,
-                     const L2FaceValues m = L2FaceValues::DoubleValued);
+   /** @brief Constructs an L2FaceRestriction.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2
+       @param[in] build    Request the NCL2FaceRestriction to compute the
+                           scatter/gather indices. False should only be used
+                           when inheriting from L2FaceRestriction.
+   */
+   L2FaceRestriction(const FiniteElementSpace& fes,
+                     const ElementDofOrdering ordering,
+                     const FaceType type,
+                     const L2FaceValues m,
+                     bool build);
 
 public:
-   L2FaceRestriction(const FiniteElementSpace&,
-                     const ElementDofOrdering,
-                     const FaceType,
+   /** @brief Constructs an L2FaceRestriction.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2 */
+   L2FaceRestriction(const FiniteElementSpace& fes,
+                     const ElementDofOrdering ordering,
+                     const FaceType type,
                      const L2FaceValues m = L2FaceValues::DoubleValued);
 
-   /** @brief Extract the face degrees of freedom from @a x into @a y.
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
 
-       @param[in]  x The L-vector of degrees of freedom.
-       @param[out] y The degrees of freedom on the face, corresponding to a face
-                     E-vector.
-   */
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf)
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf)
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
    void Mult(const Vector &x, Vector &y) const override;
 
-   /** @brief Add the face degrees of freedom @a x to the element degrees of
-       freedom @a y.
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
 
-       @param[in]     x The face degrees of freedom on the face.
-       @param[in,out] y The L-vector of degrees of freedom to which we add the
-                        face degrees of freedom.
-   */
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf)
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf)
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
    void AddMultTranspose(const Vector &x, Vector &y) const override;
 
-   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this L2FaceRestriction. */
+   /** @brief Fill the I array of SparseMatrix corresponding to the sparsity
+       pattern given by this L2FaceRestriction.
+
+       @param[in,out] mat The sparse matrix for which we want to initialize the
+                          row offsets.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows. */
    virtual void FillI(SparseMatrix &mat, const bool keep_nbr_block = false) const;
 
-   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
-       pattern given by this L2FaceRestriction, and the values of ea_data. */
-   virtual void FillJAndData(const Vector &ea_data,
+   /** @brief Fill the J and Data arrays of the SparseMatrix corresponding to
+       the sparsity pattern given by this L2FaceRestriction, and the values of
+       fea_data.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf
+                           On each face the first local matrix corresponds to
+                           the contribution of elem1 on elem2, and the second to
+                           the contribution of elem2 on elem1.
+       @param[in,out] mat The sparse matrix that is getting filled.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows. */
+   virtual void FillJAndData(const Vector &fea_data,
                              SparseMatrix &mat,
                              const bool keep_nbr_block = false) const;
 
-   /// This methods adds the DG face matrices to the element matrices.
-   void AddFaceMatricesToElementMatrices(Vector &fea_data,
-                                         Vector &ea_data) const;
+   /** @brief This methods adds the DG face matrices to the element matrices.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf
+                           On each face the first and second local matrices
+                           correspond to the contributions of elem1 and elem2 on
+                           themselves respectively.
+       @param[in,out] ea_data The dense matrices representing the element local
+                              contributions for each element to which will be
+                              added the face contributions.
+                              The format is: dofs x dofs x ne, where dofs is the
+                              number of dofs per element and ne the number of
+                              elements. */
+   virtual void AddFaceMatricesToElementMatrices(const Vector &fea_data,
+                                                 Vector &ea_data) const;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
+       for the gathering: E-vector to L-vector.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+protected:
+   mutable Array<int> face_map; // Used in the computation of GetFaceDofs
+
+   /** @brief Verify that L2FaceRestriction is build from an L2 FESpace.
+
+       @param[in] ordering The FESpace element ordering.
+   */
+   void CheckFESpace(const ElementDofOrdering ordering);
+
+   /** @brief Set the scattering indices of elem1, and increment the offsets for
+       the face described by the @a face. The ordering of the face dofs of elem1
+       is lexicographic relative to elem1.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void SetFaceDofsScatterIndices1(const Mesh::FaceInformation &face,
+                                   const int face_index);
+
+   /** @brief Permute and set the scattering indices of elem2, and increment the
+       offsets for the face described by the @a face. The permutation orders the
+       dofs of elem2 lexicographically as the ones of elem1.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void PermuteAndSetFaceDofsScatterIndices2(const Mesh::FaceInformation &face,
+                                             const int face_index);
+
+   /** @brief Permute and set the scattering indices of elem2 for the shared
+       face described by the @a face. The permutation orders the dofs of elem2 as
+       the ones of elem1.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void PermuteAndSetSharedFaceDofsScatterIndices2(
+      const Mesh::FaceInformation &face,
+      const int face_index);
+
+   /** @brief Set the scattering indices of elem2 for the boundary face
+       described by the @a face.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void SetBoundaryDofsScatterIndices2(const Mesh::FaceInformation &face,
+                                       const int face_index);
+
+   /** @brief Set the gathering indices of elem1 for the interior face described
+       by the @a face.
+
+       Note: This function modifies the offsets.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void SetFaceDofsGatherIndices1(const Mesh::FaceInformation &face,
+                                  const int face_index);
+
+   /** @brief Permute and set the gathering indices of elem2 for the interior
+       face described by the @a face. The permutation orders the dofs of elem2 as
+       the ones of elem1.
+
+       Note: This function modifies the offsets.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void PermuteAndSetFaceDofsGatherIndices2(const Mesh::FaceInformation &face,
+                                            const int face_index);
+
+public:
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::SingleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void SingleValuedConformingMult(const Vector& x, Vector& y) const;
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::DoubleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x 2 x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   virtual void DoubleValuedConformingMult(const Vector& x, Vector& y) const;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::SingleValued
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
+   void SingleValuedConformingAddMultTranspose(const Vector& x, Vector& y) const;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector. Should only be used with conforming faces and when:
+       m == L2FacesValues::DoubleValued
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     face_dofs x vdim x 2 x nf
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
+   void DoubleValuedConformingAddMultTranspose(const Vector& x, Vector& y) const;
 };
 
-// Return the face degrees of freedom returned in Lexicographic order.
-void GetFaceDofs(const int dim, const int face_id,
-                 const int dof1d, Array<int> &faceMap);
+/** This struct stores which side is the master nonconforming side and the
+    index of the interpolator, see InterpolationManager class below. */
+struct InterpConfig
+{
+   uint32_t is_non_conforming : 1;
+   uint32_t master_side : 1;
+   uint32_t index : 30;
 
-// Convert from Native ordering to lexicographic ordering
+   // default constructor, create a conforming face with index 0.
+   InterpConfig() = default;
+
+   // Non-conforming face
+   InterpConfig(int master_side, int nc_index)
+      : is_non_conforming(1), master_side(master_side), index(nc_index)
+   { }
+
+   InterpConfig(const InterpConfig&) = default;
+
+   InterpConfig &operator=(const InterpConfig &rhs) = default;
+};
+
+/** @brief This class manages the storage and computation of the interpolations
+    from master (coarse) face to slave (fine) face.
+*/
+class InterpolationManager
+{
+protected:
+   const FiniteElementSpace &fes;
+   const ElementDofOrdering ordering;
+   Array<InterpConfig> interp_config; // interpolator index for each face
+   Vector interpolators; // face_dofs x face_dofs x num_interpolators
+   int nc_cpt; // Counter for interpolators, and used as index.
+
+   /** The interpolators are associated to a key of containing the address of
+       PointMatrix and a local face identifier. */
+   using Key = std::pair<const DenseMatrix*,int>;
+   /// The temporary map used to store the different interpolators.
+   using Map = std::map<Key, std::pair<int,const DenseMatrix*>>;
+   Map interp_map; // The temporary map that stores the interpolators.
+
+public:
+   InterpolationManager() = delete;
+
+   /** @brief main constructor.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs
+    */
+   InterpolationManager(const FiniteElementSpace &fes,
+                        ElementDofOrdering ordering,
+                        FaceType type);
+
+   /** @brief Register the face with @a face and index @a face_index as a
+       conforming face for the interpolation of the degrees of freedom.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void RegisterFaceConformingInterpolation(const Mesh::FaceInformation &face,
+                                            int face_index);
+
+   /** @brief Register the face with @a face and index @a face_index as a
+       conforming face for the interpolation of the degrees of freedom.
+
+       @param[in] face The face information of the current face.
+       @param[in] face_index The interior/boundary face index.
+    */
+   void RegisterFaceCoarseToFineInterpolation(const Mesh::FaceInformation &face,
+                                              int face_index);
+
+   /** @brief Transform the interpolation matrix map into a contiguous memory
+       structure. */
+   void LinearizeInterpolatorMapIntoVector();
+
+   /// @brief Return the total number of interpolators.
+   int GetNumInterpolators() const
+   {
+      return nc_cpt;
+   }
+
+   /** @brief Return an mfem::Vector containing the interpolators in the
+       following format: face_dofs x face_dofs x num_interpolators. */
+   const Vector& GetInterpolators() const
+   {
+      return interpolators;
+   }
+
+   /** @brief Return an array containing the interpolation configuration for
+       each face registered with RegisterFaceConformingInterpolation and
+       RegisterFaceCoarseToFineInterpolation. */
+   const Array<InterpConfig>& GetFaceInterpConfig() const
+   {
+      return interp_config;
+   }
+
+private:
+   /** @brief Returns the interpolation operator from a master (coarse) face to
+       a slave (fine) face.
+
+       @param[in] face The face information of the current face.
+       @param[in] ptMat The PointMatrix describing the position and orientation
+                        of the fine face in the coarse face. This PointMatrix is
+                        usually obtained from the mesh through the method
+                        GetNCFacesPtMat.
+       @param[in] ordering  Request a specific element ordering.
+       @return The dense matrix corresponding to the interpolation of the face
+               degrees of freedom of the master (coarse) face to the slave
+               (fine) face. */
+   const DenseMatrix* GetCoarseToFineInterpolation(
+      const Mesh::FaceInformation &face,
+      const DenseMatrix* ptMat);
+};
+
+/** @brief Operator that extracts face degrees of freedom for L2 nonconforming
+    spaces.
+
+    In order to support face restrictions on nonconforming meshes, this
+    operator interpolates master (coarse) face degrees of freedom onto the
+    slave (fine) face. This allows face integrators to treat nonconforming
+    faces just as regular conforming faces. */
+class NCL2FaceRestriction : virtual public L2FaceRestriction
+{
+protected:
+   InterpolationManager interpolations;
+   mutable Vector x_interp;
+
+   /** @brief Constructs an NCL2FaceRestriction, this is a specialization of a
+       L2FaceRestriction for nonconforming meshes.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2
+       @param[in] build    Request the NCL2FaceRestriction to compute the
+                           scatter/gather indices. False should only be used
+                           when inheriting from NCL2FaceRestriction.
+   */
+   NCL2FaceRestriction(const FiniteElementSpace& fes,
+                       const ElementDofOrdering ordering,
+                       const FaceType type,
+                       const L2FaceValues m,
+                       bool build);
+public:
+   /** @brief Constructs an NCL2FaceRestriction, this is a specialization of a
+       L2FaceRestriction for nonconforming meshes.
+
+       @param[in] fes      The FiniteElementSpace on which this operates
+       @param[in] ordering Request a specific ordering
+       @param[in] type     Request internal or boundary faces dofs
+       @param[in] m        Request the face dofs for elem1, or both elem1 and
+                           elem2
+   */
+   NCL2FaceRestriction(const FiniteElementSpace& fes,
+                       const ElementDofOrdering ordering,
+                       const FaceType type,
+                       const L2FaceValues m = L2FaceValues::DoubleValued);
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in]  x The face E-Vector degrees of freedom with the given format:
+                     if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                     if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs should be ordered according to the given
+                     ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom. */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
+
+   /** @brief Fill the I array of SparseMatrix corresponding to the sparsity
+       pattern given by this NCL2FaceRestriction.
+
+       @param[in,out] mat The sparse matrix for which we want to initialize the
+                          row offsets.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows.
+
+       @warning This method is not implemented yet. */
+   void FillI(SparseMatrix &mat,
+              const bool keep_nbr_block = false) const override;
+
+   /** @brief Fill the J and Data arrays of the SparseMatrix corresponding to
+       the sparsity pattern given by this NCL2FaceRestriction, and the values of
+       ea_data.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf.
+                           On each face the first local matrix corresponds to
+                           the contribution of elem1 on elem2, and the second to
+                           the contribution of elem2 on elem1.
+       @param[in,out] mat The sparse matrix that is getting filled.
+       @param[in] keep_nbr_block When set to true the SparseMatrix will
+                                 include the rows (in addition to the columns)
+                                 corresponding to face-neighbor dofs. The
+                                 default behavior is to disregard those rows.
+
+       @warning This method is not implemented yet. */
+   void FillJAndData(const Vector &fea_data,
+                     SparseMatrix &mat,
+                     const bool keep_nbr_block = false) const override;
+
+   /** @brief This methods adds the DG face matrices to the element matrices.
+
+       @param[in] fea_data The dense matrices representing the local operators
+                           on each face. The format is:
+                           face_dofs x face_dofs x 2 x nf.
+                           On each face the first and second local matrices
+                           correspond to the contributions of elem1 and elem2 on
+                           themselves respectively.
+       @param[in,out] ea_data The dense matrices representing the element local
+                              contributions for each element to which will be
+                              added the face contributions.
+                              The format is: dofs x dofs x ne, where dofs is the
+                              number of dofs per element and ne the number of
+                              elements.
+
+       @warning This method is not implemented yet. */
+   void AddFaceMatricesToElementMatrices(const Vector &fea_data,
+                                         Vector &ea_data) const override;
+
+private:
+   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
+       for the gathering: E-vector to L-vector, and the interpolators from
+       coarse to fine face for master non-comforming faces.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+                                        const FaceType type);
+
+   /** @brief Compute the gather indices: E-vector to L-vector.
+
+       Note: Requires the gather offsets to be computed.
+
+       @param[in] ordering Request a specific element ordering.
+       @param[in] type     Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering ordering,
+                             const FaceType type);
+
+public:
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector. Should only be used with nonconforming faces and when:
+       L2FaceValues m == L2FaceValues::DoubleValued
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with the given format:
+                     (face_dofs x vdim x 2 x nf),
+                     where nf is the number of interior or boundary faces
+                     requested by @a type in the constructor.
+                     The face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   virtual void DoubleValuedNonconformingMult(const Vector& x, Vector& y) const;
+
+   /** @brief Apply a change of basis from fine element basis to coarse element
+       basis for the coarse face dofs. Should only be used when:
+       L2FaceValues m == L2FaceValues::SingleValued
+
+       @param[in] x The dofs vector that needs coarse dofs to be express in term
+                    of the coarse basis, the result is stored in x_interp.
+   */
+   void SingleValuedNonconformingTransposeInterpolation(const Vector& x) const;
+
+   /** @brief Apply a change of basis from fine element basis to coarse element
+       basis for the coarse face dofs. Should only be used when:
+       L2FaceValues m == L2FaceValues::DoubleValued
+
+       @param[in] x The dofs vector that needs coarse dofs to be express in term
+                    of the coarse basis, the result is stored in x_interp.
+   */
+   void DoubleValuedNonconformingTransposeInterpolation(const Vector& x) const;
+};
+
+/** @brief Return the face map that extracts the degrees of freedom for the
+    requested local face of a quad or hex, returned in Lexicographic order.
+
+    @param[in] dim The dimension of the space
+    @param[in] face_id The local face identifier
+    @param[in] dof1d The 1D number of degrees of freedom for each dimension
+    @param[out] face_map The map that maps each face dof to an element dof
+*/
+void GetFaceDofs(const int dim, const int face_id,
+                 const int dof1d, Array<int> &face_map);
+
+/** @brief Convert a dof face index from Native ordering to lexicographic
+    ordering for quads and hexes.
+
+    @param[in] dim The dimension of the element, 2 for quad, 3 for hex
+    @param[in] face_id The local face identifier
+    @param[in] size1d The 1D number of degrees of freedom for each dimension
+    @param[in] index The native index on the face
+    @return The lexicographic index on the face
+*/
 int ToLexOrdering(const int dim, const int face_id, const int size1d,
                   const int index);
 
-// Permute dofs or quads on a face for e2 to match with the ordering of e1
+/** @brief Compute the dof face index of elem2 corresponding to the given dof
+    face index.
+
+    @param[in] dim The dimension of the element, 2 for quad, 3 for hex
+    @param[in] face_id1 The local face identifier of elem1
+    @param[in] face_id2 The local face identifier of elem2
+    @param[in] orientation The orientation of elem2 relative to elem1 on the
+                           face
+    @param[in] size1d The 1D number of degrees of freedom for each dimension
+    @param[in] index The dof index on elem1
+    @return The dof index on elem2 facing the dof on elem1
+*/
 int PermuteFaceL2(const int dim, const int face_id1,
                   const int face_id2, const int orientation,
                   const int size1d, const int index);
 
 }
 
-#endif //MFEM_RESTRICTION
+#endif // MFEM_RESTRICTION

fem/transfer.cpp

@@ -903,7 +903,8 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
                                    const FiniteElementSpace& hFESpace_)
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize())
 {
-   if (lFESpace_.FEColl() == hFESpace_.FEColl())
+   bool isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
+   if (lFESpace_.FEColl() == hFESpace_.FEColl() && !isvar_order)
    {
       OperatorPtr P(Operator::ANY_TYPE);
       hFESpace_.GetTransferOperator(lFESpace_, P);
@@ -912,8 +913,11 @@ TransferOperator::TransferOperator(const FiniteElementSpace& lFESpace_,
    }
    else if (lFESpace_.GetMesh()->GetNE() > 0
             && hFESpace_.GetMesh()->GetNE() > 0
+            && lFESpace_.GetVDim() == 1
+            && hFESpace_.GetVDim() == 1
             && dynamic_cast<const TensorBasisElement*>(lFESpace_.GetFE(0))
             && dynamic_cast<const TensorBasisElement*>(hFESpace_.GetFE(0))
+            && !isvar_order
             && (hFESpace_.FEColl()->GetContType() ==
                 mfem::FiniteElementCollection::CONTINUOUS ||
                 hFESpace_.FEColl()->GetContType() ==
@@ -945,6 +949,7 @@ PRefinementTransferOperator::PRefinementTransferOperator(
    : Operator(hFESpace_.GetVSize(), lFESpace_.GetVSize()), lFESpace(lFESpace_),
      hFESpace(hFESpace_)
 {
+   isvar_order = lFESpace_.IsVariableOrder() || hFESpace_.IsVariableOrder();
 }
 
 PRefinementTransferOperator::~PRefinementTransferOperator() {}
@@ -969,7 +974,7 @@ void PRefinementTransferOperator::Mult(const Vector& x, Vector& y) const
       DofTransformation * doftrans_l = lFESpace.GetElementDofs(i, l_dofs);
 
       const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
-      if (geom != cached_geom)
+      if (geom != cached_geom || isvar_order)
       {
          h_fe = hFESpace.GetFE(i);
          l_fe = lFESpace.GetFE(i);
@@ -1026,7 +1031,7 @@ void PRefinementTransferOperator::MultTranspose(const Vector& x,
       DofTransformation * doftrans_l = lFESpace.GetElementDofs(i, l_dofs);
 
       const Geometry::Type geom = mesh->GetElementBaseGeometry(i);
-      if (geom != cached_geom)
+      if (geom != cached_geom || isvar_order)
       {
          h_fe = hFESpace.GetFE(i);
          l_fe = lFESpace.GetFE(i);
@@ -1424,20 +1429,36 @@ void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
    elem_restrict_lex_l->MultTranspose(localL, y);
 }
 
-#ifdef MFEM_USE_MPI
-TrueTransferOperator::TrueTransferOperator(const
-                                           ParFiniteElementSpace& lFESpace_,
-                                           const ParFiniteElementSpace& hFESpace_)
+
+TrueTransferOperator::TrueTransferOperator(const FiniteElementSpace& lFESpace_,
+                                           const FiniteElementSpace& hFESpace_)
    : Operator(hFESpace_.GetTrueVSize(), lFESpace_.GetTrueVSize()),
      lFESpace(lFESpace_),
      hFESpace(hFESpace_)
 {
    localTransferOperator = new TransferOperator(lFESpace_, hFESpace_);
 
-   tmpL.SetSize(lFESpace_.GetVSize());
-   tmpH.SetSize(hFESpace_.GetVSize());
+   P = lFESpace.GetProlongationMatrix();
+   R = hFESpace.IsVariableOrder() ? hFESpace.GetHpRestrictionMatrix() :
+       hFESpace.GetRestrictionMatrix();
 
-   hFESpace.GetRestrictionMatrix()->BuildTranspose();
+   // P and R can be both null
+   // P can be null and R not null
+   // If P is not null it is assumed that R is not null as well
+   if (P) { MFEM_VERIFY(R, "Both P and R have to be not NULL") }
+
+   if (P)
+   {
+      tmpL.SetSize(lFESpace_.GetVSize());
+      tmpH.SetSize(hFESpace_.GetVSize());
+      R->EnsureMultTranspose();
+   }
+   // P can be null and R not null
+   else if (R)
+   {
+      tmpH.SetSize(hFESpace_.GetVSize());
+      R->EnsureMultTranspose();
+   }
 }
 
 TrueTransferOperator::~TrueTransferOperator()
@@ -1447,17 +1468,40 @@ TrueTransferOperator::~TrueTransferOperator()
 
 void TrueTransferOperator::Mult(const Vector& x, Vector& y) const
 {
-   lFESpace.GetProlongationMatrix()->Mult(x, tmpL);
-   localTransferOperator->Mult(tmpL, tmpH);
-   hFESpace.GetRestrictionMatrix()->Mult(tmpH, y);
+   if (P)
+   {
+      P->Mult(x, tmpL);
+      localTransferOperator->Mult(tmpL, tmpH);
+      R->Mult(tmpH, y);
+   }
+   else if (R)
+   {
+      localTransferOperator->Mult(x, tmpH);
+      R->Mult(tmpH, y);
+   }
+   else
+   {
+      localTransferOperator->Mult(x, y);
+   }
 }
 
 void TrueTransferOperator::MultTranspose(const Vector& x, Vector& y) const
 {
-   hFESpace.GetRestrictionMatrix()->MultTranspose(x, tmpH);
-   localTransferOperator->MultTranspose(tmpH, tmpL);
-   lFESpace.GetProlongationMatrix()->MultTranspose(tmpL, y);
+   if (P)
+   {
+      R->MultTranspose(x, tmpH);
+      localTransferOperator->MultTranspose(tmpH, tmpL);
+      P->MultTranspose(tmpL, y);
+   }
+   else if (R)
+   {
+      R->MultTranspose(x, tmpH);
+      localTransferOperator->MultTranspose(tmpH, y);
+   }
+   else
+   {
+      localTransferOperator->MultTranspose(x, y);
+   }
 }
-#endif
 
 } // namespace mfem

fem/transfer.hpp

@@ -387,6 +387,7 @@ class PRefinementTransferOperator : public Operator
 private:
    const FiniteElementSpace& lFESpace;
    const FiniteElementSpace& hFESpace;
+   bool isvar_order;
 
 public:
    /// @brief Constructs a transfer operator from \p lFESpace to \p hFESpace
@@ -452,14 +453,15 @@ public:
    virtual void MultTranspose(const Vector& x, Vector& y) const override;
 };
 
-#ifdef MFEM_USE_MPI
 /// @brief Matrix-free transfer operator between finite element spaces working
 /// on true degrees of freedom
 class TrueTransferOperator : public Operator
 {
 private:
-   const ParFiniteElementSpace& lFESpace;
-   const ParFiniteElementSpace& hFESpace;
+   const FiniteElementSpace& lFESpace;
+   const FiniteElementSpace& hFESpace;
+   const Operator * P = nullptr;
+   const SparseMatrix * R = nullptr;
    TransferOperator* localTransferOperator;
    mutable Vector tmpL;
    mutable Vector tmpH;
@@ -467,8 +469,8 @@ private:
 public:
    /// @brief Constructs a transfer operator working on true degrees of freedom
    /// from \p lFESpace to \p hFESpace
-   TrueTransferOperator(const ParFiniteElementSpace& lFESpace_,
-                        const ParFiniteElementSpace& hFESpace_);
+   TrueTransferOperator(const FiniteElementSpace& lFESpace_,
+                        const FiniteElementSpace& hFESpace_);
 
    /// Destructor
    ~TrueTransferOperator();
@@ -484,7 +486,6 @@ public:
        the true dof vector \p y corresponding to the coarse space. */
    virtual void MultTranspose(const Vector& x, Vector& y) const override;
 };
-#endif
 
 } // namespace mfem
 

general/binaryio.hpp

@@ -58,16 +58,19 @@ void AppendBytes(std::vector<char> &vec, const T &val)
    vec.insert(vec.end(), ptr, ptr + sizeof(T));
 }
 
-/// Given a buffer @a buf of length @a nbytes, encode the data in base-64
-/// format, and write the encoded data to the output stream @a out.
+/// @brief Given a buffer @a bytes of length @a nbytes, encode the data in
+/// base-64 format, and write the encoded data to the output stream @a out.
 void WriteBase64(std::ostream &out, const void *bytes, size_t nbytes);
 
-/// Decode @a len base-64 encoded characters in the buffer @a src, and store the
-/// resulting decoded data in @a buf. @a buf will be resized as needed.
+/// @brief Decode @a len base-64 encoded characters in the buffer @a src, and
+/// store the resulting decoded data in @a buf. @a buf will be resized as
+/// needed.
 void DecodeBase64(const char *src, size_t len, std::vector<char> &buf);
 
-/// Return the number of characters needed to encode @a nbytes in base-64. This
-/// is equal to 4*nbytes/3, rounded up to the nearest multiple of 4.
+/// @brief Return the number of characters needed to encode @a nbytes in
+/// base-64.
+///
+/// This is equal to 4*nbytes/3, rounded up to the nearest multiple of 4.
 size_t NumBase64Chars(size_t nbytes);
 
 } // namespace mfem::bin_io

general/hash.hpp

@@ -66,6 +66,16 @@ struct Hashed4
  *
  *  All items in the container can also be accessed sequentially using the
  *  provided iterator.
+ *
+ *  Notes:
+ *   The data structure and implementation is based on a BlockArray<T> which
+ *   provides an efficient item storage that avoids heap fragmentation, and
+ *   index-based item access. The hash table implemented on top of the
+ *   BlockArray provides fast associative (key -> value) access by grouping
+ *   items into bins (buckets) of O(1) size.
+ *   - "id" denotes the index of an item in the underlying BlockArray<T>,
+ *   - "idx" denotes the index of a bin, determined by hashing a key with
+ *     the function `Hash`.
  */
 template<typename T>
 class HashTable : public BlockArray<T>
@@ -74,68 +84,212 @@ protected:
    typedef BlockArray<T> Base;
 
 public:
+   /** @brief Main constructor of the HashTable class.
+
+       @param[in] block_size The size of the storage blocks of the underlying
+                             BlockArray<T>.
+       @param[in] init_hash_size The initial size of the hash table. Must be
+                                 a power of 2. */
    HashTable(int block_size = 16*1024, int init_hash_size = 32*1024);
-   HashTable(const HashTable& other); // deep copy
+   /// @brief Deep copy
+   HashTable(const HashTable& other);
    ~HashTable();
 
-   /// Get item whose parents are 'p1', 'p2'... Create it if it doesn't exist.
+   /** @brief Item accessor with key (or parents) the pair 'p1', 'p2'. Default
+       construct an item of type T if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    T* Get(int p1, int p2);
+
+   /** @brief Item accessor with key (or parents) the quadruplet 'p1', 'p2',
+       'p3', 'p4'. The key 'p4' is optional. Default construct an item of type T
+       if no value corresponds to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The index "id" of the key in the BlockArray<T>.
+
+      @warning This method should only be called if T inherits from Hashed4. */
    T* Get(int p1, int p2, int p3, int p4 = -1 /* p4 optional */);
 
    /// Get id of item whose parents are p1, p2... Create it if it doesn't exist.
+   /** @brief Get the "id" of an item, this "id" corresponding to the index of the
+       item in the underlying BlockArray<T> object. Default construct an item
+       and id if no value corresponds to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    int GetId(int p1, int p2);
+
+   /** @brief Get the "id" of an item, this "id" corresponding to the index of the
+       item in the underlying BlockArray<T> object. Default construct an item
+       and id if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed4. */
    int GetId(int p1, int p2, int p3, int p4 = -1);
 
    /// Find item whose parents are p1, p2... Return NULL if it doesn't exist.
+   /** @brief Item accessor with key (or parents) the pair 'p1', 'p2'. Return
+       nullptr if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The item associated to the key (p1,p2).
+
+       @warning This method should only be called if T inherits from Hashed2. */
    T* Find(int p1, int p2);
+
+   /** @brief Item accessor with key (or parents) the quadruplet 'p1', 'p2',
+       'p3', 'p4'. The key 'p4' is optional. Return nullptr if no value
+       correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The item associated to the key (p1,p2,p3,p4).
+
+       @warning This method should only be called if T inherits from Hashed4. */
    T* Find(int p1, int p2, int p3, int p4 = -1);
 
+   /** @brief Item const accessor with key (or parents) the pair 'p1', 'p2'.
+       Return nullptr if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The item associated to the key (p1,p2).
+
+      @warning This method should only be called if T inherits from Hashed2. */
    const T* Find(int p1, int p2) const;
+
+   /** @brief Item const accessor with key (or parents) the quadruplet 'p1',
+       'p2', 'p3', 'p4'. The key 'p4' is optional. Return nullptr if no value
+       correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The item associated to the key (p1,p2,p3,p4).
+
+       @warning This method should only be called if T inherits from Hashed4. */
    const T* Find(int p1, int p2, int p3, int p4 = -1) const;
 
    /// Find id of item whose parents are p1, p2... Return -1 if it doesn't exist.
+   /** @brief Find the "id" of an item, this "id" corresponding to the index of
+       the item in the underlying BlockArray<T> object. Default construct an
+       item and id if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    int FindId(int p1, int p2) const;
+
+   /** @brief Find the "id" of an item, this "id" corresponding to the index of
+       the item in the underlying BlockArray<T> object. Default construct an
+       item and id if no value correspond to the requested key.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @param[in] p4 Fourth part of the key (optional).
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed4. */
    int FindId(int p1, int p2, int p3, int p4 = -1) const;
 
-   /// Return the number of elements currently stored in the HashTable.
+   /// @brief Return the number of elements currently stored in the HashTable.
    int Size() const { return Base::Size() - unused.Size(); }
 
-   /// Return the total number of ids (used and unused) in the HashTable.
+   /// @brief Return the total number of ids (used and unused) in the HashTable.
    int NumIds() const { return Base::Size(); }
 
-   /// Return the number of free/unused ids in the HashTable.
+   /// @brief Return the number of free/unused ids in the HashTable.
    int NumFreeIds() const { return unused.Size(); }
 
-   /// Return true if item 'id' exists in (is used by) the container.
-   /** It is assumed that 0 <= id < NumIds(). */
+   /** @brief Return true if item 'id' exists in (is used by) the container.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+
+       @warning It is assumed that 0 <= id < NumIds(). */
    bool IdExists(int id) const { return (Base::At(id).next != -2); }
 
-   /// Remove an item from the hash table.
-   /** Its id will be reused by newly added items. */
+   /** @brief Remove an item from the hash table.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+
+       @warning Its id will be reused by newly added items. */
    void Delete(int id);
 
-   /// Remove all items.
+   /// @brief Remove all items.
    void DeleteAll();
 
-   /// Allocate an item at 'id'. Enlarge the underlying BlockArray if necessary.
-   /** This is a special purpose method used when loading data from a file.
-       Does nothing if the slot 'id' has already been allocated. */
+   /** @brief Allocate an item at 'id'. Enlarge the underlying BlockArray if
+       necessary.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+
+       @warning This is a special purpose method used when loading data from a
+       file. Does nothing if the slot 'id' has already been allocated. */
    void Alloc(int id, int p1, int p2);
 
-   /// Reinitialize the internal list of unallocated items.
-   /** This is a special purpose method used when loading data from a file. */
+   /** @brief Reinitialize the internal list of unallocated items.
+
+       @warning This is a special purpose method used when loading data from a file. */
    void UpdateUnused();
 
-   /// Make an item hashed under different parent IDs.
+   /** @brief Change the key associated with an item.
+
+       In other words, makes an item hashed under different parent IDs.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] new_p1 First part of the new key.
+       @param[in] new_p2 Second part of the new key.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    void Reparent(int id, int new_p1, int new_p2);
+
+   /** @brief Change the key associated with an item.
+
+       In other words, makes an item hashed under different parent IDs.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] new_p1 First part of the new key.
+       @param[in] new_p2 Second part of the new key.
+       @param[in] new_p3 Third part of the new key.
+       @param[in] new_p4 Fourth part of the new key (optional).
+
+       @warning This method should only be called if T inherits from Hashed4. */
    void Reparent(int id, int new_p1, int new_p2, int new_p3, int new_p4 = -1);
 
-   /// Return total size of allocated memory (tables plus items), in bytes.
+   /// @brief Return total size of allocated memory (tables plus items), in bytes.
    long MemoryUsage() const;
 
-   /// Write details of the memory usage to the mfem output stream.
+   /// @brief Write details of the memory usage to the mfem output stream.
    void PrintMemoryDetail() const;
 
+   /// @brief Print a histogram of bin sizes for debugging purposes.
+   void PrintStats() const;
+
    class iterator : public Base::iterator
    {
    protected:
@@ -183,33 +337,114 @@ public:
    const_iterator cend() const { return const_iterator(); }
 
 protected:
+   /** The hash table: each bin is a linked list of items. For each non-empty
+       bin, this arrays stores the 'id' of the first item in the list, or -1
+       if the bin is empty. */
    int* table;
+
+   /** mask = table_size-1. Used for fast modulo operation in Hash(), to wrap
+       the raw hashed index around the current table size (which must be a power
+       of two). */
    int mask;
+
+   /** List of deleted items in the BlockArray<T>. New items are created with
+       these ids first, before they are appended to the block array. */
    Array<int> unused;
 
-   // hash functions (NOTE: the constants are arbitrary)
-   inline int Hash(int p1, int p2) const
-   { return (984120265*p1 + 125965121*p2) & mask; }
+   /** @brief hash function for Hashed2 items.
 
-   inline int Hash(int p1, int p2, int p3) const
-   { return (984120265*p1 + 125965121*p2 + 495698413*p3) & mask; }
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The hash key "idx" identifying a bin/bucket.
+
+       NOTE: the constants are arbitrary
+       @warning This method should only be called if T inherits from Hashed2. */
+   inline int Hash(size_t p1, size_t p2) const
+   { return (984120265ul*p1 + 125965121ul*p2) & mask; }
+
+   /** @brief hash function for Hashed4 items.
+
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @return The hash key "idx" identifying a bin/bucket.
+
+       NOTE: The constants are arbitrary.
+       NOTE: p4 is not hashed nor stored as p1, p2, p3 identify a face uniquely.
+       @warning This method should only be called if T inherits from Hashed4. */
+   inline int Hash(size_t p1, size_t p2, size_t p3) const
+   { return (984120265ul*p1 + 125965121ul*p2 + 495698413ul*p3) & mask; }
 
    // Delete() and Reparent() use one of these:
+   /// @brief Hash function for items of type T that inherit from Hashed2.
    inline int Hash(const Hashed2& item) const
    { return Hash(item.p1, item.p2); }
 
+   /// @brief Hash function for items of type T that inherit from Hashed4.
    inline int Hash(const Hashed4& item) const
    { return Hash(item.p1, item.p2, item.p3); }
 
+   /** @brief Search the index of the item associated to the key (p1,p2)
+       starting from the item with index @a id.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed2. */
    int SearchList(int id, int p1, int p2) const;
+
+   /** @brief Search the index of the item associated to the key (p1,p2,p3,(p4))
+       starting from the item with index @a id.
+
+       @param[in] id Index of the item in the underlying BlockArray<T>.
+       @param[in] p1 First part of the key.
+       @param[in] p2 Second part of the key.
+       @param[in] p3 Third part of the key.
+       @return The index "id" of the key in the BlockArray<T>.
+
+       @warning This method should only be called if T inherits from Hashed4. */
    int SearchList(int id, int p1, int p2, int p3) const;
 
+   /** @brief Insert the item 'id' into bin 'idx'.
+
+       @param[in] idx The bin/bucket index.
+       @param[in] id The index of the item in the BlockArray<T>.
+       @param[in] item The item to insert at the begining of the linked list.
+
+       @warning The method only works with bin 'idx' and does not check the
+                overall fill factor of the hash table. If appropriate,
+                use CheckRehash() for that. */
    inline void Insert(int idx, int id, T &item);
+
+   /** @brief Unlink an item @a id from the linked list of bin @a idx.
+
+       @param[in] idx The bin/bucket index.
+       @param[in] id The index of the item in the BlockArray<T>.
+
+       @warning The method aborts if the item is not found. */
    void Unlink(int idx, int id);
 
-   /// Check table load factor and resize if necessary
+   /** @brief Check table fill factor and resize if necessary.
+
+       The method checks the average size of the bins (i.e., the fill factor).
+       If the fill factor is > 2, the table is enlarged (see DoRehash()). */
    inline void CheckRehash();
+
+   /** @brief Double the size of the hash table (i.e., double the number of bins)
+       and reinsert all items into the new bins.
+
+       NOTE: Rehashing is computationally expensive (O(N) in the number of items),
+       but since it is only done rarely (when the number of items doubles),
+       the amortized complexity of inserting an item is still O(1). */
    void DoRehash();
+
+   /** @brief Return the size of the bin "idx".
+
+       @param[in] idx The index of the bin.
+       @return The size of the bin. */
+   int BinSize(int idx) const;
 };
 
 
@@ -591,6 +826,7 @@ void HashTable<T>::Alloc(int id, int p1, int p2)
       item.p2 = p2;
 
       Insert(Hash(p1, p2), id, item);
+      CheckRehash();
    }
 }
 
@@ -649,6 +885,48 @@ void HashTable<T>::PrintMemoryDetail() const
              << " + " << unused.MemoryUsage();
 }
 
+template<typename T>
+int HashTable<T>::BinSize(int idx) const
+{
+   int count = 0;
+   int id = table[idx];
+   while (id >= 0)
+   {
+      const T& item = Base::At(id);
+      id = item.next;
+      count++;
+   }
+   return count;
+}
+
+template<typename T>
+void HashTable<T>::PrintStats() const
+{
+   int table_size = mask+1;
+   mfem::out << "Hash table size: " << table_size << "\n";
+   mfem::out << "Item count: " << Size() << "\n";
+   mfem::out << "BlockArray size: " << Base::Size() << "\n";
+
+   const int H = 16;
+   int hist[H];
+
+   for (int i = 0; i < H; i++) { hist[i] = 0; }
+
+   for (int i = 0; i < table_size; i++)
+   {
+      int bs = BinSize(i);
+      if (bs >= H) { bs = H-1; }
+      hist[bs]++;
+   }
+
+   mfem::out << "Bin size histogram:\n";
+   for (int i = 0; i < H; i++)
+   {
+      mfem::out << "  size " << i << ": "
+                << hist[i] << " bins" << std::endl;
+   }
+}
+
 
 template <typename int_type_const_iter>
 HashFunction &HashFunction::EncodeAndHashInts(int_type_const_iter begin,

linalg/densemat.cpp

@@ -513,6 +513,119 @@ double DenseMatrix::Weight() const
    return 0.0;
 }
 
+void DenseMatrix::DetRevDiff(DenseMatrix &A_bar) const
+{
+   MFEM_ASSERT(Height() == Width() && Height() > 0,
+               "The matrix must be square and "
+               << "sized larger than zero to compute the determinant."
+               << "  Height() = " << Height()
+               << ", Width() = " << Width());
+
+   switch (Height())
+   {
+      case 1:
+         // return data[0];
+         A_bar(0,0) = 1.0;
+         return;
+
+      case 2:
+         // return data[0] * data[3] - data[1] * data[2];
+         A_bar(0,0) = data[3]; // data[0]
+         A_bar(1,1) = data[0]; // data[3]
+         A_bar(1,0) = -data[2]; // data[1]
+         A_bar(0,1) = -data[1]; // data[2]
+         return;
+
+      case 3:
+      {
+         const double *d = data;
+         // return
+         //   d[0] * (d[4] * d[8] - d[5] * d[7]) +
+         //   d[3] * (d[2] * d[7] - d[1] * d[8]) +
+         //   d[6] * (d[1] * d[5] - d[2] * d[4]);
+         A_bar(0,0) = d[4]*d[8] - d[5]*d[7]; // d[0]
+         A_bar(1,0) = d[6]*d[5] - d[3]*d[8]; // d[1]
+         A_bar(2,0) = d[3]*d[7] - d[6]*d[4]; // d[2]
+         A_bar(0,1) = d[2]*d[7] - d[1]*d[8]; // d[3]
+         A_bar(1,1) = d[0]*d[8] - d[6]*d[2]; // d[4]
+         A_bar(2,1) = d[6]*d[1] - d[0]*d[7]; // d[5]
+         A_bar(0,2) = d[1]*d[5] - d[2]*d[4]; // d[6]
+         A_bar(1,2) = d[3]*d[2] - d[0]*d[5]; // d[7]
+         A_bar(2,2) = d[0]*d[4] - d[3]*d[1]; // d[8]
+         return;
+
+      }
+      default:
+      {
+         // In the general case we compute the gradient of the determinant
+         // using the relation from Mike Giles document:
+         // "An extended collection of matrix derivative results for forward
+         // and reverse mode algorithmic differentiation"
+         DenseMatrixInverse lu_factors(*this);
+         lu_factors.GetInverseMatrix(A_bar);
+         A_bar.Transpose();
+         A_bar *= lu_factors.Det();
+         return;
+
+      }
+   }
+   // not reachable
+}
+
+void DenseMatrix::WeightRevDiff(DenseMatrix &A_bar) const
+{
+#ifdef MFEM_DEBUG
+   if (Height() != A_bar.Height() || Width() != A_bar.Width())
+   {
+      mfem_error("DenseMatrix::WeightRevDiff()");
+   }
+#endif
+   if (Height() == Width())
+   {
+      // return Det();
+      DetRevDiff(A_bar);
+      return;
+   }
+   else if ((Height() == 2) && (Width() == 1))
+   {
+      // return sqrt(data[0] * data[0] + data[1] * data[1]);
+      double wgt = sqrt(data[0] * data[0] + data[1] * data[1]);
+      A_bar(0,0) = data[0]/wgt;
+      A_bar(1,0) = data[1]/wgt;
+      return;
+   }
+   else if ((Height() == 3) && (Width() == 1))
+   {
+      // return sqrt(data[0] * data[0] + data[1] * data[1] + data[2] * data[2]);
+      double wgt = sqrt(data[0] * data[0] + data[1] * data[1] + data[2] * data[2]);
+      for (int i = 0; i < 3; ++i)
+      {
+         A_bar(i,0) = data[i]/wgt;
+      }
+      return;
+   }
+   else if ((Height() == 3) && (Width() == 2))
+   {
+      const double *d = data;
+      double E = d[0] * d[0] + d[1] * d[1] + d[2] * d[2];
+      double G = d[3] * d[3] + d[4] * d[4] + d[5] * d[5];
+      double F = d[0] * d[3] + d[1] * d[4] + d[2] * d[5];
+      double wgt = sqrt(E * G - F * F);
+      // start reverse sweep
+      double E_bar = 0.5*G/wgt;
+      double G_bar = 0.5*E/wgt;
+      double F_bar = -F/wgt;
+      A_bar(0,0) = F_bar*d[3] + 2.0*E_bar*d[0]; // d[0]
+      A_bar(1,0) = F_bar*d[4] + 2.0*E_bar*d[1]; // d[1]
+      A_bar(2,0) = F_bar*d[5] + 2.0*E_bar*d[2]; // d[2]
+      A_bar(0,1) = F_bar*d[0] + 2.0*G_bar*d[3]; // d[3]
+      A_bar(1,1) = F_bar*d[1] + 2.0*G_bar*d[4]; // d[4]
+      A_bar(2,1) = F_bar*d[2] + 2.0*G_bar*d[5]; // d[5]
+      return;
+   }
+   mfem_error("DenseMatrix::WeightRevDiff()");
+}
+
 void DenseMatrix::Set(double alpha, const double *A)
 {
    const int s = Width()*Height();
@@ -2175,6 +2288,143 @@ void CalcAdjugateTranspose(const DenseMatrix &a, DenseMatrix &adjat)
    }
 }
 
+void CalcAdjugateRevDiff(const DenseMatrix &a, const DenseMatrix &adja_bar,
+                         DenseMatrix &a_bar)
+{
+#ifdef MFEM_DEBUG
+   if (a.Width() > a.Height() || a.Width() < 1 || a.Height() > 3)
+   {
+      mfem_error("CalcAdjugateRevDiff(...)");
+   }
+   if (a.Width() != a_bar.Width() ||
+       a.Height() != a_bar.Height() ||
+       a_bar.Width() != adja_bar.Height() ||
+       a_bar.Height() != adja_bar.Width())
+   {
+      mfem_error("CalcAdjugateRefDiff(...)");
+   }
+#endif
+
+   if (a.Width() < a.Height())
+   {
+      const double *d = a.Data();
+      const double *ad_bar = adja_bar.Data();
+      double *d_bar = a_bar.Data();
+      if (a.Width() == 1)
+      {
+         // N x 1, N = 2,3
+         // ad[0] = d[0];
+         d_bar[0] = ad_bar[0];
+         // ad[1] = d[1];
+         d_bar[1] = ad_bar[1];
+         if (a.Height() == 3)
+         {
+            // ad[2] = d[2];
+            d_bar[2] = ad_bar[2];
+         }
+      }
+      else
+      {
+         // 3 x 2
+         // e, g, and f are needed during the reverse sweep
+         double e, g, f;
+         e = d[0]*d[0] + d[1]*d[1] + d[2]*d[2];
+         g = d[3]*d[3] + d[4]*d[4] + d[5]*d[5];
+         f = d[0]*d[3] + d[1]*d[4] + d[2]*d[5];
+
+         // start reverse sweep
+         a_bar = 0.0; // this zeros out d_bar[]
+         double e_bar = 0.0;
+         double g_bar = 0.0;
+         double f_bar = 0.0;
+         // ad[0] = d[0]*g - d[3]*f;
+         d_bar[0] += g*ad_bar[0];
+         d_bar[3] -= f*ad_bar[0];
+         g_bar += d[0]*ad_bar[0];
+         f_bar -= d[3]*ad_bar[0];
+         // ad[1] = d[3]*e - d[0]*f;
+         d_bar[3] += e*ad_bar[1];
+         d_bar[0] -= f*ad_bar[1];
+         e_bar += d[3]*ad_bar[1];
+         f_bar -= d[0]*ad_bar[1];
+         // ad[2] = d[1]*g - d[4]*f;
+         d_bar[1] += g*ad_bar[2];
+         d_bar[4] -= f*ad_bar[2];
+         g_bar += d[1]*ad_bar[2];
+         f_bar -= d[4]*ad_bar[2];
+         // ad[3] = d[4]*e - d[1]*f;
+         d_bar[4] += e*ad_bar[3];
+         d_bar[1] -= f*ad_bar[3];
+         e_bar += d[4]*ad_bar[3];
+         f_bar -= d[1]*ad_bar[3];
+         // ad[4] = d[2]*g - d[5]*f;
+         d_bar[2] += g*ad_bar[4];
+         d_bar[5] -= f*ad_bar[4];
+         g_bar += d[2]*ad_bar[4];
+         f_bar -= d[5]*ad_bar[4];
+         // ad[5] = d[5]*e - d[2]*f;
+         d_bar[5] += e*ad_bar[5];
+         d_bar[2] -= f*ad_bar[5];
+         e_bar += d[5]*ad_bar[5];
+         f_bar -= d[2]*ad_bar[5];
+
+         // e = d[0]*d[0] + d[1]*d[1] + d[2]*d[2];
+         d_bar[0] += 2.0*d[0]*e_bar;
+         d_bar[1] += 2.0*d[1]*e_bar;
+         d_bar[2] += 2.0*d[2]*e_bar;
+         // g = d[3]*d[3] + d[4]*d[4] + d[5]*d[5];
+         d_bar[3] += 2.0*d[3]*g_bar;
+         d_bar[4] += 2.0*d[4]*g_bar;
+         d_bar[5] += 2.0*d[5]*g_bar;
+         // f = d[0]*d[3] + d[1]*d[4] + d[2]*d[5];
+         d_bar[0] += d[3]*f_bar;
+         d_bar[3] += d[0]*f_bar;
+         d_bar[1] += d[4]*f_bar;
+         d_bar[4] += d[1]*f_bar;
+         d_bar[2] += d[5]*f_bar;
+         d_bar[5] += d[2]*f_bar;
+
+      }
+      return;
+   }
+
+   if (a.Width() == 1)
+   {
+      // adja(0,0) = 1.0;
+      a_bar(0,0) = 0.0;
+   }
+   else if (a.Width() == 2)
+   {
+      // adja(0,0) =  a(1,1);
+      a_bar(1,1) = adja_bar(0,0);
+      // adja(0,1) = -a(0,1);
+      a_bar(0,1) = -adja_bar(0,1);
+      // adja(1,0) = -a(1,0);
+      a_bar(1,0) = -adja_bar(1,0);
+      // adja(1,1) =  a(0,0);
+      a_bar(0,0) = adja_bar(1,1);
+   }
+   else
+   {
+      a_bar = 0.0;
+      for (int di1 = 0; di1 < 3; ++di1)
+      {
+         int it11 = (di1 + 1) % 3;
+         int it12 = (di1 + 2) % 3;
+         for (int di2 = 0; di2 < 3; ++di2)
+         {
+            int it21 = (di2 + 1) % 3;
+            int it22 = (di2 + 2) % 3;
+            // adja(di2,di1) = a(it11,it21)*a(it12,it22) - a(it11,it22)*a(it12,it21);
+            a_bar(it11,it21) += a(it12,it22)*adja_bar(di2,di1);
+            a_bar(it12,it22) += a(it11,it21)*adja_bar(di2,di1);
+            a_bar(it11,it22) -= a(it12,it21)*adja_bar(di2,di1);
+            a_bar(it12,it21) -= a(it11,it22)*adja_bar(di2,di1);
+         }
+      }
+   }
+}
+
 void CalcInverse(const DenseMatrix &a, DenseMatrix &inva)
 {
    MFEM_ASSERT(a.Width() <= a.Height() && a.Width() >= 1 && a.Height() <= 3, "");
@@ -2209,7 +2459,9 @@ void CalcInverse(const DenseMatrix &a, DenseMatrix &inva)
             g = d[3]*d[3] + d[4]*d[4] + d[5]*d[5];
             f = d[0]*d[3] + d[1]*d[4] + d[2]*d[5];
             t = 1.0 / (e*g - f*f);
-            e *= t; g *= t; f *= t;
+            e *= t;
+            g *= t;
+            f *= t;
 
             id[0] = d[0]*g - d[3]*f;
             id[1] = d[3]*e - d[0]*f;
@@ -2281,6 +2533,194 @@ void CalcInverseTranspose(const DenseMatrix &a, DenseMatrix &inva)
    }
 }
 
+void CalcInverseRevDiff(const DenseMatrix &a, const DenseMatrix &inva_bar,
+                        DenseMatrix &a_bar)
+{
+#ifdef MFEM_DEBUG
+   if (a.Width() > a.Height() || a.Width() < 1 || a.Height() > 3)
+   {
+      mfem_error("CalcInverseRevDiff(...)");
+   }
+   if (a.Width() != a_bar.Width() ||
+       a.Height() != a_bar.Height() ||
+       a_bar.Width() != inva_bar.Height() ||
+       a_bar.Height() != inva_bar.Width())
+   {
+      mfem_error("CalcInverseRevDiff(...)");
+   }
+#endif
+
+   if (a.Width() < a.Height())
+   {
+      const double *d = a.Data();
+      const double *id_bar = inva_bar.Data();
+      double *d_bar = a_bar.Data();
+      if (a.Height() == 2)
+      {
+         double t = 1.0 / (d[0]*d[0] + d[1]*d[1]);
+
+         /// id[0] = d[0] * t;
+         d_bar[0] += id_bar[0] * t;
+         double t_bar = id_bar[0] * d[0];
+
+         /// id[1] = d[1] * t;
+         d_bar[1] += id_bar[1] * t;
+         t_bar += id_bar[1] * d[1];
+
+         /// t = 1.0 / (d[0]*d[0] + d[1]*d[1]);
+         d_bar[0] -= t_bar * 2 * d[0] / pow(d[0]*d[0] + d[1]*d[1], 2);
+         d_bar[1] -= t_bar * 2 * d[1] / pow(d[0]*d[0] + d[1]*d[1], 2);
+      }
+      else
+      {
+         if (a.Width() == 1)
+         {
+            double t = 1.0 / (d[0]*d[0] + d[1]*d[1] + d[2]*d[2]);
+
+            /// id[0] = d[0] * t;
+            d_bar[0] += id_bar[0] * t;
+            double t_bar = id_bar[0] * d[0];
+
+            /// id[1] = d[1] * t;
+            d_bar[1] += id_bar[1] * t;
+            t_bar += id_bar[1] * d[1];
+
+            /// id[2] = d[2] * t;
+            d_bar[2] += id_bar[2] * t;
+            t_bar += id_bar[2] * d[2];
+
+            /// t = 1.0 / (d[0]*d[0] + d[1]*d[1] + d[2]*d[2]);
+            d_bar[0] -= t_bar * 2 * d[0] / pow(d[0]*d[0] + d[1]*d[1] + d[2]*d[2], 2);
+            d_bar[1] -= t_bar * 2 * d[1] / pow(d[0]*d[0] + d[1]*d[1] + d[2]*d[2], 2);
+            d_bar[2] -= t_bar * 2 * d[2] / pow(d[0]*d[0] + d[1]*d[1] + d[2]*d[2], 2);
+         }
+         else
+         {
+            double e = d[0]*d[0] + d[1]*d[1] + d[2]*d[2];
+            double g = d[3]*d[3] + d[4]*d[4] + d[5]*d[5];
+            double f = d[0]*d[3] + d[1]*d[4] + d[2]*d[5];
+            double t = 1.0 / (e*g - f*f);
+
+            double ee = e * t;
+            double gg = g * t;
+            double ff = f * t;
+
+            /// id[0] = d[0]*g - d[3]*f;
+            d_bar[0] += id_bar[0]*gg;
+            double gg_bar = id_bar[0] * d[0];
+            d_bar[3] += -id_bar[0] * ff;
+            double ff_bar = -id_bar[0] * d[3];
+
+            /// id[1] = d[3]*e - d[0]*f;
+            d_bar[3] += id_bar[1] * ee;
+            double ee_bar = id_bar[1] * d[3];
+            d_bar[0] += -id_bar[1] * ff;
+            ff_bar -= id_bar[1] * d[0];
+
+            /// id[2] = d[1]*g - d[4]*f;
+            d_bar[1] += id_bar[2] * gg;
+            gg_bar += id_bar[2] * d[1];
+            d_bar[4] += -id_bar[2] * ff;
+            ff_bar -= id_bar[2] * d[4];
+
+            /// id[3] = d[4]*e - d[1]*f;
+            d_bar[4] += id_bar[3] * ee;
+            ee_bar += id_bar[3] * d[4];
+            d_bar[1] += -id_bar[3] * ff;
+            ff_bar -= id_bar[3] * d[1];
+
+            /// id[4] = d[2]*g - d[5]*f;
+            d_bar[2] += id_bar[4] * gg;
+            gg_bar += id_bar[4] * d[2];
+            d_bar[5] += -id_bar[4] * ff;
+            ff_bar -= id_bar[4] * d[5];
+
+            /// id[5] = d[5]*e - d[2]*f;
+            d_bar[5] += id_bar[5] * ee;
+            ee_bar += id_bar[5] * d[5];
+            d_bar[2] += -id_bar[5] * ff;
+            ff_bar -= id_bar[5] * d[2];
+
+
+            /// double ff = f * t;
+            double t_bar = ff_bar * f;
+            double f_bar = ff_bar * t;
+
+            /// double gg = g * t;
+            t_bar += gg_bar * g;
+            double g_bar = gg_bar * t;
+
+            /// double ee = e * t;
+            t_bar += ee_bar * e;
+            double e_bar = ee_bar * t;
+
+            // /// f *= t;
+            // double t_bar = f_bar * f / t;
+            // f_bar *= t;
+
+            // /// g *= t;
+            // t_bar += g_bar * g / t;
+            // g_bar *= t;
+
+            // /// e *= t;
+            // t_bar += e_bar * e / t;
+            // e_bar *= t;
+
+            /// double t = 1.0 / (e*g - f*f);
+            e_bar -= t_bar * g / pow(e*g - f*f, 2);
+            g_bar -= t_bar * e / pow(e*g - f*f, 2);
+            f_bar += t_bar * 2*f / pow(e*g - f*f, 2);
+
+            /// double f = d[0]*d[3] + d[1]*d[4] + d[2]*d[5];
+            d_bar[0] += f_bar * d[3];
+            d_bar[3] += f_bar * d[0];
+            d_bar[1] += f_bar * d[4];
+            d_bar[4] += f_bar * d[1];
+            d_bar[2] += f_bar * d[5];
+            d_bar[5] += f_bar * d[2];
+
+            /// double g = d[3]*d[3] + d[4]*d[4] + d[5]*d[5];
+            d_bar[3] += g_bar * 2 * d[3];
+            d_bar[4] += g_bar * 2 * d[4];
+            d_bar[5] += g_bar * 2 * d[5];
+
+            /// double e = d[0]*d[0] + d[1]*d[1] + d[2]*d[2];
+            d_bar[0] += e_bar * 2 * d[0];
+            d_bar[1] += e_bar * 2 * d[1];
+            d_bar[2] += e_bar * 2 * d[2];
+         }
+      }
+      return;
+   }
+
+#ifdef MFEM_DEBUG
+   double t = a.Det();
+   MFEM_ASSERT(std::abs(t) > 1.0e-14 * pow(a.FNorm()/a.Width(), a.Width()),
+               "singular matrix!");
+#endif
+
+   double inva_buffer[9] = {};
+   DenseMatrix inva(inva_buffer, a.Height(), a.Width());
+
+   switch (a.Height())
+   {
+      case 1:
+         inva(0,0) = 1.0 / a.Det();
+         break;
+      case 2:
+         kernels::CalcInverse<2>(a.Data(), inva.Data());
+         break;
+      case 3:
+         kernels::CalcInverse<3>(a.Data(), inva.Data());
+         break;
+   }
+
+   double tmp_buffer[9] = {};
+   DenseMatrix tmp(tmp_buffer, a.Height(), a.Width());
+   MultAtB(inva, inva_bar, tmp);
+   AddMult_a_ABt(-1, tmp, inva, a_bar);
+}
+
 void CalcOrtho(const DenseMatrix &J, Vector &n)
 {
    MFEM_ASSERT( ((J.Height() == 2 && J.Width() == 1)
@@ -2307,6 +2747,50 @@ void CalcOrtho(const DenseMatrix &J, Vector &n)
    }
 }
 
+void CalcOrthoRevDiff(const DenseMatrix &J, const Vector &n_bar,
+                      DenseMatrix &J_bar)
+{
+   MFEM_ASSERT(((J.Height() == 2 && J.Width() == 1) ||
+                (J.Height() == 3 && J.Width() == 2)) &&
+               (J.Height() == n_bar.Size()),
+               "Matrix must be 3x2 or 2x1, "
+               << "and the Vector must be sized with the rows. "
+               << " J.Height() = " << J.Height()
+               << ", J.Width() = " << J.Width()
+               << ", n_bar.Size() = " << n_bar.Size());
+   MFEM_ASSERT((J.Height() == J_bar.Height() && J.Width() == J_bar.Width()),
+               "Input matrix and derivative matrix must be the same size.");
+
+   const double *d = J.Data();
+   double *d_bar = J_bar.Data();
+   if (J.Height() == 2)
+   {
+      // n(0) =  d[1];
+      d_bar[1] = n_bar(0);
+      // n(1) = -d[0];
+      d_bar[0] = -n_bar(1);
+   }
+   else
+   {
+      J_bar = 0.0;
+      // n(0) = d[1]*d[5] - d[2]*d[4];
+      d_bar[1] += d[5]*n_bar(0);
+      d_bar[5] += d[1]*n_bar(0);
+      d_bar[2] -= d[4]*n_bar(0);
+      d_bar[4] -= d[2]*n_bar(0);
+      // n(1) = d[2]*d[3] - d[0]*d[5];
+      d_bar[2] += d[3]*n_bar(1);
+      d_bar[3] += d[2]*n_bar(1);
+      d_bar[0] -= d[5]*n_bar(1);
+      d_bar[5] -= d[0]*n_bar(1);
+      // n(2) = d[0]*d[4] - d[1]*d[3];
+      d_bar[0] += d[4]*n_bar(2);
+      d_bar[4] += d[0]*n_bar(2);
+      d_bar[1] -= d[3]*n_bar(2);
+      d_bar[3] -= d[1]*n_bar(2);
+   }
+}
+
 void MultAAt(const DenseMatrix &a, DenseMatrix &aat)
 {
    const int height = a.Height();

linalg/densemat.hpp

@@ -197,6 +197,12 @@ public:
 
    double Weight() const;
 
+   /// Evaluate the derivative of Det() w.r.t. the matrix entries
+   void DetRevDiff(DenseMatrix &A_bar) const;
+
+   /// Evaluate the derivative of Weight() w.r.t. the matrix entries
+   void WeightRevDiff(DenseMatrix &A_bar) const;
+
    /** @brief Set the matrix to alpha * A, assuming that A has the same
        dimensions as the matrix and uses column-major layout. */
    void Set(double alpha, const double *A);
@@ -457,6 +463,10 @@ void CalcAdjugate(const DenseMatrix &a, DenseMatrix &adja);
 /// Calculate the transposed adjugate of a matrix (for NxN matrices, N=1,2,3)
 void CalcAdjugateTranspose(const DenseMatrix &a, DenseMatrix &adjat);
 
+/// Reverse-mode sensitivities of adj(A) with respect to the entries in A
+void CalcAdjugateRevDiff(const DenseMatrix &a, const DenseMatrix &adja_bar,
+                         DenseMatrix &a_bar);
+
 /** Calculate the inverse of a matrix (for NxN matrices, N=1,2,3) or the
     left inverse (A^t.A)^{-1}.A^t (for 2x1, 3x1, or 3x2 matrices) */
 void CalcInverse(const DenseMatrix &a, DenseMatrix &inva);
@@ -464,11 +474,19 @@ void CalcInverse(const DenseMatrix &a, DenseMatrix &inva);
 /// Calculate the inverse transpose of a matrix (for NxN matrices, N=1,2,3)
 void CalcInverseTranspose(const DenseMatrix &a, DenseMatrix &inva);
 
+/// Reverse-mode sensitivities of inv(A) with respect to the entries in A
+void CalcInverseRevDiff(const DenseMatrix &a, const DenseMatrix &inva_bar,
+                        DenseMatrix &a_bar);
+
 /** For a given Nx(N-1) (N=2,3) matrix J, compute a vector n such that
     n_k = (-1)^{k+1} det(J_k), k=1,..,N, where J_k is the matrix J with the
     k-th row removed. Note: J^t.n = 0, det([n|J])=|n|^2=det(J^t.J). */
 void CalcOrtho(const DenseMatrix &J, Vector &n);
 
+/// The reverse-mode differentiation of CalcOrtho()
+void CalcOrthoRevDiff(const DenseMatrix &J, const Vector &n_bar,
+                      DenseMatrix &J_bar);
+
 /// Calculate the matrix A.At
 void MultAAt(const DenseMatrix &a, DenseMatrix &aat);
 

linalg/petsc.cpp

@@ -3399,6 +3399,7 @@ void PetscBDDCSolver::BDDCSolverConstructor(const PetscBDDCSolverParams &opts)
                                                              hvec_coords->Size(),false);
 
          // likely elasticity -> we attach rigid-body modes as near-null space information to the local matrices
+         // and to the global matrix
          if (vdim == sdim)
          {
             MatNullSpace nnsp;
@@ -3413,7 +3414,15 @@ void PetscBDDCSolver::BDDCSolverConstructor(const PetscBDDCSolverParams &opts)
             ierr = VecCreateMPIWithArray(comm,sdim,hvec_coords->Size(),
                                          hvec_coords->GlobalSize(),data_coords,&pvec_coords);
             CCHKERRQ(comm,ierr);
-            ierr = MatISGetLocalMat(pA,&lA); CCHKERRQ(PETSC_COMM_SELF,ierr);
+            ierr = MatGetNearNullSpace(pA,&nnsp); CCHKERRQ(comm,ierr);
+            if (!nnsp)
+            {
+               ierr = MatNullSpaceCreateRigidBody(pvec_coords,&nnsp);
+               CCHKERRQ(comm,ierr);
+               ierr = MatSetNearNullSpace(pA,nnsp); CCHKERRQ(comm,ierr);
+               ierr = MatNullSpaceDestroy(&nnsp); CCHKERRQ(comm,ierr);
+            }
+            ierr = MatISGetLocalMat(pA,&lA); CCHKERRQ(comm,ierr);
             ierr = MatCreateVecs(lA,&lvec_coords,NULL); CCHKERRQ(PETSC_COMM_SELF,ierr);
             ierr = VecSetBlockSize(lvec_coords,sdim); CCHKERRQ(PETSC_COMM_SELF,ierr);
             ierr = MatGetLocalToGlobalMapping(pA,&l2g,NULL); CCHKERRQ(comm,ierr);

linalg/sparsemat.cpp

@@ -848,8 +848,9 @@ void SparseMatrix::AddMultTranspose(const Vector &x, Vector &y,
    }
    else
    {
-      MFEM_VERIFY(Device::IsDisabled(), "transpose action on device is not "
-                  "enabled; see BuildTranspose() for details.");
+      MFEM_VERIFY(!Device::Allows(~Backend::CPU_MASK), "transpose action with "
+                  "this backend is not enabled; see EnsureMultTranspose() for "
+                  "details.");
       for (int i = 0; i < height; i++)
       {
          const double xi = a * x[i];
@@ -877,6 +878,14 @@ void SparseMatrix::ResetTranspose() const
    At = NULL;
 }
 
+void SparseMatrix::EnsureMultTranspose() const
+{
+   if (Device::Allows(~Backend::CPU_MASK))
+   {
+      BuildTranspose();
+   }
+}
+
 void SparseMatrix::PartMult(
    const Array<int> &rows, const Vector &x, Vector &y) const
 {
@@ -1054,8 +1063,9 @@ void SparseMatrix::AbsMultTranspose(const Vector &x, Vector &y) const
    }
    else
    {
-      MFEM_VERIFY(Device::IsDisabled(), "transpose action on device is not "
-                  "enabled; see BuildTranspose() for details.");
+      MFEM_VERIFY(!Device::Allows(~Backend::CPU_MASK), "transpose action with "
+                  "this backend is not enabled; see EnsureMultTranspose() for "
+                  "details.");
       for (int i = 0; i < height; i++)
       {
          const double xi = x[i];

linalg/sparsemat.hpp

@@ -346,30 +346,45 @@ public:
                          const double a = 1.0) const;
 
    /** @brief Build and store internally the transpose of this matrix which will
-       be used in the methods AddMultTranspose() and MultTranspose(). */
+       be used in the methods AddMultTranspose(), MultTranspose(), and
+       AbsMultTranspose(). */
    /** If this method has been called, the internal transpose matrix will be
        used to perform the action of the transpose matrix in AddMultTranspose(),
-       and MultTranspose().
+       MultTranspose(), and AbsMultTranspose().
 
        Warning: any changes in this matrix will invalidate the internal
        transpose. To rebuild the transpose, call ResetTranspose() followed by a
        call to this method. If the internal transpose is already built, this
        method has no effect.
 
-       When any non-default backend is enabled, i.e. Device::IsEnabled() is
-       true, the methods AddMultTranspose(), and MultTranspose(), require the
-       internal transpose to be built. If that is not the case (i.e. the
-       internal transpose is not built), these methods will raise an error with
-       an appropriate message pointing to this method. When using the default
-       backend, calling this method is optional.
+       When any non-serial-CPU backend is enabled, i.e. the call
+       Device::Allows(~ Backend::CPU_MASK) returns true, the above methods
+       require the internal transpose to be built. If that is not the case (i.e.
+       the internal transpose is not built), these methods will raise an error
+       with an appropriate message pointing to EnsureMultTranspose(). When using
+       any backend from Backend::CPU_MASK, calling this method is optional.
 
-       This method can only be used when the sparse matrix is finalized. */
+       This method can only be used when the sparse matrix is finalized.
+
+       @sa EnsureMultTranspose(), ResetTranspose(). */
    void BuildTranspose() const;
 
    /** Reset (destroy) the internal transpose matrix. See BuildTranspose() for
        more details. */
    void ResetTranspose() const;
 
+   /** @brief Ensures that the matrix is capable of performing MultTranspose(),
+       AddMultTranspose(), and AbsMultTranspose(). */
+   /** For non-serial-CPU backends (e.g. GPU, OpenMP), multiplying by the
+       transpose requires that the internal transpose matrix be already built.
+       When such a backend is enabled, this function will build the internal
+       transpose matrix, see BuildTranspose().
+
+       For the serial CPU backends, the internal transpose is not required, and
+       this function is a no-op. This allows for significant memory savings
+       when the internal transpose matrix is not required. */
+   void EnsureMultTranspose() const;
+
    void PartMult(const Array<int> &rows, const Vector &x, Vector &y) const;
    void PartAddMult(const Array<int> &rows, const Vector &x, Vector &y,
                     const double a=1.0) const;

mesh/mesh.cpp

@@ -1096,7 +1096,7 @@ FaceElementTransformations *Mesh::GetBdrFaceTransformations(int BdrElemNo)
    FaceElementTransformations *tr;
    int fn = GetBdrFace(BdrElemNo);
 
-   // Check if the face is interior, shared, or non-conforming.
+   // Check if the face is interior, shared, or nonconforming.
    if (FaceIsTrueInterior(fn) || faces_info[fn].NCFace >= 0)
    {
       return NULL;
@@ -1127,6 +1127,269 @@ int Mesh::GetBdrFace(int BdrElemNo) const
    return fn;
 }
 
+Mesh::FaceInformation Mesh::GetFaceInformation(int f) const
+{
+   FaceInformation face;
+   int e1, e2;
+   int inf1, inf2;
+   int ncface;
+   GetFaceElements(f, &e1, &e2);
+   GetFaceInfos(f, &inf1, &inf2, &ncface);
+   face.element[0].index = e1;
+   face.element[0].location = ElementLocation::Local;
+   face.element[0].orientation = inf1%64;
+   face.element[0].local_face_id = inf1/64;
+   face.element[1].local_face_id = inf2/64;
+   face.ncface = ncface;
+   face.point_matrix = nullptr;
+   // The following figures out face.location, face.conformity,
+   // face.element[1].index, and face.element[1].orientation.
+   if (f < GetNumFaces()) // Non-ghost face
+   {
+      if (e2>=0)
+      {
+         if (ncface==-1)
+         {
+            face.tag = FaceInfoTag::LocalConforming;
+            face.topology = FaceTopology::Conforming;
+            face.element[1].location = ElementLocation::Local;
+            face.element[0].conformity = ElementConformity::Coincident;
+            face.element[1].conformity = ElementConformity::Coincident;
+            face.element[1].index = e2;
+            face.element[1].orientation = inf2%64;
+         }
+         else // ncface >= 0
+         {
+            face.tag = FaceInfoTag::LocalSlaveNonconforming;
+            face.topology = FaceTopology::Nonconforming;
+            face.element[1].location = ElementLocation::Local;
+            face.element[0].conformity = ElementConformity::Coincident;
+            face.element[1].conformity = ElementConformity::Superset;
+            face.element[1].index = e2;
+            MFEM_ASSERT(inf2%64==0, "unexpected slave face orientation.");
+            face.element[1].orientation = inf2%64;
+            face.point_matrix = nc_faces_info[ncface].PointMatrix;
+         }
+      }
+      else // e2<0
+      {
+         if (ncface==-1)
+         {
+            if (inf2<0)
+            {
+               face.tag = FaceInfoTag::Boundary;
+               face.topology = FaceTopology::Boundary;
+               face.element[1].location = ElementLocation::NA;
+               face.element[0].conformity = ElementConformity::Coincident;
+               face.element[1].conformity = ElementConformity::NA;
+               face.element[1].index = -1;
+               face.element[1].orientation = -1;
+            }
+            else // inf2 >= 0
+            {
+               face.tag = FaceInfoTag::SharedConforming;
+               face.topology = FaceTopology::Conforming;
+               face.element[0].conformity = ElementConformity::Coincident;
+               face.element[1].conformity = ElementConformity::Coincident;
+               face.element[1].location = ElementLocation::FaceNbr;
+               face.element[1].index = -1 - e2;
+               face.element[1].orientation = inf2%64;
+            }
+         }
+         else // ncface >= 0
+         {
+            if (inf2 < 0)
+            {
+               face.tag = FaceInfoTag::MasterNonconforming;
+               face.topology = FaceTopology::Nonconforming;
+               face.element[1].location = ElementLocation::NA;
+               face.element[0].conformity = ElementConformity::Coincident;
+               face.element[1].conformity = ElementConformity::Subset;
+               face.element[1].index = -1;
+               face.element[1].orientation = -1;
+            }
+            else
+            {
+               face.tag = FaceInfoTag::SharedSlaveNonconforming;
+               face.topology = FaceTopology::Nonconforming;
+               face.element[1].location = ElementLocation::FaceNbr;
+               face.element[0].conformity = ElementConformity::Coincident;
+               face.element[1].conformity = ElementConformity::Superset;
+               face.element[1].index = -1 - e2;
+               face.element[1].orientation = inf2%64;
+            }
+            face.point_matrix = nc_faces_info[ncface].PointMatrix;
+         }
+      }
+   }
+   else // Ghost face
+   {
+      if (e1==-1)
+      {
+         face.tag = FaceInfoTag::GhostMaster;
+         face.topology = FaceTopology::NA;
+         face.element[1].location = ElementLocation::NA;
+         face.element[0].conformity = ElementConformity::NA;
+         face.element[1].conformity = ElementConformity::NA;
+         face.element[1].index = -1;
+         face.element[1].orientation = -1;
+      }
+      else
+      {
+         face.tag = FaceInfoTag::GhostSlave;
+         face.topology = FaceTopology::Nonconforming;
+         face.element[1].location = ElementLocation::FaceNbr;
+         face.element[0].conformity = ElementConformity::Superset;
+         face.element[1].conformity = ElementConformity::Coincident;
+         face.element[1].index = -1 - e2;
+         face.element[1].orientation = inf2%64;
+         face.point_matrix = nc_faces_info[ncface].PointMatrix;
+      }
+   }
+   return face;
+}
+
+Mesh::FaceInformation::operator Mesh::FaceInfo() const
+{
+   FaceInfo res {-1, -1, -1, -1, -1};
+   switch (tag)
+   {
+      case FaceInfoTag::LocalConforming:
+         res.Elem1No = element[0].index;
+         res.Elem2No = element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         res.NCFace = ncface;
+         break;
+      case FaceInfoTag::LocalSlaveNonconforming:
+         res.Elem1No = element[0].index;
+         res.Elem2No = element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         res.NCFace = ncface;
+         break;
+      case FaceInfoTag::Boundary:
+         res.Elem1No = element[0].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         break;
+      case FaceInfoTag::SharedConforming:
+         res.Elem1No = element[0].index;
+         res.Elem2No = -1 - element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         break;
+      case FaceInfoTag::MasterNonconforming:
+         res.Elem1No = element[0].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         break;
+      case FaceInfoTag::SharedSlaveNonconforming:
+         res.Elem1No = element[0].index;
+         res.Elem2No = -1 - element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         break;
+      case FaceInfoTag::GhostMaster:
+         break;
+      case FaceInfoTag::GhostSlave:
+         res.Elem1No = element[0].index;
+         res.Elem2No = -1 - element[1].index;
+         res.Elem1Inf = element[0].orientation + element[0].local_face_id*64;
+         res.Elem2Inf = element[1].orientation + element[1].local_face_id*64;
+         break;
+   }
+   return res;
+}
+
+std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info)
+{
+   os << "face topology=";
+   switch (info.topology)
+   {
+      case Mesh::FaceTopology::Boundary:
+         os << "Boundary";
+         break;
+      case Mesh::FaceTopology::Conforming:
+         os << "Conforming";
+         break;
+      case Mesh::FaceTopology::Nonconforming:
+         os << "Non-conforming";
+         break;
+      case Mesh::FaceTopology::NA:
+         os << "NA";
+         break;
+   }
+   os << "element[0].location=";
+   switch (info.element[0].location)
+   {
+      case Mesh::ElementLocation::Local:
+         os << "Local";
+         break;
+      case Mesh::ElementLocation::FaceNbr:
+         os << "FaceNbr";
+         break;
+      case Mesh::ElementLocation::NA:
+         os << "NA";
+         break;
+   }
+   os << std::endl;
+   os << "element[1].location=";
+   switch (info.element[1].location)
+   {
+      case Mesh::ElementLocation::Local:
+         os << "Local";
+         break;
+      case Mesh::ElementLocation::FaceNbr:
+         os << "FaceNbr";
+         break;
+      case Mesh::ElementLocation::NA:
+         os << "NA";
+         break;
+   }
+   os << std::endl;
+   os << "element[0].conformity=";
+   switch (info.element[0].conformity)
+   {
+      case Mesh::ElementConformity::Coincident:
+         os << "Coincident";
+         break;
+      case Mesh::ElementConformity::Superset:
+         os << "Superset";
+         break;
+      case Mesh::ElementConformity::Subset:
+         os << "Subset";
+         break;
+      case Mesh::ElementConformity::NA:
+         os << "NA";
+         break;
+   }
+   os << std::endl;
+   os << "element[1].conformity=";
+   switch (info.element[1].conformity)
+   {
+      case Mesh::ElementConformity::Coincident:
+         os << "Coincident";
+         break;
+      case Mesh::ElementConformity::Superset:
+         os << "Superset";
+         break;
+      case Mesh::ElementConformity::Subset:
+         os << "Subset";
+         break;
+      case Mesh::ElementConformity::NA:
+         os << "NA";
+         break;
+   }
+   os << std::endl;
+   os << "element[0].index=" << info.element[0].index << std::endl
+      << "element[1].index=" << info.element[1].index << std::endl
+      << "element[0].local_face_id=" << info.element[0].local_face_id << std::endl
+      << "element[1].local_face_id=" << info.element[1].local_face_id << std::endl
+      << "element[0].orientation=" << info.element[0].orientation << std::endl
+      << "element[1].orientation=" << info.element[1].orientation << std::endl
+      << "ncface=" << info.ncface << std::endl;
+   return os;
+}
+
 void Mesh::GetFaceElements(int Face, int *Elem1, int *Elem2) const
 {
    *Elem1 = faces_info[Face].Elem1No;
@@ -5090,26 +5353,32 @@ int Mesh::GetNumFaces() const
    return 0;
 }
 
-static int CountFacesByType(const Mesh &mesh, const FaceType type)
+int Mesh::GetNumFacesWithGhost() const
 {
-   int e1, e2;
-   int inf1, inf2;
-   int nf = 0;
-   for (int f = 0; f < mesh.GetNumFaces(); ++f)
-   {
-      mesh.GetFaceElements(f, &e1, &e2);
-      mesh.GetFaceInfos(f, &inf1, &inf2);
-      if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-          (type==FaceType::Boundary && e2<0 && inf2<0) ) { nf++; }
-   }
-   return nf;
+   return faces_info.Size();
 }
 
 int Mesh::GetNFbyType(FaceType type) const
 {
    const bool isInt = type==FaceType::Interior;
    int &nf = isInt ? nbInteriorFaces : nbBoundaryFaces;
-   if (nf<0) { nf = CountFacesByType(*this, type); }
+   if (nf<0)
+   {
+      nf = 0;
+      for (int f = 0; f < GetNumFacesWithGhost(); ++f)
+      {
+         FaceInformation face = GetFaceInformation(f);
+         if ( face.IsOfFaceType(type) )
+         {
+            if (face.IsNonconformingCoarse())
+            {
+               // We don't count nonconforming coarse faces.
+               continue;
+            }
+            nf++;
+         }
+      }
+   }
    return nf;
 }
 
@@ -10152,61 +10421,6 @@ void Mesh::PrintBdrVTU(std::string fname,
    PrintVTU(fname, format, high_order_output, compression_level, true);
 }
 
-template <typename T>
-void WriteBinaryOrASCII(std::ostream &out, std::vector<char> &buf, const T &val,
-                        const char *suffix, VTKFormat format)
-{
-   if (format == VTKFormat::ASCII) { out << val << suffix; }
-   else { bin_io::AppendBytes(buf, val); }
-}
-
-// Ensure ASCII output of uint8_t to stream is integer rather than character
-template <>
-void WriteBinaryOrASCII<uint8_t>(std::ostream &out, std::vector<char> &buf,
-                                 const uint8_t &val, const char *suffix,
-                                 VTKFormat format)
-{
-   if (format == VTKFormat::ASCII) { out << static_cast<int>(val) << suffix; }
-   else { bin_io::AppendBytes(buf, val); }
-}
-
-template <>
-void WriteBinaryOrASCII<double>(std::ostream &out, std::vector<char> &buf,
-                                const double &val, const char *suffix,
-                                VTKFormat format)
-{
-   if (format == VTKFormat::BINARY32)
-   {
-      bin_io::AppendBytes<float>(buf, float(val));
-   }
-   else if (format == VTKFormat::BINARY)
-   {
-      bin_io::AppendBytes(buf, val);
-   }
-   else
-   {
-      out << val << suffix;
-   }
-}
-
-template <>
-void WriteBinaryOrASCII<float>(std::ostream &out, std::vector<char> &buf,
-                               const float &val, const char *suffix,
-                               VTKFormat format)
-{
-   if (format == VTKFormat::BINARY) { bin_io::AppendBytes<double>(buf, val); }
-   else if (format == VTKFormat::BINARY32) { bin_io::AppendBytes(buf, val); }
-   else { out << val << suffix; }
-}
-
-void WriteBase64WithSizeAndClear(std::ostream &out, std::vector<char> &buf,
-                                 int compression_level)
-{
-   WriteVTKEncodedCompressed(out, buf.data(), buf.size(), compression_level);
-   out << '\n';
-   buf.clear();
-}
-
 void Mesh::PrintVTU(std::ostream &out, int ref, VTKFormat format,
                     bool high_order_output, int compression_level,
                     bool bdr_elements)

mesh/mesh.hpp

@@ -91,6 +91,65 @@ protected:
    Array<Element *> boundary;
    Array<Element *> faces;
 
+   /** @brief This structure stores the low level information necessary to
+       interpret the configuration of elements on a specific face. This
+       information can be accessed using methods like GetFaceElements(),
+       GetFaceInfos(), FaceIsInterior(), etc.
+
+       For accessing higher level deciphered information look at
+       Mesh::FaceInformation, and its accessor Mesh::GetFaceInformation().
+
+       Each face contains information on the indices, local reference faces,
+       orientations, and potential nonconformity for the two neighboring
+       elements on a face.
+       Each face can either be an interior, boundary, or shared interior face.
+       Each interior face is shared by two elements referred as Elem1 and Elem2.
+       For boundary faces only the information on Elem1 is relevant.
+       Shared interior faces correspond to faces where Elem1 and Elem2 are
+       distributed on different MPI ranks.
+       Regarding conformity, three cases are distinguished, conforming faces,
+       nonconforming slave faces, and nonconforming master faces. Master and
+       slave referring to the coarse and fine elements respectively on a
+       nonconforming face.
+       Nonconforming slave faces always have the slave element as Elem1 and
+       the master element as Elem2. On the other side, nonconforming master
+       faces always have the master element as Elem1, and one of the slave
+       element as Elem2. Except for ghost nonconforming slave faces, where
+       Elem1 is the master side and Elem2 is the slave side.
+
+       The indices of Elem1 and Elem2 can be indirectly extracted from
+       FaceInfo::Elem1No and FaceInfo::Elem2No, read the note below for special
+       cases on the index of Elem2.
+
+       The local face identifiers are deciphered from FaceInfo::Elem1Inf and
+       FaceInfo::Elem2Inf through the formula: LocalFaceIndex = ElemInf/64,
+       the semantic of the computed local face identifier can be found in
+       fem/geom.cpp. The local face identifier corresponds to an index
+       in the Constants<Geometry>::Edges arrays for 2D element geometries, and
+       to an index in the Constants<Geometry>::FaceVert arrays for 3D element
+       geometries.
+
+       The orientation of each element relative to a face is obtained through
+       the formula: Orientation = ElemInf%64, the semantic of the orientation
+       can also be found in fem/geom.cpp. The orientation corresponds to
+       an index in the Constants<Geometry>::Orient arrays, providing the
+       sequence of vertices identifying the orientation of an edge/face. By
+       convention the orientation of Elem1 is always set to 0, serving as the
+       reference orientation. The orientation of Elem2 relatively to Elem1 is
+       therefore determined just by using the orientation of Elem2. An important
+       special case is the one of nonconforming faces, the orientation should
+       be composed with the PointMatrix, which also contains orientation
+       information. A special treatment should be done for 2D, the orientation
+       in the PointMatrix is not included, therefore when applying the
+       PointMatrix transformation, the PointMatrix should be flipped, except for
+       shared nonconforming slave faces where the transformation can be applied
+       as is.
+
+       Another special case is the case of shared nonconforming faces. Ghost
+       faces use a different design based on so called "ghost" faces.
+       Ghost faces, as their name suggest are very well hidden, and they
+       usually have a separate interface from "standard" faces.
+   */
    struct FaceInfo
    {
       // Inf = 64 * LocalFaceIndex + FaceOrientation
@@ -104,12 +163,12 @@ protected:
    //
    // A local face is one generated from a local element and has index i in
    // faces_info such that i < GetNumFaces(). Also, Elem1No always refers to the
-   // element (slave or master, in the non-conforming case) that generated the
+   // element (slave or master, in the nonconforming case) that generated the
    // face.
    // Classification of a local (non-ghost) face based on its FaceInfo:
    // - Elem2No >= 0 --> local interior face; can be either:
    //    - NCFace == -1 --> conforming face, or
-   //    - NCFace >= 0 --> non-conforming slave face; Elem2No is the index of
+   //    - NCFace >= 0 --> nonconforming slave face; Elem2No is the index of
    //      the master volume element; Elem2Inf%64 is 0, see the note in
    //      Mesh::GenerateNCFaceInfo().
    // - Elem2No < 0 --> local "boundary" face; can be one of:
@@ -118,14 +177,14 @@ protected:
    //       - Elem2Inf >= 0 --> shared face where element 2 is a face-neighbor
    //         element with index -1-Elem2No. This state is initialized by
    //         ParMesh::ExchangeFaceNbrData().
-   //    - NCFace >= 0 --> non-conforming face; can be one of:
-   //       - Elem2Inf < 0 --> master non-conforming face, interior or shared;
+   //    - NCFace >= 0 --> nonconforming face; can be one of:
+   //       - Elem2Inf < 0 --> master nonconforming face, interior or shared;
    //         In this case, Elem2No is -1; see GenerateNCFaceInfo().
-   //       - Elem2Inf >= 0 --> shared slave non-conforming face where element 2
+   //       - Elem2Inf >= 0 --> shared slave nonconforming face where element 2
    //         is the master face-neighbor element with index -1-Elem2No; see
    //         ParNCMesh::GetFaceNeighbors().
    //
-   // A ghost face is a non-conforming face that is generated by a non-local,
+   // A ghost face is a nonconforming face that is generated by a non-local,
    // i.e. ghost, element. A ghost face has index i in faces_info such that
    // i >= GetNumFaces().
    // Classification of a ghost (non-local) face based on its FaceInfo:
@@ -211,7 +270,7 @@ public:
    Array<int> bdr_attributes;
 
    NURBSExtension *NURBSext; ///< Optional NURBS mesh extension.
-   NCMesh *ncmesh;           ///< Optional non-conforming mesh extension.
+   NCMesh *ncmesh;           ///< Optional nonconforming mesh extension.
    Array<GeometricFactors*> geom_factors; ///< Optional geometric factors.
    Array<FaceGeometricFactors*>
    face_geom_factors; ///< Optional face geometric factors.
@@ -651,7 +710,7 @@ public:
 
    int AddVertex(double x, double y = 0.0, double z = 0.0);
    int AddVertex(const double *coords);
-   /// Mark vertex @a i as non-conforming, with parent vertices @a p1 and @a p2.
+   /// Mark vertex @a i as nonconforming, with parent vertices @a p1 and @a p2.
    void AddVertexParents(int i, int p1, int p2);
 
    int AddSegment(int v1, int v2, int attr = 1);
@@ -880,13 +939,19 @@ public:
    /// Return the number of faces (3D), edges (2D) or vertices (1D).
    int GetNumFaces() const;
 
-   /// Returns the number of faces according to the requested type.
-   /** If type==Boundary returns only the "true" number of boundary faces
-       contrary to GetNBE() that returns "fake" boundary faces associated to
-       visualization for GLVis.
-       Similarly, if type==Interior, the "fake" boundary faces associated to
-       visualization are counted as interior faces. */
-   int GetNFbyType(FaceType type) const;
+   /** @brief Return the number of faces (3D), edges (2D) or vertices (1D)
+       including ghost faces. */
+   int GetNumFacesWithGhost() const;
+
+   /** @brief Returns the number of faces according to the requested type, does
+       not count master nonconforming faces.
+
+       If type==Boundary returns only the number of true boundary faces
+       contrary to GetNBE() that returns all "boundary" elements which may
+       include actual interior faces.
+       Similarly, if type==Interior, only the true interior faces are counted
+       excluding all master nonconforming faces. */
+   virtual int GetNFbyType(FaceType type) const;
 
    /// Utility function: sum integers from all processors (Allreduce).
    virtual long ReduceInt(int value) const { return value; }
@@ -1170,8 +1235,9 @@ public:
    ///    mask & 4 - Loc1, mask & 8 - Loc2, mask & 16 - Face.
    /// These mask values are defined in the ConfigMasks enum type as part of the
    /// FaceElementTransformations class in fem/eltrans.hpp.
-   FaceElementTransformations *GetFaceElementTransformations(int FaceNo,
-                                                             int mask = 31);
+   virtual FaceElementTransformations *GetFaceElementTransformations(
+      int FaceNo,
+      int mask = 31);
 
    FaceElementTransformations *GetInteriorFaceTransformations (int FaceNo)
    {
@@ -1189,6 +1255,172 @@ public:
    {
       return (faces_info[FaceNo].Elem2No >= 0);
    }
+
+   /** This enumerated type describes the three main face topologies:
+       - Boundary, for faces on the boundary of the computational domain,
+       - Conforming, for conforming faces interior to the computational domain,
+       - Nonconforming, for nonconforming faces interior to the computational
+         domain. */
+   enum class FaceTopology { Boundary,
+                             Conforming,
+                             Nonconforming,
+                             NA
+                           };
+
+   /** This enumerated type describes the location of the two elements sharing a
+       face, Local meaning that the element is local to the MPI rank, FaceNbr
+       meaning that the element is distributed on a different MPI rank, this
+       typically means that methods with FaceNbr should be used to access the
+       relevant information, e.g., ParFiniteElementSpace::GetFaceNbrElementVDofs.
+    */
+   enum class ElementLocation { Local, FaceNbr, NA };
+
+   /** This enumerated type describes the topological relation of an element to
+       a face:
+       - Coincident meaning that the element's face is topologically equal to
+         the mesh face.
+       - Superset meaning that the element's face is topologically coarser than
+         the mesh face, i.e., the element's face contains the mesh face.
+       - Subset meaning that the element's face is topologically finer than the
+         mesh face, i.e., the element's face is contained in the mesh face.
+       Superset and Subset are only relevant for nonconforming faces.
+       Master nonconforming faces have a conforming element on one side, and a
+       fine element on the other side. Slave nonconforming faces have a
+       conforming element on one side, and a coarse element on the other side.
+    */
+   enum class ElementConformity { Coincident, Superset, Subset, NA };
+
+   /** This enumerated type describes the corresponding FaceInfo internal
+       representation (encoded cases), c.f. FaceInfo's documentation:
+       Classification of a local (non-ghost) face based on its FaceInfo:
+         - Elem2No >= 0 --> local interior face; can be either:
+            - NCFace == -1 --> LocalConforming,
+            - NCFace >= 0 --> LocalSlaveNonconforming,
+         - Elem2No < 0 --> local "boundary" face; can be one of:
+            - NCFace == -1 --> conforming face; can be either:
+               - Elem2Inf < 0 --> Boundary,
+               - Elem2Inf >= 0 --> SharedConforming,
+            - NCFace >= 0 --> nonconforming face; can be one of:
+               - Elem2Inf < 0 --> MasterNonconforming (shared or not shared),
+               - Elem2Inf >= 0 --> SharedSlaveNonconforming.
+       Classification of a ghost (non-local) face based on its FaceInfo:
+         - Elem1No == -1 --> GhostMaster (includes other unused ghost faces),
+         - Elem1No >= 0 --> GhostSlave.
+    */
+   enum class FaceInfoTag { Boundary,
+                            LocalConforming,
+                            LocalSlaveNonconforming,
+                            SharedConforming,
+                            SharedSlaveNonconforming,
+                            MasterNonconforming,
+                            GhostSlave,
+                            GhostMaster
+                          };
+
+   /** @brief This structure is used as a human readable output format that
+       decipheres the information contained in Mesh::FaceInfo when using the
+       Mesh::GetFaceInformation() method.
+
+       The element indices in this structure don't need further processing,
+       contrary to the ones obtained through Mesh::GetFacesElements and can
+       directly be used, e.g., Elem1 and Elem2 indices.
+       Likewise the orientations for Elem1 and Elem2 already take into account
+       special cases and can be used as is.
+   */
+   struct FaceInformation
+   {
+      FaceTopology topology;
+
+      struct
+      {
+         ElementLocation location;
+         ElementConformity conformity;
+         int index;
+         int local_face_id;
+         int orientation;
+      } element[2];
+
+      FaceInfoTag tag;
+      int ncface;
+      const DenseMatrix* point_matrix;
+
+      /** @brief Return true if the face is a local interior face which is NOT
+          a master nonconforming face. */
+      bool IsLocal() const
+      {
+         return element[1].location == Mesh::ElementLocation::Local;
+      }
+
+      /** @brief Return true if the face is a shared interior face which is NOT
+          a master nonconforming face. */
+      bool IsShared() const
+      {
+         return element[1].location == Mesh::ElementLocation::FaceNbr;
+      }
+
+      /** @brief return true if the face is an interior face to the computaion
+          domain, either a local or shared interior face (not a boundary face)
+          which is NOT a master nonconforming face.
+       */
+      bool IsInterior() const
+      {
+         return topology == FaceTopology::Conforming ||
+                topology == FaceTopology::Nonconforming;
+      }
+
+      /** @brief Return true if the face is a boundary face. */
+      bool IsBoundary() const
+      {
+         return topology == FaceTopology::Boundary;
+      }
+
+      /// @brief Return true if the face is of the same type as @a type.
+      bool IsOfFaceType(FaceType type) const
+      {
+         switch (type)
+         {
+            case FaceType::Interior:
+               return IsInterior();
+            case FaceType::Boundary:
+               return IsBoundary();
+            default:
+               return false;
+         }
+      }
+
+      /// @brief Return true if the face is a conforming face.
+      bool IsConforming() const
+      {
+         return topology == FaceTopology::Conforming;
+      }
+
+      /// @brief Return true if the face is a nonconforming fine face.
+      bool IsNonconformingFine() const
+      {
+         return topology == FaceTopology::Nonconforming &&
+                (element[0].conformity == ElementConformity::Superset ||
+                 element[1].conformity == ElementConformity::Superset);
+      }
+
+      /// @brief Return true if the face is a nonconforming coarse face.
+      /** Note that ghost nonconforming master faces cannot be clearly
+          identified as such with the currently available information, so this
+          method will return false for such faces. */
+      bool IsNonconformingCoarse() const
+      {
+         return topology == FaceTopology::Nonconforming &&
+                element[1].conformity == ElementConformity::Subset;
+      }
+
+      /// @brief cast operator from FaceInformation to FaceInfo.
+      operator Mesh::FaceInfo() const;
+   };
+
+   /** This method aims to provide face information in a deciphered format, i.e.
+       Mesh::FaceInformation, compared to the raw encoded information returned
+       by Mesh::GetFaceElements() and Mesh::GetFaceInfos(). */
+   FaceInformation GetFaceInformation(int f) const;
+
    void GetFaceElements (int Face, int *Elem1, int *Elem2) const;
    void GetFaceInfos (int Face, int *Inf1, int *Inf2) const;
    void GetFaceInfos (int Face, int *Inf1, int *Inf2, int *NCFace) const;
@@ -1329,7 +1561,7 @@ public:
 
    /** Refine selected mesh elements. Refinement type can be specified for each
        element. The function can do conforming refinement of triangles and
-       tetrahedra and non-conforming refinement (i.e., with hanging-nodes) of
+       tetrahedra and nonconforming refinement (i.e., with hanging-nodes) of
        triangles, quadrilaterals and hexahedra. If 'nonconforming' = -1,
        suitable refinement method is selected automatically (namely, conforming
        refinement for triangles). Use nonconforming = 0/1 to force the method.
@@ -1381,9 +1613,9 @@ public:
    void DegreeElevate(int rel_degree, int degree = 16);
    ///@}
 
-   /** Make sure that a quad/hex mesh is considered to be non-conforming (i.e.,
+   /** Make sure that a quad/hex mesh is considered to be nonconforming (i.e.,
        has an associated NCMesh object). Simplex meshes can be both conforming
-       (default) or non-conforming. */
+       (default) or nonconforming. */
    void EnsureNCMesh(bool simplices_nonconforming = false);
 
    bool Conforming() const { return ncmesh == NULL; }
@@ -1710,6 +1942,9 @@ inline void ShiftRight(int &a, int &b, int &c)
    a = c;  c = b;  b = t;
 }
 
+/// @brief Print function for Mesh::FaceInformation.
+std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info);
+
 }
 
 #endif

mesh/ncmesh.cpp

@@ -942,7 +942,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
       //     +-----------+           *--X
       //    0             1
 
-      if (ref_type == 1) // split along X axis
+      if (ref_type == Refinement::X) // split along X axis
       {
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid23 = GetMidEdgeNode(no[2], no[3]);
@@ -962,7 +962,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckAnisoFace(no[4], no[5], no[6], no[7], mid45, mid67);
          CheckAnisoFace(no[3], no[2], no[1], no[0], mid23, mid01);
       }
-      else if (ref_type == 2) // split along Y axis
+      else if (ref_type == Refinement::Y) // split along Y axis
       {
          int mid12 = GetMidEdgeNode(no[1], no[2]);
          int mid30 = GetMidEdgeNode(no[3], no[0]);
@@ -982,7 +982,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckAnisoFace(no[5], no[6], no[7], no[4], mid56, mid74);
          CheckAnisoFace(no[0], no[3], no[2], no[1], mid30, mid12);
       }
-      else if (ref_type == 4) // split along Z axis
+      else if (ref_type == Refinement::Z) // split along Z axis
       {
          int mid04 = GetMidEdgeNode(no[0], no[4]);
          int mid15 = GetMidEdgeNode(no[1], no[5]);
@@ -1002,7 +1002,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckAnisoFace(no[6], no[2], no[3], no[7], mid26, mid37);
          CheckAnisoFace(no[7], no[3], no[0], no[4], mid37, mid04);
       }
-      else if (ref_type == 3) // XY split
+      else if (ref_type == Refinement::XY) // XY split
       {
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1041,7 +1041,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckIsoFace(no[3], no[2], no[1], no[0], mid23, mid12, mid01, mid30, midf0);
          CheckIsoFace(no[4], no[5], no[6], no[7], mid45, mid56, mid67, mid74, midf5);
       }
-      else if (ref_type == 5) // XZ split
+      else if (ref_type == Refinement::XZ) // XZ split
       {
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid23 = GetMidEdgeNode(no[2], no[3]);
@@ -1080,7 +1080,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckIsoFace(no[0], no[1], no[5], no[4], mid01, mid15, mid45, mid04, midf1);
          CheckIsoFace(no[2], no[3], no[7], no[6], mid23, mid37, mid67, mid26, midf3);
       }
-      else if (ref_type == 6) // YZ split
+      else if (ref_type == Refinement::YZ) // YZ split
       {
          int mid12 = GetMidEdgeNode(no[1], no[2]);
          int mid30 = GetMidEdgeNode(no[3], no[0]);
@@ -1119,7 +1119,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckIsoFace(no[1], no[2], no[6], no[5], mid12, mid26, mid56, mid15, midf2);
          CheckIsoFace(no[3], no[0], no[4], no[7], mid30, mid04, mid74, mid37, midf4);
       }
-      else if (ref_type == 7) // full isotropic refinement
+      else if (ref_type == Refinement::XYZ) // full isotropic refinement
       {
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1189,7 +1189,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          MFEM_ABORT("invalid refinement type.");
       }
 
-      if (ref_type != 7) { Iso = false; }
+      if (ref_type != Refinement::XYZ) { Iso = false; }
    }
    else if (el.Geom() == Geometry::PRISM)
    {
@@ -1209,7 +1209,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
 
       if (ref_type < 4) // XY refinement (split in 4 wedges)
       {
-         ref_type = 3; // for consistence
+         ref_type = Refinement::XY; // for consistence
 
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1239,7 +1239,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          CheckAnisoFace(no[1], no[2], no[5], no[4], mid12, mid45);
          CheckAnisoFace(no[2], no[0], no[3], no[5], mid20, mid53);
       }
-      else if (ref_type == 4) // Z refinement only (split in 2 wedges)
+      else if (ref_type == Refinement::Z) // Z refinement only (split in 2 wedges)
       {
          int mid03 = GetMidEdgeNode(no[0], no[3]);
          int mid14 = GetMidEdgeNode(no[1], no[4]);
@@ -1259,7 +1259,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
       }
       else if (ref_type > 4) // full isotropic refinement (split in 8 wedges)
       {
-         ref_type = 7; // for consistence
+         ref_type = Refinement::XYZ; // for consistence
 
          int mid01 = GetMidEdgeNode(no[0], no[1]);
          int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1318,7 +1318,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          MFEM_ABORT("invalid refinement type.");
       }
 
-      if (ref_type != 7) { Iso = false; }
+      if (ref_type != Refinement::XYZ) { Iso = false; }
    }
    else if (el.Geom() == Geometry::TETRAHEDRON)
    {
@@ -1335,7 +1335,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
       //     +------------+     *--X
       //    0              1
 
-      ref_type = 7; // for consistence
+      ref_type = Refinement::XYZ; // for consistence
 
       int mid01 = GetMidEdgeNode(no[0], no[1]);
       int mid12 = GetMidEdgeNode(no[1], no[2]);
@@ -1412,7 +1412,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
    {
       ref_type &= 0x3; // ignore Z bit
 
-      if (ref_type == 1) // X split
+      if (ref_type == Refinement::X) // X split
       {
          int mid01 = nodes.GetId(no[0], no[1]);
          int mid23 = nodes.GetId(no[2], no[3]);
@@ -1423,7 +1423,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          child[1] = NewQuadrilateral(mid01, no[1], no[2], mid23,
                                      attr, fa[0], fa[1], fa[2], -1);
       }
-      else if (ref_type == 2) // Y split
+      else if (ref_type == Refinement::Y) // Y split
       {
          int mid12 = nodes.GetId(no[1], no[2]);
          int mid30 = nodes.GetId(no[3], no[0]);
@@ -1434,7 +1434,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
          child[1] = NewQuadrilateral(mid30, mid12, no[2], no[3],
                                      attr, -1, fa[1], fa[2], fa[3]);
       }
-      else if (ref_type == 3) // iso split
+      else if (ref_type == Refinement::XY) // iso split
       {
          int mid01 = nodes.GetId(no[0], no[1]);
          int mid12 = nodes.GetId(no[1], no[2]);
@@ -1460,11 +1460,11 @@ void NCMesh::RefineElement(int elem, char ref_type)
          MFEM_ABORT("Invalid refinement type.");
       }
 
-      if (ref_type != 3) { Iso = false; }
+      if (ref_type != Refinement::XY) { Iso = false; }
    }
    else if (el.Geom() == Geometry::TRIANGLE)
    {
-      ref_type = 3; // for consistence
+      ref_type = Refinement::XY; // for consistence
 
       // isotropic split - the only ref_type available for triangles
       int mid01 = nodes.GetId(no[0], no[1]);
@@ -1478,7 +1478,7 @@ void NCMesh::RefineElement(int elem, char ref_type)
    }
    else if (el.Geom() == Geometry::SEGMENT)
    {
-      ref_type = 1; // for consistence
+      ref_type = Refinement::X; // for consistence
 
       int mid = nodes.GetId(no[0], no[1]);
       child[0] = NewSegment(no[0], mid, attr, fa[0], -1);
@@ -1624,43 +1624,62 @@ void NCMesh::DerefineElement(int elem)
       }
    }
 
-   int fa[6];
-   int rt1 = el.ref_type - 1;
+   int faces_attribute[6];
+   int ref_type_key = el.ref_type - 1;
 
    for (int i = 0; i < 8; i++) { el.node[i] = -1; }
 
    // retrieve original corner nodes and face attributes from the children
    if (el.Geom() == Geometry::CUBE)
    {
-      for (int i = 0; i < 8; i++)
+      // Sets corner nodes from childs
+      constexpr int nb_cube_childs = 8;
+      for (int i = 0; i < nb_cube_childs; i++)
       {
-         Element &ch = elements[child[hex_deref_table[rt1][i]]];
+         const int child_local_index = hex_deref_table[ref_type_key][i];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          el.node[i] = ch.node[i];
       }
-      for (int i = 0; i < 6; i++)
+      // Sets faces attributes from childs' faces
+      constexpr int nb_cube_faces = 6;
+      for (int i = 0; i < nb_cube_faces; i++)
       {
-         Element &ch = elements[child[hex_deref_table[rt1][i + 8]]];
+         const int child_local_index = hex_deref_table[ref_type_key]
+                                       [i + nb_cube_childs];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
-                            ch.node[fv[2]], ch.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
+                                         ch.node[fv[2]], ch.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::PRISM)
    {
-      MFEM_ASSERT(prism_deref_table[rt1][0] != -1, "invalid prism refinement");
-      for (int i = 0; i < 6; i++)
+      MFEM_ASSERT(prism_deref_table[ref_type_key][0] != -1,
+                  "invalid prism refinement");
+      constexpr int nb_prism_childs = 6;
+      for (int i = 0; i < nb_prism_childs; i++)
       {
-         Element &ch = elements[child[prism_deref_table[rt1][i]]];
+         const int child_local_index = prism_deref_table[ref_type_key][i];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          el.node[i] = ch.node[i];
       }
       el.node[6] = el.node[7] = -1;
 
-      for (int i = 0; i < 5; i++)
+      constexpr int nb_prism_faces = 5;
+      for (int i = 0; i < nb_prism_faces; i++)
       {
-         Element &ch = elements[child[prism_deref_table[rt1][i + 6]]];
+         const int child_local_index = prism_deref_table[ref_type_key]
+                                       [i + nb_prism_childs];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
-                            ch.node[fv[2]], ch.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
+                                         ch.node[fv[2]], ch.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::TETRAHEDRON)
@@ -1671,43 +1690,55 @@ void NCMesh::DerefineElement(int elem)
          Element& ch2 = elements[child[(i+1) & 0x3]];
          el.node[i] = ch1.node[i];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch2.node[fv[0]], ch2.node[fv[1]],
-                            ch2.node[fv[2]], ch2.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch2.node[fv[0]], ch2.node[fv[1]],
+                                         ch2.node[fv[2]], ch2.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::SQUARE)
    {
-      for (int i = 0; i < 4; i++)
+      constexpr int nb_square_childs = 4;
+      for (int i = 0; i < nb_square_childs; i++)
       {
-         Element &ch = elements[child[quad_deref_table[rt1][i]]];
+         const int child_local_index = quad_deref_table[ref_type_key][i];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          el.node[i] = ch.node[i];
       }
-      for (int i = 0; i < 4; i++)
+      constexpr int nb_square_faces = 4;
+      for (int i = 0; i < nb_square_faces; i++)
       {
-         Element &ch = elements[child[quad_deref_table[rt1][i + 4]]];
+         const int child_local_index = quad_deref_table[ref_type_key]
+                                       [i + nb_square_childs];
+         const int child_global_index = child[child_local_index];
+         Element &ch = elements[child_global_index];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
-                            ch.node[fv[2]], ch.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
+                                         ch.node[fv[2]], ch.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::TRIANGLE)
    {
-      for (int i = 0; i < 3; i++)
+      constexpr int nb_triangle_childs = 3;
+      for (int i = 0; i < nb_triangle_childs; i++)
       {
          Element& ch = elements[child[i]];
          el.node[i] = ch.node[i];
          const int* fv = GI[el.Geom()].faces[i];
-         fa[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
-                            ch.node[fv[2]], ch.node[fv[3]])->attribute;
+         faces_attribute[i] = faces.Find(ch.node[fv[0]], ch.node[fv[1]],
+                                         ch.node[fv[2]], ch.node[fv[3]])
+                              ->attribute;
       }
    }
    else if (el.Geom() == Geometry::SEGMENT)
    {
-      for (int i = 0; i < 2; i++)
+      constexpr int nb_segment_childs = 2;
+      for (int i = 0; i < nb_segment_childs; i++)
       {
          int ni = elements[child[i]].node[i];
          el.node[i] = ni;
-         fa[i] = faces.Find(ni, ni, ni, ni)->attribute;
+         faces_attribute[i] = faces.Find(ni, ni, ni, ni)->attribute;
       }
    }
    else
@@ -1731,7 +1762,7 @@ void NCMesh::DerefineElement(int elem)
       FreeElement(child[i]);
    }
 
-   RegisterFaces(elem, fa);
+   RegisterFaces(elem, faces_attribute);
 
    // delete unused faces
    childFaces.Sort();
@@ -1901,7 +1932,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
    {
       if (el.rank >= 0) // skip elements beyond the ghost layer in parallel
       {
-         if (el.rank == MyRank)
+         if (!IsGhost(el))
          {
             leaf_elements.Append(elem);
          }
@@ -1922,7 +1953,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
          el.index = -1;
       }
    }
-   else
+   else // Refined element
    {
       // in non-leaf elements, the 'rank' and 'index' members have no meaning
       el.rank = -1;
@@ -1930,7 +1961,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
 
       // recurse to subtrees; try to order leaf elements along a space-filling
       // curve by changing the order the children are visited at each level
-      if (el.Geom() == Geometry::SQUARE && el.ref_type == 3)
+      if (el.Geom() == Geometry::SQUARE && el.ref_type == Refinement::XY)
       {
          for (int i = 0; i < 4; i++)
          {
@@ -1939,7 +1970,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
             CollectLeafElements(el.child[ch], st, ghosts, counter);
          }
       }
-      else if (el.Geom() == Geometry::CUBE && el.ref_type == 7)
+      else if (el.Geom() == Geometry::CUBE && el.ref_type == Refinement::XYZ)
       {
          for (int i = 0; i < 8; i++)
          {
@@ -1948,7 +1979,7 @@ void NCMesh::CollectLeafElements(int elem, int state, Array<int> &ghosts,
             CollectLeafElements(el.child[ch], st, ghosts, counter);
          }
       }
-      else // no SFC tables yet for remaining cases
+      else // no space filling curve tables yet for remaining cases
       {
          for (int i = 0; i < 8; i++)
          {
@@ -1965,7 +1996,8 @@ void NCMesh::UpdateLeafElements()
 {
    Array<int> ghosts;
 
-   // collect leaf elements from all roots
+   // collect leaf elements in leaf_elements and ghosts elements in ghosts from
+   // all roots
    leaf_elements.SetSize(0);
    for (int i = 0, counter = 0; i < root_state.Size(); i++)
    {
@@ -2355,7 +2387,7 @@ void NCMesh::GetMeshComponents(Mesh &mesh) const
             else
             {
                MFEM_ASSERT(nc_elem.geom == Geometry::SEGMENT, "");
-               auto* point = (Segment*) mesh.NewElement(Geometry::POINT);
+               auto* point = (mfem::Point*) mesh.NewElement(Geometry::POINT);
                point->SetAttribute(face->attribute);
                point->GetVertices()[0] = nodes[node[fv[0]]].vert_index;
                mesh.boundary.Append(point);
@@ -2738,7 +2770,7 @@ void NCMesh::TraverseQuadFace(int vn0, int vn1, int vn2, int vn3,
 
          // reorder the point matrix according to slave face orientation
          PointMatrix pm_r;
-         sl.local = ReorderFacePointMat(vn0, vn1, vn2, vn3, elem, pm, pm_r);;
+         sl.local = ReorderFacePointMat(vn0, vn1, vn2, vn3, elem, pm, pm_r);
          sl.matrix = matrix_map.GetIndex(pm_r);
 
          eface[0] = eface[2] = fa;

mesh/ncmesh.hpp

@@ -34,11 +34,14 @@ namespace mfem
     in the X, Y and Z directions, respectively (Z is ignored for quads). */
 struct Refinement
 {
+   enum : char { X = 1, Y = 2, Z = 4, XY = 3, XZ = 5, YZ = 6, XYZ = 7 };
    int index; ///< Mesh element number
    char ref_type; ///< refinement XYZ bit mask (7 = full isotropic)
 
    Refinement() = default;
-   Refinement(int index, int type = 7) : index(index), ref_type(type) {}
+
+   Refinement(int index, int type = Refinement::XYZ)
+      : index(index), ref_type(type) {}
 };
 
 
@@ -134,11 +137,16 @@ public:
 
    virtual ~NCMesh();
 
+   /// Return the dimension of the NCMesh.
    int Dimension() const { return Dim; }
+   /// Return the space dimension of the NCMesh.
    int SpaceDimension() const { return spaceDim; }
 
+   /// Return the number of vertices in the NCMesh.
    int GetNVertices() const { return NVertices; }
+   /// Return the number of edges in the NCMesh.
    int GetNEdges() const { return NEdges; }
+   /// Return the number of (2D) faces in the NCMesh.
    int GetNFaces() const { return NFaces; }
    virtual int GetNGhostElements() const { return 0; }
 
@@ -531,8 +539,34 @@ protected: // implementation
    Table element_vertex; ///< leaf-element to vertex table, see FindSetNeighbors
 
 
+   /// Update the leaf elements indices in leaf_elements
    void UpdateLeafElements();
+
+   /** @brief This method assigns indices to vertices (Node::vert_index) that
+       will be seen by the Mesh class and the rest of MFEM.
+
+       We must be careful to:
+       1. Stay compatible with the conforming code, which expects top-level
+          (original) vertices to be indexed first, otherwise GridFunctions
+          defined on a conforming mesh would no longer be valid when the
+          mesh is converted to an NC mesh.
+
+       2. Make sure serial NCMesh is compatible with the parallel ParNCMesh,
+          so it is possible to read parallel partial solutions in serial code
+          (e.g., serial GLVis). This means handling ghost elements, if present.
+
+       3. Assign vertices in a globally consistent order for parallel meshes:
+          if two vertices i,j are shared by two ranks r1,r2, and i<j on r1,
+          then i<j on r2 as well. This is true for top-level vertices but also
+          for the remaining shared vertices thanks to the globally consistent
+          SFC ordering of the leaf elements. This property reduces communication
+          and simplifies ParNCMesh. */
    void UpdateVertices(); ///< update Vertex::index and vertex_nodeId
+
+   /** Collect the leaf elements in leaf_elements, and the ghost elements in
+       ghosts. Compute and set the element indices of @a elements. On quad and
+       hex refined elements tries to order leaf elements along a space-filling
+       curve according to the given @a state variable. */
    void CollectLeafElements(int elem, int state, Array<int> &ghosts,
                             int &counter);
 
@@ -542,11 +576,17 @@ protected: // implementation
        Mesh::GetGeckoElementOrdering. */
    void InitRootState(int root_count);
 
+   /** Compute the Geometry::Type present in the root elements (coarse elements)
+       and set @a Geoms bitmask accordingly. */
    void InitGeomFlags();
 
+   /// Return true if the mesh contains prism elements.
    bool HavePrisms() const { return Geoms & (1 << Geometry::PRISM); }
+
+   /// Return true if the mesh contains tetrahedral elements.
    bool HaveTets() const   { return Geoms & (1 << Geometry::TETRAHEDRON); }
 
+   /// Return true if the Element @a el is a ghost element.
    bool IsGhost(const Element &el) const { return el.rank != MyRank; }
 
 
@@ -558,9 +598,14 @@ protected: // implementation
 
    Table derefinements; ///< possible derefinements, see GetDerefinementTable
 
+   /** Refine the element @a elem with the refinement @a ref_type
+       (c.f. Refinement::enum) */
    void RefineElement(int elem, char ref_type);
+
+   /// Derefine the element @a elem, does nothing on leaf elements.
    void DerefineElement(int elem);
 
+   // Add an Element @a el to the NCMesh, optimized to reuse freed elements.
    int AddElement(const Element &el)
    {
       if (free_element_ids.Size())
@@ -572,6 +617,8 @@ protected: // implementation
       }
       return elements.Append(el);
    }
+
+   // Free the element with index @a id.
    void FreeElement(int id)
    {
       free_element_ids.Append(id);
@@ -776,6 +823,22 @@ protected: // implementation
       }
    };
 
+   /** @brief The PointMatrix stores the coordinates of the slave face using the
+       master face coordinate as reference.
+
+       In 2D, the point matrix has the orientation of the parent
+       edge, so its columns need to be flipped when applying it, see
+       ApplyLocalSlaveTransformation.
+
+       In 3D, the orientation part of Elem2Inf is encoded in the point
+       matrix.
+
+       The following transformation gives the relation betwen the
+       reference quad face coordinates (xi, eta) in [0,1]^2, and the fine quad
+       face coordinates (x, y):
+       x = a0*(1-xi)*(1-eta) + a1*xi*(1-eta) + a2*xi*eta + a3*(1-xi)*eta
+       y = b0*(1-xi)*(1-eta) + b1*xi*(1-eta) + b2*xi*eta + b3*(1-xi)*eta
+   */
    struct PointMatrix
    {
       int np;

mesh/pmesh.cpp

@@ -2859,12 +2859,34 @@ void ParMesh::GetGhostFaceTransformation(
    }
 }
 
+FaceElementTransformations *ParMesh::GetFaceElementTransformations(
+   int FaceNo,
+   int mask)
+{
+   if (FaceNo < GetNumFaces())
+   {
+      return Mesh::GetFaceElementTransformations(FaceNo, mask);
+   }
+   else
+   {
+      const bool fill2 = mask & 10; // Elem2 and/or Loc2
+      return GetSharedFaceTransformationsByLocalIndex(FaceNo, fill2);
+   }
+}
+
 FaceElementTransformations *ParMesh::
 GetSharedFaceTransformations(int sf, bool fill2)
 {
    int FaceNo = GetSharedFace(sf);
 
+   return GetSharedFaceTransformationsByLocalIndex(FaceNo, fill2);
+}
+
+FaceElementTransformations *ParMesh::
+GetSharedFaceTransformationsByLocalIndex(int FaceNo, bool fill2)
+{
    FaceInfo &face_info = faces_info[FaceNo];
+   MFEM_VERIFY(face_info.Elem2Inf >= 0, "The face must be shared.");
 
    bool is_slave = Nonconforming() && IsSlaveFace(face_info);
    bool is_ghost = Nonconforming() && FaceNo >= GetNumFaces();
@@ -3010,6 +3032,13 @@ int ParMesh::GetSharedFace(int sface) const
    }
 }
 
+int ParMesh::GetNFbyType(FaceType type) const
+{
+   MFEM_VERIFY(have_face_nbr_data,
+               "ExchangeFaceNbrData() should be called before using GetNFbyType");
+   return Mesh::GetNFbyType(type);
+}
+
 // shift cyclically 3 integers a, b, c, so that the smallest of
 // order[a], order[b], order[c] is first
 static inline

mesh/pmesh.hpp

@@ -95,7 +95,7 @@ protected:
 
    // Mark all tets to ensure consistency across MPI tasks; also mark the
    // shared and boundary triangle faces using the consistently marked tets.
-   virtual void MarkTetMeshForRefinement(DSTable &v_to_v);
+   void MarkTetMeshForRefinement(DSTable &v_to_v) override;
 
    /// Return a number(0-1) identifying how the given edge has been split
    int GetEdgeSplittings(Element *edge, const DSTable &v_to_v, int *middle);
@@ -132,23 +132,23 @@ protected:
    void ExchangeFaceNbrData(Table *gr_sface, int *s2l_face);
 
    /// Refine a mixed 2D mesh uniformly.
-   virtual void UniformRefinement2D();
+   void UniformRefinement2D() override;
 
    /// Refine a mixed 3D mesh uniformly.
-   virtual void UniformRefinement3D();
+   void UniformRefinement3D() override;
 
-   virtual void NURBSUniformRefinement();
+   void NURBSUniformRefinement() override;
 
    /// This function is not public anymore. Use GeneralRefinement instead.
-   virtual void LocalRefinement(const Array<int> &marked_el, int type = 3);
+   void LocalRefinement(const Array<int> &marked_el, int type = 3) override;
 
    /// This function is not public anymore. Use GeneralRefinement instead.
-   virtual void NonconformingRefinement(const Array<Refinement> &refinements,
-                                        int nc_limit = 0);
+   void NonconformingRefinement(const Array<Refinement> &refinements,
+                                int nc_limit = 0) override;
 
-   virtual bool NonconformingDerefinement(Array<double> &elem_error,
-                                          double threshold, int nc_limit = 0,
-                                          int op = 1);
+   bool NonconformingDerefinement(Array<double> &elem_error,
+                                  double threshold, int nc_limit = 0,
+                                  int op = 1) override;
 
    void RebalanceImpl(const Array<int> *partition);
 
@@ -278,9 +278,9 @@ public:
        See @a Mesh::MakeSimplicial for more details. */
    static ParMesh MakeSimplicial(ParMesh &orig_mesh);
 
-   virtual void Finalize(bool refine = false, bool fix_orientation = false);
+   void Finalize(bool refine = false, bool fix_orientation = false) override;
 
-   virtual void SetAttributes();
+   void SetAttributes() override;
 
    MPI_Comm GetComm() const { return MyComm; }
    int GetNRanks() const { return NRanks; }
@@ -342,8 +342,8 @@ public:
    void ExchangeFaceNbrData();
    void ExchangeFaceNbrNodes();
 
-   virtual void SetCurvature(int order, bool discont = false, int space_dim = -1,
-                             int ordering = 1);
+   void SetCurvature(int order, bool discont = false, int space_dim = -1,
+                     int ordering = 1) override;
 
    int GetNFaceNeighbors() const { return face_nbr_group.Size(); }
    int GetNFaceNeighborElements() const { return face_nbr_elements.Size(); }
@@ -357,12 +357,56 @@ public:
        with indices offset by the local number of elements. */
    Table *GetFaceToAllElementTable() const;
 
-   /** Get the FaceElementTransformations for the given shared face (edge 2D).
+   /// Returns (a pointer to an object containing) the following data:
+   ///
+   /// 1) Elem1No - the index of the first element that contains this face this
+   ///    is the element that has the same outward unit normal vector as the
+   ///    face;
+   ///
+   /// 2) Elem2No - the index of the second element that contains this face this
+   ///    element has outward unit normal vector as the face multiplied with -1;
+   ///
+   /// 3) Elem1, Elem2 - pointers to the ElementTransformation's of the first
+   ///    and the second element respectively;
+   ///
+   /// 4) Face - pointer to the ElementTransformation of the face;
+   ///
+   /// 5) Loc1, Loc2 - IntegrationPointTransformation's mapping the face
+   ///    coordinate system to the element coordinate system (both in their
+   ///    reference elements). Used to transform IntegrationPoints from face to
+   ///    element. More formally, let:
+   ///       TL1, TL2 be the transformations represented by Loc1, Loc2,
+   ///       TE1, TE2 - the transformations represented by Elem1, Elem2,
+   ///       TF - the transformation represented by Face, then
+   ///       TF(x) = TE1(TL1(x)) = TE2(TL2(x)) for all x in the reference face.
+   ///
+   /// 6) FaceGeom - the base geometry for the face.
+   ///
+   /// The mask specifies which fields in the structure to return:
+   ///    mask & 1 - Elem1, mask & 2 - Elem2
+   ///    mask & 4 - Loc1, mask & 8 - Loc2, mask & 16 - Face.
+   /// These mask values are defined in the ConfigMasks enum type as part of the
+   /// FaceElementTransformations class in fem/eltrans.hpp.
+   FaceElementTransformations *GetFaceElementTransformations(
+      int FaceNo,
+      int mask = 31) override;
+
+   /** Get the FaceElementTransformations for the given shared face (edge 2D)
+       using the shared face index @a sf. @a fill2 specify if the information
+       for elem2 of the face should be computed or not.
        In the returned object, 1 and 2 refer to the local and the neighbor
        elements, respectively. */
    FaceElementTransformations *
    GetSharedFaceTransformations(int sf, bool fill2 = true);
 
+   /** Get the FaceElementTransformations for the given shared face (edge 2D)
+       using the face index @a FaceNo. @a fill2 specify if the information
+       for elem2 of the face should be computed or not.
+       In the returned object, 1 and 2 refer to the local and the neighbor
+       elements, respectively. */
+   FaceElementTransformations *
+   GetSharedFaceTransformationsByLocalIndex(int FaceNo, bool fill2 = true);
+
    ElementTransformation *
    GetFaceNbrElementTransformation(int i)
    {
@@ -381,11 +425,21 @@ public:
    /// Return the local face index for the given shared face.
    int GetSharedFace(int sface) const;
 
+   /** @brief Returns the number of local faces according to the requested type,
+       does not count master non-conforming faces.
+
+       If type==Boundary returns only the number of true boundary faces
+       contrary to GetNBE() that returns all "boundary" elements which may
+       include actual interior faces.
+       Similarly, if type==Interior, only the true interior faces (including
+       shared faces) are counted excluding all master non-conforming faces. */
+   int GetNFbyType(FaceType type) const override;
+
    /// See the remarks for the serial version in mesh.hpp
-   MFEM_DEPRECATED virtual void ReorientTetMesh();
+   MFEM_DEPRECATED void ReorientTetMesh() override;
 
    /// Utility function: sum integers from all processors (Allreduce).
-   virtual long ReduceInt(int value) const;
+   long ReduceInt(int value) const override;
 
    /** Load balance the mesh by equipartitioning the global space-filling
        sequence of elements. Works for nonconforming meshes only. */
@@ -401,23 +455,23 @@ public:
 
    /** Print the part of the mesh in the calling processor adding the interface
        as boundary (for visualization purposes) using the mfem v1.0 format. */
-   virtual void Print(std::ostream &out = mfem::out) const;
+   void Print(std::ostream &out = mfem::out) const override;
 
    /// Save the ParMesh to files (one for each MPI rank). The files will be
    /// given suffixes according to the MPI rank. The mesh will be written to the
    /// files using ParMesh::Print. The given @a precision will be used for ASCII
    /// output.
-   virtual void Save(const char *fname, int precision=16) const;
+   void Save(const char *fname, int precision=16) const override;
 
 #ifdef MFEM_USE_ADIOS2
    /** Print the part of the mesh in the calling processor using adios2 bp
        format. */
-   virtual void Print(adios2stream &out) const;
+   void Print(adios2stream &out) const override;
 #endif
 
    /** Print the part of the mesh in the calling processor adding the interface
        as boundary (for visualization purposes) using Netgen/Truegrid format .*/
-   virtual void PrintXG(std::ostream &out = mfem::out) const;
+   void PrintXG(std::ostream &out = mfem::out) const override;
 
    /** Write the mesh to the stream 'out' on Process 0 in a form suitable for
        visualization: the mesh is written as a disjoint mesh and the shared
@@ -435,15 +489,15 @@ public:
    /** Print the mesh in parallel PVTU format. The PVTU and VTU files will be
        stored in the directory specified by @a pathname. If the directory does
        not exist, it will be created. */
-   virtual void PrintVTU(std::string pathname,
-                         VTKFormat format=VTKFormat::ASCII,
-                         bool high_order_output=false,
-                         int compression_level=0,
-                         bool bdr=false);
+   void PrintVTU(std::string pathname,
+                 VTKFormat format=VTKFormat::ASCII,
+                 bool high_order_output=false,
+                 int compression_level=0,
+                 bool bdr=false) override;
 
    /// Parallel version of Mesh::Load().
-   virtual void Load(std::istream &input, int generate_edges = 0,
-                     int refine = 1, bool fix_orientation = true);
+   void Load(std::istream &input, int generate_edges = 0,
+             int refine = 1, bool fix_orientation = true) override;
 
    /// Returns the minimum and maximum corners of the mesh bounding box. For
    /// high-order meshes, the geometry is refined first "ref" times.
@@ -457,11 +511,11 @@ public:
    void Swap(ParMesh &other);
 
    /// Print various parallel mesh stats
-   virtual void PrintInfo(std::ostream &out = mfem::out);
+   void PrintInfo(std::ostream &out = mfem::out) override;
 
-   virtual int FindPoints(DenseMatrix& point_mat, Array<int>& elem_ids,
-                          Array<IntegrationPoint>& ips, bool warn = true,
-                          InverseElementTransformation *inv_trans = NULL);
+   int FindPoints(DenseMatrix& point_mat, Array<int>& elem_ids,
+                  Array<IntegrationPoint>& ips, bool warn = true,
+                  InverseElementTransformation *inv_trans = NULL) override;
 
    /// Debugging method
    void PrintSharedEntities(const char *fname_prefix) const;

mesh/vtk.cpp

@@ -600,4 +600,51 @@ const char *VTKByteOrder()
 
 }
 
+// Ensure ASCII output of uint8_t to stream is integer rather than character
+template <>
+void WriteBinaryOrASCII<uint8_t>(std::ostream &out, std::vector<char> &buf,
+                                 const uint8_t &val, const char *suffix,
+                                 VTKFormat format)
+{
+   if (format == VTKFormat::ASCII) { out << static_cast<int>(val) << suffix; }
+   else { bin_io::AppendBytes(buf, val); }
+}
+
+template <>
+void WriteBinaryOrASCII<double>(std::ostream &out, std::vector<char> &buf,
+                                const double &val, const char *suffix,
+                                VTKFormat format)
+{
+   if (format == VTKFormat::BINARY32)
+   {
+      bin_io::AppendBytes<float>(buf, float(val));
+   }
+   else if (format == VTKFormat::BINARY)
+   {
+      bin_io::AppendBytes(buf, val);
+   }
+   else
+   {
+      out << ZeroSubnormal(val) << suffix;
+   }
+}
+
+template <>
+void WriteBinaryOrASCII<float>(std::ostream &out, std::vector<char> &buf,
+                               const float &val, const char *suffix,
+                               VTKFormat format)
+{
+   if (format == VTKFormat::BINARY) { bin_io::AppendBytes<double>(buf, val); }
+   else if (format == VTKFormat::BINARY32) { bin_io::AppendBytes(buf, val); }
+   else { out << ZeroSubnormal(val) << suffix; }
+}
+
+void WriteBase64WithSizeAndClear(std::ostream &out, std::vector<char> &buf,
+                                 int compression_level)
+{
+   WriteVTKEncodedCompressed(out, buf.data(), buf.size(), compression_level);
+   out << '\n';
+   buf.clear();
+}
+
 } // namespace mfem

mesh/vtk.hpp

@@ -13,23 +13,34 @@
 #define MFEM_VTK
 
 #include "../fem/geom.hpp"
+#include "../general/binaryio.hpp"
 
 namespace mfem
 {
 
 // Helpers for reading and writing VTK format
 
-// VTK element types defined at: https://git.io/JvZLm
+/// @brief Helper class for converting between MFEM and VTK geometry types.
+///
+/// Note: The VTK element types defined are at: https://git.io/JvZLm
 struct VTKGeometry
 {
+   /// @name VTK geometry types
+   ///@{
    static const int POINT = 1;
+
+   /// @name Low-order (linear, straight-sided) VTK geometric types
+   ///@{
    static const int SEGMENT = 3;
    static const int TRIANGLE = 5;
    static const int SQUARE = 9;
    static const int TETRAHEDRON = 10;
    static const int CUBE = 12;
    static const int PRISM = 13;
+   ///@}
 
+   /// @name Legacy quadratic VTK geometric types
+   ///@{
    static const int QUADRATIC_SEGMENT = 21;
    static const int QUADRATIC_TRIANGLE = 22;
    static const int BIQUADRATIC_SQUARE = 28;
@@ -37,49 +48,138 @@ struct VTKGeometry
    static const int TRIQUADRATIC_CUBE = 29;
    static const int QUADRATIC_PRISM = 26;
    static const int BIQUADRATIC_QUADRATIC_PRISM = 32;
+   ///@}
 
+   /// @name Arbitrary-order VTK geometric types
+   ///@{
    static const int LAGRANGE_SEGMENT = 68;
    static const int LAGRANGE_TRIANGLE = 69;
    static const int LAGRANGE_SQUARE = 70;
    static const int LAGRANGE_TETRAHEDRON = 71;
    static const int LAGRANGE_CUBE = 72;
    static const int LAGRANGE_PRISM = 73;
+   ///@}
+   ///@}
 
+   /// Permutation from MFEM's prism ordering to VTK's prism ordering.
    static const int PrismMap[6];
+
+   /// @brief Permutation from MFEM's vertex ordering to VTK's vertex ordering.
+   /// @note If the MFEM and VTK orderings are the same, the vertex permutation
+   /// will be NULL.
    static const int *VertexPermutation[Geometry::NUM_GEOMETRIES];
 
+   /// Map from MFEM's Geometry::Type to linear VTK geometries.
    static const int Map[Geometry::NUM_GEOMETRIES];
+   /// Map from MFEM's Geometry::Type to legacy quadratic VTK geometries/
    static const int QuadraticMap[Geometry::NUM_GEOMETRIES];
+   /// Map from MFEM's Geometry::Type to arbitrary-order Lagrange VTK geometries
    static const int HighOrderMap[Geometry::NUM_GEOMETRIES];
 
+   /// Given a VTK geometry type, return the corresponding MFEM Geometry::Type.
    static Geometry::Type GetMFEMGeometry(int vtk_geom);
+   /// @brief Does the given VTK geometry type describe an arbitrary-order
+   /// Lagrange element?
    static bool IsLagrange(int vtk_geom);
+   /// @brief Does the given VTK geometry type describe a legacy quadratic
+   /// element?
    static bool IsQuadratic(int vtk_geom);
+   /// @brief For the given VTK geometry type and number of points, return the
+   /// order of the element.
    static int GetOrder(int vtk_geom, int npoints);
 };
 
+/// Data array format for VTK and VTU files.
 enum class VTKFormat
 {
+   /// Data arrays will be written in ASCII format.
    ASCII,
+   /// Data arrays will be written in binary format. Floating point numbers will
+   /// be be output with 64 bits of precision.
    BINARY,
+   /// Data arrays will be written in binary format. Floating point numbers will
+   /// be be output with 32 bits of precision.
    BINARY32
 };
 
-/// Create the VTK element connectivity array for a given element geometry and
-/// refinement level. Converts node numbers from MFEM to VTK ordering.
+/// @brief Create the VTK element connectivity array for a given element
+/// geometry and refinement level.
+///
+/// The output array @a con will be such that, for the @a ith VTK node index,
+/// con[i] will contain the index of the corresponding node in MFEM ordering.
 void CreateVTKElementConnectivity(Array<int> &con, Geometry::Type geom,
                                   int ref);
 
-/// Outputs encoded binary data in the format needed by VTK. The binary data
-/// will be base 64 encoded, and compressed if @a compression_level is not
-/// zero. The proper header will be prepended to the data.
+/// @brief Outputs encoded binary data in the base 64 format needed by VTK.
+///
+/// The binary data will be base 64 encoded, and compressed if @a
+/// compression_level is not zero. The proper header will be prepended to the
+/// data.
 void WriteVTKEncodedCompressed(std::ostream &out, const void *bytes,
                                uint32_t nbytes, int compression_level);
 
+/// @brief Return the VTK node index of the barycentric point @a b in a
+/// triangle with refinement level @a ref.
+///
+/// The barycentric index @a b has three components, satisfying b[0] + b[1] +
+/// b[2] == ref.
 int BarycentricToVTKTriangle(int *b, int ref);
 
+/// Determine the byte order and return either "BigEndian" or "LittleEndian"
 const char *VTKByteOrder();
 
+/// @brief Write either ASCII data to the stream or binary data to the buffer
+/// depending on the given format.
+///
+/// If @a format is VTK::ASCII, write the canonical ASCII representation of @a
+/// val to the output stream. Subnormal floating point numbers are rounded to
+/// zero. Otherwise, append its raw binary data to the byte buffer @a buf.
+///
+/// Note that there are specializations for @a uint8_t (to write as a numeric
+/// value rather than a character), and for @a float and @a double values to use
+/// the precision specified by @a format.
+template <typename T>
+void WriteBinaryOrASCII(std::ostream &out, std::vector<char> &buf, const T &val,
+                        const char *suffix, VTKFormat format)
+{
+   if (format == VTKFormat::ASCII) { out << val << suffix; }
+   else { bin_io::AppendBytes(buf, val); }
+}
+
+/// @brief Specialization of @ref WriteBinaryOrASCII for @a uint8_t to ensure
+/// ASCII output is numeric (rather than interpreting @a val as a character.)
+template <>
+void WriteBinaryOrASCII<uint8_t>(std::ostream &out, std::vector<char> &buf,
+                                 const uint8_t &val, const char *suffix,
+                                 VTKFormat format);
+
+/// @brief Specialization of @ref WriteBinaryOrASCII for @a double.
+///
+/// If @a format is equal to VTKFormat::BINARY32, @a val is converted to a @a
+/// float and written as 32 bits. Subnormals are rounded to zero in ASCII
+/// output.
+template <>
+void WriteBinaryOrASCII<double>(std::ostream &out, std::vector<char> &buf,
+                                const double &val, const char *suffix,
+                                VTKFormat format);
+
+/// @brief Specialization of @ref WriteBinaryOrASCII<T> for @a float.
+///
+/// If @a format is equal to VTKFormat::BINARY, @a val is converted to a @a
+/// double and written as 64 bits. Subnormals are rounded to zero in ASCII
+/// output.
+template <>
+void WriteBinaryOrASCII<float>(std::ostream &out, std::vector<char> &buf,
+                               const float &val, const char *suffix,
+                               VTKFormat format);
+
+/// @brief Encode in base 64 (and potentially compress) the given data, write it
+/// to the output stream (with a header) and clear the buffer.
+///
+/// @sa WriteVTKEncodedCompressed.
+void WriteBase64WithSizeAndClear(std::ostream &out, std::vector<char> &buf,
+                                 int compression_level);
+
 } // namespace mfem
 
 #endif

miniapps/common/pfem_extras.cpp

@@ -310,7 +310,7 @@ void VisualizeMesh(socketstream &sock, const char *vishost, int visport,
 }
 
 void VisualizeField(socketstream &sock, const char *vishost, int visport,
-                    ParGridFunction &gf, const char *title,
+                    const ParGridFunction &gf, const char *title,
                     int x, int y, int w, int h, const char *keys, bool vec)
 {
    ParMesh &pmesh = *gf.ParFESpace()->GetParMesh();

miniapps/common/pfem_extras.hpp

@@ -197,7 +197,7 @@ void VisualizeMesh(socketstream &sock, const char *vishost, int visport,
 /// specified host and port. Set the visualization window title, and optionally,
 /// its geometry.
 void VisualizeField(socketstream &sock, const char *vishost, int visport,
-                    ParGridFunction &gf, const char *title,
+                    const ParGridFunction &gf, const char *title,
                     int x = 0, int y = 0, int w = 400, int h = 400,
                     const char *keys = NULL, bool vec = false);
 

miniapps/shifted/CMakeLists.txt

@@ -13,12 +13,14 @@ if (MFEM_USE_MPI)
   list(APPEND DIST_COMMON_SOURCES
         dist_solver.cpp
         sbm_solver.cpp
-        marking.cpp)
+        marking.cpp
+        extrapolator.cpp)
   list(APPEND DIST_COMMON_HEADERS
         dist_solver.hpp
         sbm_solver.hpp
         sbm_aux.hpp
-        marking.hpp)
+        marking.hpp
+        extrapolator.hpp)
 
   convert_filenames_to_full_paths(DIST_COMMON_SOURCES)
   convert_filenames_to_full_paths(DIST_COMMON_HEADERS)
@@ -37,6 +39,11 @@ if (MFEM_USE_MPI)
     ${DIST_COMMON_FILES}
     LIBRARIES mfem mfem-common)
 
+  add_mfem_miniapp(extrapolate
+    MAIN extrapolate.cpp
+    ${DIST_COMMON_FILES}
+    LIBRARIES mfem mfem-common)
+
   if (MFEM_ENABLE_TESTING)
     add_test(NAME shifted_distance_np${MFEM_MPI_NP}
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}

miniapps/shifted/extrapolate.cpp

@@ -0,0 +1,218 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//            ------------------------------------------------
+//            Extrapolation Miniapp: PDE-based extrapolation
+//            ------------------------------------------------
+//
+// This miniapp extrapolates a finite element function from a set of elements
+// (known values) to the rest of the domain. The set of elements that contains
+// the known values is specified by the positive values of a level set
+// Coefficient. The known values are not modified. The miniapp supports two
+// PDE-based approaches [1, 2], both of which rely on solving a sequence of
+// advection problems in the direction of the unknown parts of the domain.
+// The extrapolation can be constant (1st order), linear (2nd order), or
+// quadratic (3rd order). These formal orders hold for a limited band around
+// the zero level set, see the given references for more info.
+//
+// [1] Aslam, "A Partial Differential Equation Approach to Multidimensional
+// Extrapolation", JCP 193(1), 2004.
+// [2] Bochkov, Gibou, "PDE-Based Multidimensional Extrapolation of Scalar
+// Fields over Interfaces with Kinks and High Curvatures", SISC 42(4), 2020.
+//
+// Compile with: make extrapolate
+//
+// Sample runs:
+//     mpirun -np 4 extrapolate -m "../../data/inline-segment.mesh" -rs 6 -ed 2
+//     mpirun -np 4 extrapolate -rs 5 -p 0 -ed 2
+//     mpirun -np 4 extrapolate -rs 5 -p 1 -ed 2
+//     mpirun -np 4 extrapolate -rs 5 -p 1 -et 1 -ed 1 -dg 1
+//     mpirun -np 4 extrapolate -m "../../data/inline-hex.mesh" -ed 1 -rs 1
+//     mpirun -np 4 extrapolate -m "../../data/inline-hex.mesh" -p 1 -ed 1 -rs 1
+
+#include "extrapolator.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int problem = 0;
+
+double domainLS(const Vector &coord)
+{
+   // Map from [0,1] to [-1,1].
+   const int dim = coord.Size();
+   const double x = coord(0)*2.0 - 1.0,
+                y = (dim > 1) ? coord(1)*2.0 - 1.0 : 0.0,
+                z = (dim > 2) ? coord(2)*2.0 - 1.0 : 0.0;
+
+   switch (problem)
+   {
+      case 0:
+      {
+         // Sphere.
+         return 0.75 - sqrt(x*x + y*y + z*z + 1e-12);
+      }
+      case 1:
+      {
+         // Star.
+         MFEM_VERIFY(dim > 1, "Problem 1 is not applicable to 1D.");
+
+         return 0.60 - sqrt(x*x + y*y + z*z + 1e-12) +
+                0.25 * (y*y*y*y*y + 5.0*x*x*x*x*y - 10.0*x*x*y*y*y) /
+                pow(x*x + y*y + z*z + 1e-12, 2.5) *
+                std::cos(0.5*M_PI * z / 0.6);
+      }
+      default: MFEM_ABORT("Bad option for --problem!"); return 0.0;
+   }
+}
+
+double solution0(const Vector &coord)
+{
+   // Map from [0,1] to [-1,1].
+   const int dim = coord.Size();
+   const double x = coord(0)*2.0 - 1.0 + 0.25,
+                y = (dim > 1) ? coord(1)*2.0 - 1.0 : 0.0,
+                z = (dim > 2) ? coord(2)*2.0 - 1.0 : 0.0;
+
+   return std::cos(M_PI * x) * std::cos(M_PI * y) * std::cos(M_PI * z);
+}
+
+void PrintNorm(int myid, Vector &v, std::string text)
+{
+   double norm = v.Norml1();
+   MPI_Allreduce(MPI_IN_PLACE, &norm, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+   if (myid == 0)
+   {
+      std::cout << std::setprecision(12) << std::fixed
+                << text << norm << std::endl;
+   }
+}
+
+void PrintIntegral(int myid, ParGridFunction &g, std::string text)
+{
+   ConstantCoefficient zero(0.0);
+   double norm = g.ComputeL1Error(zero);
+   if (myid == 0)
+   {
+      std::cout << std::setprecision(12) << std::fixed
+                << text << norm << std::endl;
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   // Initialize MPI.
+   MPI_Session mpi;
+   int myid = mpi.WorldRank();
+
+   // Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int rs_levels = 2;
+   Extrapolator::XtrapType ex_type   = Extrapolator::ASLAM;
+   AdvectionOper::AdvectionMode dg_mode = AdvectionOper::HO;
+   int ex_degree = 1;
+   int order = 2;
+   double distance = 0.35;
+   bool vis_on = true;
+   int vis_steps_cnt = 50;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&rs_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption((int*)&ex_type, "-et", "--extrap-type",
+                  "Extrapolation type: Aslam (0) or Bochkov (1).");
+   args.AddOption((int*)&dg_mode, "-dg", "--dg-mode",
+                  "DG advection mode: 0 - Standard High-Order,\n\t"
+                  "                   1 - Low-Order Upwind Diffusion.");
+   args.AddOption(&ex_degree, "-ed", "--extrap-degree",
+                  "Extrapolation degree: 0/1/2 for constant/linear/quadratic.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&distance, "-d", "--distance",
+                  "Extrapolation distance.");
+   args.AddOption(&problem, "-p", "--problem",
+                  "0 - 2D circle,\n\t"
+                  "1 - 2D star");
+   args.AddOption(&vis_on, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&vis_steps_cnt, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0) { args.PrintUsage(cout); }
+      return 1;
+   }
+   if (myid == 0) { args.PrintOptions(cout); }
+
+   // Refine the mesh and distribute.
+   Mesh mesh(mesh_file, 1, 1);
+   for (int lev = 0; lev < rs_levels; lev++) { mesh.UniformRefinement(); }
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+   const int dim = pmesh.Dimension();
+
+   // Input function.
+   L2_FECollection fec_L2(order, dim);
+   ParFiniteElementSpace pfes_L2(&pmesh, &fec_L2);
+   ParGridFunction u(&pfes_L2);
+   FunctionCoefficient u0_coeff(solution0);
+   u.ProjectCoefficient(u0_coeff);
+
+   // Extrapolate.
+   Extrapolator xtrap;
+   xtrap.xtrap_type     = ex_type;
+   xtrap.advection_mode = dg_mode;
+   xtrap.xtrap_degree   = ex_degree;
+   xtrap.visualization  = vis_on;
+   xtrap.vis_steps      = vis_steps_cnt;
+   FunctionCoefficient ls_coeff(domainLS);
+   ParGridFunction ux(&pfes_L2);
+   xtrap.Extrapolate(ls_coeff, u, distance, ux);
+
+   PrintNorm(myid, ux, "Solution l1 norm: ");
+   PrintIntegral(myid, ux, "Solution L1 norm: ");
+
+   GridFunctionCoefficient u_exact_coeff(&u);
+   double err_L1 = ux.ComputeL1Error(u_exact_coeff),
+          err_L2 = ux.ComputeL2Error(u_exact_coeff);
+   if (myid == 0)
+   {
+      std::cout << "Global L1 error: " << err_L1 << std::endl
+                << "Global L2 error: " << err_L2 << std::endl;
+   }
+   double loc_error_L1, loc_error_L2, loc_error_LI;
+   xtrap.ComputeLocalErrors(ls_coeff, u, ux,
+                            loc_error_L1, loc_error_L2, loc_error_LI);
+   if (myid == 0)
+   {
+      std::cout << "Local  L1 error: " << loc_error_L1 << std::endl
+                << "Local  L2 error: " << loc_error_L2 << std::endl
+                << "Local  Li error: " << loc_error_LI << std::endl;
+   }
+
+   // ParaView output.
+   ParGridFunction ls_gf(&pfes_L2);
+   ls_gf.ProjectCoefficient(ls_coeff);
+   ParaViewDataCollection dacol("ParaViewExtrapolate", &pmesh);
+   dacol.SetLevelsOfDetail(order);
+   dacol.RegisterField("Level Set Function", &ls_gf);
+   dacol.RegisterField("Extrapolated Solution", &ux);
+   dacol.SetTime(1.0);
+   dacol.SetCycle(1);
+   dacol.Save();
+
+   return 0;
+}

miniapps/shifted/extrapolator.cpp

@@ -0,0 +1,565 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "extrapolator.hpp"
+#include "../common/mfem-common.hpp"
+#include "marking.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+const char vishost[] = "localhost";
+const int  visport   = 19916;
+int wsize            = 350; // glvis window size
+
+AdvectionOper::AdvectionOper(Array<bool> &zones, ParBilinearForm &Mbf,
+                             ParBilinearForm &Kbf, const Vector &rhs)
+   : TimeDependentOperator(Mbf.Size()),
+     active_zones(zones),
+     M(Mbf), K(Kbf), K_mat(NULL), b(rhs),
+     lo_solver(NULL), lumpedM(NULL)
+{
+   K_mat = K.ParallelAssemble(&K.SpMat());
+
+   ParBilinearForm M_Lump(M.ParFESpace());
+   lumpedM = new Vector;
+   M_Lump.AddDomainIntegrator(new LumpedIntegrator(new MassIntegrator));
+   M_Lump.Assemble();
+   M_Lump.Finalize();
+   M_Lump.SpMat().GetDiag(*lumpedM);
+   lo_solver = new DiscreteUpwindLOSolver(*M.ParFESpace(),
+                                          K.SpMat(), *lumpedM);
+}
+
+AdvectionOper::~AdvectionOper()
+{
+   delete lo_solver;
+   delete lumpedM;
+   delete K_mat;
+}
+
+void AdvectionOper::Mult(const Vector &x, Vector &dx) const
+{
+   ParFiniteElementSpace &pfes = *M.ParFESpace();
+   const int NE = pfes.GetNE();
+   const int nd = pfes.GetFE(0)->GetDof();
+   Array<int> dofs(nd);
+
+   if (adv_mode == LO)
+   {
+      lo_solver->CalcLOSolution(x, b, dx);
+      for (int k = 0; k < NE; k++)
+      {
+         pfes.GetElementDofs(k, dofs);
+         if (active_zones[k] == false)
+         {
+            dx.SetSubVector(dofs, 0.0);
+            continue;
+         }
+      }
+      return;
+   }
+
+   MFEM_VERIFY(adv_mode == HO, "Wrong input for avection mode (-dg).");
+
+   Vector rhs(x.Size());
+   K_mat->Mult(x, rhs);
+   rhs += b;
+
+   DenseMatrix M_loc(nd);
+   DenseMatrixInverse M_loc_inv(&M_loc);
+   Vector rhs_loc(nd), dx_loc(nd);
+   for (int k = 0; k < NE; k++)
+   {
+      pfes.GetElementDofs(k, dofs);
+
+      if (active_zones[k] == false)
+      {
+         dx.SetSubVector(dofs, 0.0);
+         continue;
+      }
+
+      rhs.GetSubVector(dofs, rhs_loc);
+      M.SpMat().GetSubMatrix(dofs, dofs, M_loc);
+      M_loc_inv.Factor();
+      M_loc_inv.Mult(rhs_loc, dx_loc);
+      dx.SetSubVector(dofs, dx_loc);
+   }
+}
+
+void AdvectionOper::ComputeElementsMinMax(const ParGridFunction &gf,
+                                          Vector &el_min, Vector &el_max) const
+{
+   ParFiniteElementSpace &pfes = *gf.ParFESpace();
+   const int NE = pfes.GetNE(), ndof = pfes.GetFE(0)->GetDof();
+   for (int k = 0; k < NE; k++)
+   {
+      el_min(k) = numeric_limits<double>::infinity();
+      el_max(k) = -numeric_limits<double>::infinity();
+
+      for (int i = 0; i < ndof; i++)
+      {
+         el_min(k) = min(el_min(k), gf(k*ndof + i));
+         el_max(k) = max(el_max(k), gf(k*ndof + i));
+      }
+   }
+}
+
+void AdvectionOper::ComputeBounds(const ParFiniteElementSpace &pfes,
+                                  const Vector &el_min, const Vector &el_max,
+                                  Vector &dof_min, Vector &dof_max) const
+{
+   ParMesh *pmesh = pfes.GetParMesh();
+   L2_FECollection fec_bounds(0, pmesh->Dimension());
+   ParFiniteElementSpace pfes_bounds(pmesh, &fec_bounds);
+   ParGridFunction el_min_gf(&pfes_bounds), el_max_gf(&pfes_bounds);
+   const int NE = pmesh->GetNE(), ndofs = dof_min.Size() / NE;
+
+   el_min_gf = el_min;
+   el_max_gf = el_max;
+
+   el_min_gf.ExchangeFaceNbrData(); el_max_gf.ExchangeFaceNbrData();
+   const Vector &min_nbr = el_min_gf.FaceNbrData();
+   const Vector &max_nbr = el_max_gf.FaceNbrData();
+   const Table &el_to_el = pmesh->ElementToElementTable();
+   Array<int> face_nbr_el;
+   for (int k = 0; k < NE; k++)
+   {
+      double k_min = el_min_gf(k), k_max = el_max_gf(k);
+
+      el_to_el.GetRow(k, face_nbr_el);
+      for (int n = 0; n < face_nbr_el.Size(); n++)
+      {
+         if (face_nbr_el[n] < NE)
+         {
+            // Local neighbor.
+            k_min = std::min(k_min, el_min_gf(face_nbr_el[n]));
+            k_max = std::max(k_max, el_max_gf(face_nbr_el[n]));
+         }
+         else
+         {
+            // MPI face neighbor.
+            k_min = std::min(k_min, min_nbr(face_nbr_el[n] - NE));
+            k_max = std::max(k_max, max_nbr(face_nbr_el[n] - NE));
+         }
+      }
+
+      for (int j = 0; j < ndofs; j++)
+      {
+         dof_min(k*ndofs + j) = k_min;
+         dof_max(k*ndofs + j) = k_max;
+      }
+   }
+}
+
+void Extrapolator::Extrapolate(Coefficient &level_set,
+                               const ParGridFunction &input,
+                               const double time_period,
+                               ParGridFunction &xtrap)
+{
+   ParMesh &pmesh = *input.ParFESpace()->GetParMesh();
+   const int order = input.ParFESpace()->GetOrder(0),
+             dim   = pmesh.Dimension(), NE = pmesh.GetNE();
+
+   // Get a ParGridFunction and mark elements.
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace pfes_H1(&pmesh, &fec);
+   ParGridFunction ls_gf(&pfes_H1);
+   ls_gf.ProjectCoefficient(level_set);
+   if (visualization)
+   {
+      socketstream sock1, sock2;
+      common::VisualizeField(sock1, vishost, visport, ls_gf,
+                             "Domain level set", 0, 0, wsize, wsize,
+                             "rRjlmm********A");
+      common::VisualizeField(sock2, vishost, visport, input,
+                             "Input u", 0, wsize+60, wsize, wsize,
+                             "rRjlmm********A");
+      MPI_Barrier(pmesh.GetComm());
+   }
+   // Mark elements.
+   Array<int> elem_marker;
+   ShiftedFaceMarker marker(pmesh, pfes_H1, false);
+   ls_gf.ExchangeFaceNbrData();
+   marker.MarkElements(ls_gf, elem_marker);
+
+   // The active zones are where we extrapolate (where the PDE is solved).
+   Array<bool> active_zones(NE);
+   for (int k = 0; k < NE; k++)
+   {
+      // Extrapolation is done in zones that are CUT or OUTSIDE.
+      active_zones[k] =
+         (elem_marker[k] == ShiftedFaceMarker::INSIDE) ? false : true;
+   }
+
+   // Setup a VectorCoefficient for n = - grad_ls / |grad_ls|.
+   // The sign makes it point out of the known region.
+   // The coefficient must be continuous to have well-defined transport.
+   LevelSetNormalGradCoeff ls_n_coeff_L2(ls_gf);
+   ParFiniteElementSpace pfes_H1_vec(&pmesh, &fec, dim);
+   ParGridFunction lsn_gf(&pfes_H1_vec);
+   ls_gf.ExchangeFaceNbrData();
+   lsn_gf.ProjectDiscCoefficient(ls_n_coeff_L2, GridFunction::ARITHMETIC);
+   VectorGridFunctionCoefficient ls_n_coeff(&lsn_gf);
+
+   // Initial solution.
+   // Trim to the known values (only elements inside the known region).
+   Array<int> dofs;
+   L2_FECollection fec_L2(order, dim);
+   ParFiniteElementSpace pfes_L2(&pmesh, &fec_L2);
+   ParGridFunction u(&pfes_L2), vis_marking(&pfes_L2);
+   u.ProjectGridFunction(input);
+   for (int k = 0; k < NE; k++)
+   {
+      pfes_L2.GetElementDofs(k, dofs);
+      if (elem_marker[k] != ShiftedFaceMarker::INSIDE)
+      { u.SetSubVector(dofs, 0.0); }
+      vis_marking.SetSubVector(dofs, elem_marker[k]);
+   }
+   if (visualization)
+   {
+      socketstream sock1, sock2;
+      common::VisualizeField(sock1, vishost, visport, u,
+                             "Fixed (known) u values", wsize, 0,
+                             wsize, wsize, "rRjlmm********A");
+      common::VisualizeField(sock2, vishost, visport, vis_marking,
+                             "Element markings", 0, 2*wsize+60,
+                             wsize, wsize, "rRjlmm********A");
+   }
+
+   // Normal derivative function.
+   ParGridFunction n_grad_u(&pfes_L2);
+   NormalGradCoeff n_grad_u_coeff(u, ls_n_coeff);
+   n_grad_u.ProjectCoefficient(n_grad_u_coeff);
+   if (visualization && xtrap_degree >= 1)
+   {
+      socketstream sock;
+      common::VisualizeField(sock, vishost, visport, n_grad_u,
+                             "n.grad(u)", 2*wsize, 0, wsize, wsize,
+                             "rRjlmm********A");
+   }
+
+   // 2nd normal derivative function.
+   ParGridFunction n_grad_n_grad_u(&pfes_L2);
+   NormalGradCoeff n_grad_n_grad_u_coeff(n_grad_u, ls_n_coeff);
+   n_grad_n_grad_u.ProjectCoefficient(n_grad_n_grad_u_coeff);
+   if (visualization && xtrap_degree == 2)
+   {
+      socketstream sock;
+      common::VisualizeField(sock, vishost, visport, n_grad_n_grad_u,
+                             "n.grad(n.grad(u))", 3*wsize, 0, wsize, wsize,
+                             "rRjmm********A");
+   }
+
+   ParBilinearForm lhs_bf(&pfes_L2), rhs_bf(&pfes_L2);
+   lhs_bf.AddDomainIntegrator(new MassIntegrator);
+   const double alpha = -1.0;
+   rhs_bf.AddDomainIntegrator(new ConvectionIntegrator(ls_n_coeff, alpha));
+   auto trace_i = new NonconservativeDGTraceIntegrator(ls_n_coeff, alpha);
+   rhs_bf.AddInteriorFaceIntegrator(trace_i);
+   rhs_bf.KeepNbrBlock(true);
+
+   ls_gf.ExchangeFaceNbrData();
+   lhs_bf.Assemble();
+   lhs_bf.Finalize();
+   rhs_bf.Assemble(0);
+   rhs_bf.Finalize(0);
+
+   // Compute a CFL time step.
+   double h_min = std::numeric_limits<double>::infinity();
+   for (int k = 0; k < NE; k++)
+   {
+      h_min = std::min(h_min, pmesh.GetElementSize(k));
+   }
+   MPI_Allreduce(MPI_IN_PLACE, &h_min, 1, MPI_DOUBLE, MPI_MIN, pmesh.GetComm());
+   // The propagation speed is 1.
+   double dt = 0.25 * h_min / order / 1.0;
+   double half_dt = 0.5 * dt;
+   if (advection_mode == AdvectionOper::LO)
+   {
+      dt = half_dt;
+   }
+
+   // Time loops.
+   Vector rhs(pfes_L2.GetVSize());
+   AdvectionOper adv_oper(active_zones, lhs_bf, rhs_bf, rhs);
+   adv_oper.adv_mode = advection_mode;
+   RK2Solver ode_solver(1.0);
+   ode_solver.Init(adv_oper);
+
+   if (xtrap_degree == 0)
+   {
+      // Constant extrapolation of u (always LO).
+      rhs = 0.0;
+      adv_oper.adv_mode = AdvectionOper::LO;
+      TimeLoop(u, ode_solver, time_period, half_dt,
+               wsize, "Extrap const u -- LO");
+      xtrap.ProjectGridFunction(u);
+      return;
+   }
+
+   std::string mode_text = "HO";
+   if (advection_mode == AdvectionOper::LO)  { mode_text = "LO"; }
+
+   MFEM_VERIFY(xtrap_degree == 1 || xtrap_degree == 2, "Wrong order input.");
+   if (xtrap_type == ASLAM)
+   {
+      if (xtrap_degree == 1)
+      {
+         // Constant extrapolation of [n.grad_u] (always LO).
+         rhs = 0.0;
+         adv_oper.adv_mode = AdvectionOper::LO;
+         TimeLoop(n_grad_u, ode_solver, time_period, half_dt,
+                  2*wsize, "Extrap const n.grad(u) -- Aslam -- LO");
+
+         adv_oper.adv_mode = advection_mode;
+
+         // Linear extrapolation of u.
+         lhs_bf.Mult(n_grad_u, rhs);
+         TimeLoop(u, ode_solver, time_period, dt,
+                  wsize, "Extrap linear u -- Aslam -- " + mode_text);
+      }
+
+      if (xtrap_degree == 2)
+      {
+         // Constant extrapolation of [n.grad(n.grad(u))] (always LO).
+         rhs = 0.0;
+         adv_oper.adv_mode = AdvectionOper::LO;
+         TimeLoop(n_grad_n_grad_u, ode_solver, time_period, half_dt,
+                  3*wsize, "Extrap const n.grad(n.grad(u)) -- Aslam -- LO");
+
+         adv_oper.adv_mode = advection_mode;
+
+         // Linear extrapolation of [n.grad_u].
+         lhs_bf.Mult(n_grad_n_grad_u, rhs);
+         TimeLoop(n_grad_u, ode_solver, time_period, dt,
+                  2*wsize, "Extrap linear n.grad(u) -- Aslam -- " + mode_text);
+
+         // Quadratic extrapolation of u.
+         lhs_bf.Mult(n_grad_u, rhs);
+         TimeLoop(u, ode_solver, time_period, dt,
+                  wsize, "Extrap quadratic u -- Aslam -- " + mode_text);
+      }
+   }
+   else if (xtrap_type == BOCHKOV)
+   {
+      if (xtrap_degree == 1)
+      {
+         // Constant extrapolation of all grad(u) components (always LO).
+         rhs = 0.0;
+         adv_oper.adv_mode = AdvectionOper::LO;
+         ParGridFunction grad_u_0(&pfes_L2), grad_u_1(&pfes_L2);
+         GradComponentCoeff grad_u_0_coeff(u, 0), grad_u_1_coeff(u, 1);
+         grad_u_0.ProjectCoefficient(grad_u_0_coeff);
+         grad_u_1.ProjectCoefficient(grad_u_1_coeff);
+         TimeLoop(grad_u_0, ode_solver, time_period, half_dt,
+                  2*wsize, "Extrap const du_dx -- Bochkov -- LO");
+         TimeLoop(grad_u_1, ode_solver, time_period, half_dt,
+                  3*wsize, "Extrap const du_dy -- Bochkov -- LO");
+
+         adv_oper.adv_mode = advection_mode;
+
+         // Linear extrapolation of u.
+         ParLinearForm rhs_lf(&pfes_L2);
+         NormalGradComponentCoeff grad_u_n(grad_u_0, grad_u_1, ls_n_coeff);
+         rhs_lf.AddDomainIntegrator(new DomainLFIntegrator(grad_u_n));
+         rhs_lf.Assemble();
+         rhs = rhs_lf;
+         TimeLoop(u, ode_solver, time_period, dt,
+                  wsize, "Extrap linear u -- Bochkov -- " + mode_text);
+      }
+
+      if (xtrap_degree == 2)
+      {
+         MFEM_ABORT("Quadratic Bochkov method is not implemented.");
+      }
+   }
+   else { MFEM_ABORT("Wrong input for extrapolation type (-et)."); }
+
+   xtrap.ProjectGridFunction(u);
+}
+
+// Errors in cut elements.
+void Extrapolator::ComputeLocalErrors(Coefficient &level_set,
+                                      const ParGridFunction &exact,
+                                      const ParGridFunction &xtrap,
+                                      double &err_L1, double &err_L2,
+                                      double &err_LI)
+{
+   ParMesh &pmesh = *exact.ParFESpace()->GetParMesh();
+   const int order = exact.ParFESpace()->GetOrder(0),
+             dim   = pmesh.Dimension(), NE = pmesh.GetNE();
+
+   // Get a ParGridFunction and mark elements.
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace pfes_H1(&pmesh, &fec);
+   ParGridFunction ls_gf(&pfes_H1);
+   ls_gf.ProjectCoefficient(level_set);
+   // Mark elements.
+   Array<int> elem_marker;
+   ShiftedFaceMarker marker(pmesh, pfes_H1, false);
+   ls_gf.ExchangeFaceNbrData();
+   marker.MarkElements(ls_gf, elem_marker);
+
+   Vector errors_L1(NE), errors_L2(NE), errors_LI(NE);
+   GridFunctionCoefficient exact_coeff(&exact);
+
+   xtrap.ComputeElementL1Errors(exact_coeff, errors_L1);
+   xtrap.ComputeElementL2Errors(exact_coeff, errors_L2);
+   xtrap.ComputeElementMaxErrors(exact_coeff, errors_LI);
+   err_L1 = 0.0, err_L2 = 0.0, err_LI = 0.0;
+   double cut_volume = 0.0;
+   for (int k = 0; k < NE; k++)
+   {
+      if (elem_marker[k] == ShiftedFaceMarker::CUT)
+      {
+         err_L1 += errors_L1(k);
+         err_L2 += errors_L2(k);
+         err_LI = std::max(err_LI, errors_LI(k));
+         cut_volume += pmesh.GetElementVolume(k);
+      }
+   }
+   MPI_Comm comm = pmesh.GetComm();
+   MPI_Allreduce(MPI_IN_PLACE, &err_L1, 1, MPI_DOUBLE, MPI_SUM, comm);
+   MPI_Allreduce(MPI_IN_PLACE, &err_L2, 1, MPI_DOUBLE, MPI_SUM, comm);
+   MPI_Allreduce(MPI_IN_PLACE, &err_LI, 1, MPI_DOUBLE, MPI_MAX, comm);
+   MPI_Allreduce(MPI_IN_PLACE, &cut_volume, 1, MPI_DOUBLE, MPI_SUM, comm);
+   err_L1 /= cut_volume;
+   err_L2 /= cut_volume;
+}
+
+void Extrapolator::TimeLoop(ParGridFunction &sltn, ODESolver &ode_solver,
+                            double t_final, double dt,
+                            int vis_x_pos, std::string vis_name)
+{
+   socketstream sock;
+
+   const int myid  = sltn.ParFESpace()->GetMyRank();
+   bool done = false;
+   double t = 0.0;
+   for (int ti = 0; !done;)
+   {
+      double dt_real = min(dt, t_final - t);
+      ode_solver.Step(sltn, t, dt_real);
+      ti++;
+
+      done = (t >= t_final - 1e-8*dt);
+      if (done || ti % vis_steps == 0)
+      {
+         if (myid == 0)
+         {
+            cout << vis_name+" / time step: " << ti << ", time: " << t << endl;
+         }
+         if (visualization)
+         {
+            common::VisualizeField(sock, vishost, visport, sltn,
+                                   vis_name.c_str(), vis_x_pos, wsize+60,
+                                   wsize, wsize, "rRjlmm********A");
+            MPI_Barrier(sltn.ParFESpace()->GetComm());
+         }
+      }
+   }
+}
+
+
+
+DiscreteUpwindLOSolver::DiscreteUpwindLOSolver(ParFiniteElementSpace &space,
+                                               const SparseMatrix &adv,
+                                               const Vector &Mlump)
+   : pfes(space), K(adv), D(adv), K_smap(), M_lumped(Mlump)
+{
+   // Assuming it is finalized.
+   const int *I = K.GetI(), *J = K.GetJ(), n = K.Size();
+   K_smap.SetSize(I[n]);
+   for (int row = 0, j = 0; row < n; row++)
+   {
+      for (int end = I[row+1]; j < end; j++)
+      {
+         int col = J[j];
+         // Find the offset, _j, of the (col,row) entry and store it in smap[j].
+         for (int _j = I[col], _end = I[col+1]; true; _j++)
+         {
+            MFEM_VERIFY(_j != _end, "Can't find the symmetric entry!");
+
+            if (J[_j] == row) { K_smap[j] = _j; break; }
+         }
+      }
+   }
+
+   ComputeDiscreteUpwindMatrix();
+}
+
+void DiscreteUpwindLOSolver::CalcLOSolution(const Vector &u, const Vector &rhs,
+                                            Vector &du) const
+{
+   ParGridFunction u_gf(&pfes);
+   u_gf = u;
+   ApplyDiscreteUpwindMatrix(u_gf, du);
+
+   const int s = du.Size();
+   for (int i = 0; i < s; i++)
+   {
+      du(i) = (du(i) + rhs(i)) / M_lumped(i);
+   }
+}
+
+void DiscreteUpwindLOSolver::ComputeDiscreteUpwindMatrix() const
+{
+   const int *I = K.HostReadI(), *J = K.HostReadJ(), n = K.Size();
+
+   const double *K_data = K.HostReadData();
+
+   double *D_data = D.HostReadWriteData();
+   D.HostReadWriteI(); D.HostReadWriteJ();
+
+   for (int i = 0, k = 0; i < n; i++)
+   {
+      double rowsum = 0.;
+      for (int end = I[i+1]; k < end; k++)
+      {
+         int j = J[k];
+         double kij = K_data[k];
+         double kji = K_data[K_smap[k]];
+         double dij = fmax(fmax(0.0,-kij),-kji);
+         D_data[k] = kij + dij;
+         D_data[K_smap[k]] = kji + dij;
+         if (i != j) { rowsum += dij; }
+      }
+      D(i,i) = K(i,i) - rowsum;
+   }
+}
+
+void DiscreteUpwindLOSolver::ApplyDiscreteUpwindMatrix(ParGridFunction &u,
+                                                       Vector &du) const
+{
+   const int s = u.Size();
+   const int *I = D.HostReadI(), *J = D.HostReadJ();
+   const double *D_data = D.HostReadData();
+
+   u.ExchangeFaceNbrData();
+   const Vector &u_np = u.FaceNbrData();
+
+   for (int i = 0; i < s; i++)
+   {
+      du(i) = 0.0;
+      for (int k = I[i]; k < I[i + 1]; k++)
+      {
+         int j = J[k];
+         double u_j  = (j < s) ? u(j) : u_np[j - s];
+         double d_ij = D_data[k];
+         du(i) += d_ij * u_j;
+      }
+   }
+}
+
+}

miniapps/shifted/extrapolator.hpp

@@ -0,0 +1,191 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_EXTRAPOLATOR_HPP
+#define MFEM_EXTRAPOLATOR_HPP
+
+#include "mfem.hpp"
+
+namespace mfem
+{
+
+class DiscreteUpwindLOSolver;
+class FluxBasedFCT;
+
+class AdvectionOper : public TimeDependentOperator
+{
+private:
+   Array<bool> &active_zones;
+   ParBilinearForm &M, &K;
+   HypreParMatrix *K_mat;
+   const Vector &b;
+
+   DiscreteUpwindLOSolver *lo_solver;
+   Vector *lumpedM;
+
+   void ComputeElementsMinMax(const ParGridFunction &gf,
+                              Vector &el_min, Vector &el_max) const;
+   void ComputeBounds(const ParFiniteElementSpace &pfes,
+                      const Vector &el_min, const Vector &el_max,
+                      Vector &dof_min, Vector &dof_max) const;
+   void ZeroOutInactiveZones(Vector &dx);
+
+public:
+   // HO is standard FE advection solve; LO is upwind diffusion.
+   enum AdvectionMode {HO, LO} adv_mode = AdvectionOper::HO;
+
+   AdvectionOper(Array<bool> &zones, ParBilinearForm &Mbf,
+                 ParBilinearForm &Kbf, const Vector &rhs);
+
+   ~AdvectionOper();
+
+   virtual void Mult(const Vector &x, Vector &dx) const;
+};
+
+// Extrapolates through DG advection based on:
+// [1] Aslam, "A Partial Differential Equation Approach to Multidimensional
+// Extrapolation", JCP 193(1), 2004.
+// [2] Bochkov, Gibou, "PDE-Based Multidimensional Extrapolation of Scalar
+// Fields over Interfaces with Kinks and High Curvatures", SISC 42(4), 2020.
+class Extrapolator
+{
+public:
+   enum XtrapType {ASLAM, BOCHKOV} xtrap_type = ASLAM;
+   AdvectionOper::AdvectionMode advection_mode = AdvectionOper::HO;
+   int xtrap_degree   = 1;
+   bool visualization = false;
+   int vis_steps      = 5;
+
+   Extrapolator() { }
+
+   // The known values taken from elements where level_set > 0, and extrapolated
+   // to all other elements. The known values are not changed.
+   void Extrapolate(Coefficient &level_set, const ParGridFunction &input,
+                    const double time_period, ParGridFunction &xtrap);
+
+   // Errors in cut elements, given an exact solution.
+   void ComputeLocalErrors(Coefficient &level_set, const ParGridFunction &exact,
+                           const ParGridFunction &xtrap,
+                           double &err_L1, double &err_L2, double &err_LI);
+
+private:
+   void TimeLoop(ParGridFunction &sltn, ODESolver &ode_solver, double t_final,
+                 double dt, int vis_x_pos, std::string vis_name);
+};
+
+class LevelSetNormalGradCoeff : public VectorCoefficient
+{
+private:
+   const ParGridFunction &ls_gf;
+
+public:
+   LevelSetNormalGradCoeff(const ParGridFunction &ls) :
+      VectorCoefficient(ls.ParFESpace()->GetMesh()->Dimension()), ls_gf(ls) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip)
+   {
+      Vector grad_ls(vdim), n(vdim);
+      ls_gf.GetGradient(T, grad_ls);
+      const double norm_grad = grad_ls.Norml2();
+      V = grad_ls;
+      if (norm_grad > 0.0) { V /= norm_grad; }
+
+      // Since positive level set values correspond to the known region, we
+      // transport into the opposite direction of the gradient.
+      V *= -1;
+   }
+};
+
+class GradComponentCoeff : public Coefficient
+{
+private:
+   const ParGridFunction &u_gf;
+   int comp;
+
+public:
+   GradComponentCoeff(const ParGridFunction &u, int c) : u_gf(u), comp(c) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      Vector grad_u(T.GetDimension());
+      u_gf.GetGradient(T, grad_u);
+      return grad_u(comp);
+   }
+};
+
+class NormalGradCoeff : public Coefficient
+{
+private:
+   const ParGridFunction &u_gf;
+   VectorCoefficient &n_coeff;
+
+public:
+   NormalGradCoeff(const ParGridFunction &u, VectorCoefficient &n)
+      : u_gf(u), n_coeff(n) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      const int dim = T.GetDimension();
+      Vector n(dim), grad_u(dim);
+      n_coeff.Eval(n, T, ip);
+      u_gf.GetGradient(T, grad_u);
+      return n * grad_u;
+   }
+};
+
+class NormalGradComponentCoeff : public Coefficient
+{
+private:
+   const ParGridFunction &du_dx, &du_dy;
+   VectorCoefficient &n_coeff;
+
+public:
+   NormalGradComponentCoeff(const ParGridFunction &dx,
+                            const ParGridFunction &dy, VectorCoefficient &n)
+      : du_dx(dx), du_dy(dy), n_coeff(n) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      const int dim = T.GetDimension();
+      Vector n(dim), grad_u(dim);
+      n_coeff.Eval(n, T, ip);
+      grad_u(0) = du_dx.GetValue(T, ip);
+      grad_u(1) = du_dy.GetValue(T, ip);
+      return n * grad_u;
+   }
+};
+
+class DiscreteUpwindLOSolver
+{
+public:
+   DiscreteUpwindLOSolver(ParFiniteElementSpace &space, const SparseMatrix &adv,
+                          const Vector &Mlump);
+
+   void CalcLOSolution(const Vector &u, const Vector &rhs, Vector &du) const;
+
+   Array<int> &GetKmap() { return K_smap; }
+
+protected:
+   ParFiniteElementSpace &pfes;
+   const SparseMatrix &K;
+   mutable SparseMatrix D;
+
+   Array<int> K_smap;
+   const Vector &M_lumped;
+
+   void ComputeDiscreteUpwindMatrix() const;
+   void ApplyDiscreteUpwindMatrix(ParGridFunction &u, Vector &du) const;
+};
+
+} // namespace mfem
+
+#endif

miniapps/shifted/makefile

@@ -29,8 +29,10 @@ DIFFUSION_SRC = diffusion.cpp dist_solver.cpp sbm_solver.cpp marking.cpp
 DIFFUSION_OBJ = $(DIFFUSION_SRC:.cpp=.o)
 DISTANCE_SRC = distance.cpp dist_solver.cpp
 DISTANCE_OBJ = $(DISTANCE_SRC:.cpp=.o)
+EXTRAPOLATE_SRC = extrapolate.cpp extrapolator.cpp marking.cpp
+EXTRAPOLATE_OBJ = $(EXTRAPOLATE_SRC:.cpp=.o)
 
-PAR_MINIAPPS = distance diffusion
+PAR_MINIAPPS = distance diffusion extrapolate
 
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS =
@@ -65,6 +67,9 @@ distance: $(DISTANCE_OBJ)
 diffusion: $(DIFFUSION_OBJ)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(DIFFUSION_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
 
+extrapolate: $(EXTRAPOLATE_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(EXTRAPOLATE_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
 # Rule for building lib-common
 lib-common:
 	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
@@ -89,9 +94,9 @@ $(MFEM_LIB_FILE):
 clean: clean-build clean-exec
 
 clean-build:
-	rm -f *.o *~ distance diffusion
+	rm -f *.o *~ distance diffusion extrapolate
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
 	@rm -f diffusion.mesh diffusion.gf
-	@rm -rf ParaViewDistance ParaViewDiffusion
+	@rm -rf ParaViewDistance ParaViewDiffusion ParaViewExtrapolate

miniapps/shifted/marking.hpp

@@ -51,7 +51,9 @@ public:
         include_cut_cell(include_cut_cell_), initial_marking_done(false),
         level_set_index(0) { }
 
-   /// Mark all the elements in the mesh using the @a SBElementType
+   /// Mark all the elements in the mesh using the @a SBElementType.
+   /// A point is considered inside when the level set function is positive.
+   /// Assumes the ExchangeFaceNbrData() has been called for pmesh, ls_func.
    void MarkElements(const ParGridFunction &ls_func, Array<int> &elem_marker);
 
    /// List dofs associated with the surrogate boundary.

tests/unit/CMakeLists.txt

@@ -56,17 +56,20 @@ set(UNIT_TESTS_SRCS
   fem/test_blocknonlinearform.cpp
   fem/test_calcshape.cpp
   fem/test_coefficient.cpp
+  fem/test_coeff_revdiff.cpp
   fem/test_datacollection.cpp
   fem/test_derefine.cpp
   fem/test_estimator.cpp
   fem/test_face_elem_trans.cpp
   fem/test_face_permutation.cpp
   fem/test_fe.cpp
+  fem/test_fe_revdiff.cpp
   fem/test_get_value.cpp
   fem/test_getderivative.cpp
   fem/test_intrules.cpp
   fem/test_intruletypes.cpp
   fem/test_inversetransform.cpp
+  fem/test_eltrans_revdiff.cpp
   fem/test_lexicographic_ordering.cpp
   fem/test_lin_interp.cpp
   fem/test_linear_fes.cpp

tests/unit/fem/test_coeff_revdiff.cpp

@@ -0,0 +1,325 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include "catch.hpp"
+
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <fstream>
+#include <random>
+
+using namespace mfem;
+
+namespace
+{
+// String used to define a single element mesh, with a c-shaped quad element
+std::string mesh_str =
+   "MFEM mesh v1.0"                    "\n\n"
+   "dimension"                           "\n"
+   "2"                                 "\n\n"
+   "elements"                            "\n"
+   "1"                                   "\n"
+   "1 3 0 1 2 3"                       "\n\n"
+   "boundary"                            "\n"
+   "0"                                 "\n\n"
+   "vertices"                            "\n"
+   "4"                                 "\n\n"
+   "nodes"                               "\n"
+   "FiniteElementSpace"                  "\n"
+   "FiniteElementCollection: Quadratic"  "\n"
+   "VDim: 2"                             "\n"
+   "Ordering: 1"                         "\n"
+   "0 0"                                 "\n"
+   "0 2"                                 "\n"
+   "0 6"                                 "\n"
+   "0 8"                                 "\n"
+   "0 1"                                 "\n"
+   "-6 4"                                "\n"
+   "0 7"                                 "\n"
+   "-8 4"                                "\n"
+   "-7 4"                                "\n";
+
+double scalar_func(const Vector &x)
+{
+   double q = 0;
+   for (int i = 0; i < x.Size(); ++i)
+   {
+      q += pow(x(i), 2);
+   }
+   return q;
+}
+
+void scalar_funcRevDiff(const mfem::Vector &x, const double q_bar,
+                        mfem::Vector &x_bar)
+{
+   for (int i = 0; i < x.Size(); ++i)
+   {
+      x_bar(i) += q_bar * 2 * x(i);
+   }
+}
+
+double scalar_func2(const Vector &x)
+{
+   double q = 0;
+   for (int i = 0; i < x.Size(); ++i)
+   {
+      q += x(i);
+   }
+   return q;
+}
+
+void scalar_func2RevDiff(const mfem::Vector &x, const double q_bar,
+                         mfem::Vector &x_bar)
+{
+   for (int i = 0; i < x.Size(); ++i)
+   {
+      x_bar(i) += q_bar;
+   }
+}
+
+void func2D(const Vector &x, Vector &y)
+{
+   y.SetSize(2);
+   y(0) = x(0)*x(0) - x(1);
+   y(1) = x(0) * exp(x(1));
+}
+
+void func2DRevDiff(const Vector &x, const Vector &v_bar, Vector &x_bar)
+{
+   x_bar(0) = v_bar(0) * 2*x(0) + v_bar(1) * exp(x(1));
+   x_bar(1) = -v_bar(0) + v_bar(1) * x(0) * exp(x(1));
+}
+
+void func3D(const Vector &x, Vector &y)
+{
+   y.SetSize(3);
+   y(0) = x(0)*x(0) - x(1);
+   y(1) = x(0) * exp(x(1));
+   y(2) = x(2)*x(0) - x(1);
+}
+
+void func3DRevDiff(const Vector &x, const Vector &v_bar, Vector &x_bar)
+{
+   x_bar(0) = v_bar(0) * 2*x(0) + v_bar(1) * exp(x(1)) + v_bar(2)*x(2);
+   x_bar(1) = -v_bar(0) + v_bar(1) * x(0) * exp(x(1)) - v_bar(2);
+   x_bar(2) = v_bar(2) * x(0);
+}
+
+void runScalarTest(Mesh &mesh, Coefficient &q)
+{
+   constexpr double eps_fd = 1e-5;
+   std::default_random_engine generator;
+   std::uniform_real_distribution<double> distribution(-1.0,1.0);
+
+   for (int p = 1; p <= 4; ++p)
+   {
+      const int dim = mesh.Dimension();
+      H1_FECollection fec(p, dim);
+      FiniteElementSpace fes(&mesh, &fec);
+
+      const FiniteElement &el = *fes.GetFE(0);
+      IsoparametricTransformation trans;
+      mesh.GetElementTransformation(0, &trans);
+
+      DenseMatrix &coords = trans.GetPointMat();
+      DenseMatrix coords_bar(coords.Height(), coords.Width());
+
+      double Q_bar = distribution(generator);
+
+      int order = trans.OrderW() + 2 * el.GetOrder();
+      const IntegrationRule *ir = &IntRules.Get(el.GetGeomType(), order);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint (&ip);
+
+         // reverse-mode differentiation of Eval
+         coords_bar = 0.0;
+         q.EvalRevDiff(Q_bar, trans, ip, coords_bar);
+
+         // get the weighted derivatives using finite difference method
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               double Q_fd = q.Eval(trans, ip);
+               coords(di, n) -= 2.0*eps_fd;
+               Q_fd -= q.Eval(trans, ip);
+               Q_fd /= (2.0 * eps_fd);
+               coords(di, n) += eps_fd;
+               double x_bar_fd = Q_bar * Q_fd;
+
+               REQUIRE(coords_bar(di, n) == Approx(x_bar_fd));
+            }
+         }
+      }
+   }
+}
+
+void runVectorTest(Mesh &mesh, VectorCoefficient &vc)
+{
+   constexpr double eps_fd = 1e-5;
+   std::default_random_engine generator;
+   std::uniform_real_distribution<double> distribution(-1.0,1.0);
+
+   for (int p = 1; p <= 4; ++p)
+   {
+      const int dim = mesh.Dimension();
+      ND_FECollection fec(p, dim);
+      FiniteElementSpace fes(&mesh, &fec);
+
+      const FiniteElement &el = *fes.GetFE(0);
+      IsoparametricTransformation trans;
+      mesh.GetElementTransformation(0, &trans);
+
+      DenseMatrix &coords = trans.GetPointMat();
+      DenseMatrix coords_bar(coords.Height(), coords.Width());
+
+      Vector V_bar(dim), V_fd(dim), V_pert(dim);
+      for (int i = 0; i < V_bar.Size(); ++i)
+      {
+         V_bar(i) = distribution(generator);
+      }
+
+      int order = trans.OrderW() + 2 * el.GetOrder();
+      const IntegrationRule *ir = &IntRules.Get(el.GetGeomType(), order);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint (&ip);
+
+         // reverse-mode differentiation of Eval
+         coords_bar = 0.0;
+         vc.EvalRevDiff(V_bar, trans, ip, coords_bar);
+
+         // get the weighted derivatives using finite difference method
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               vc.Eval(V_fd, trans, ip);
+               coords(di, n) -= 2.0*eps_fd;
+               vc.Eval(V_pert, trans, ip);
+               V_fd -= V_pert;
+               V_fd /= (2.0 * eps_fd);
+               coords(di, n) += eps_fd;
+               double x_bar_fd = V_bar * V_fd;
+
+               REQUIRE(coords_bar(di, n) == Approx(x_bar_fd));
+            }
+         }
+      }
+   }
+}
+
+} // anonymous namespace
+
+namespace coeff_revdiff
+{
+
+TEST_CASE("CoeffRevDiff::FunctionCoefficient::EvalRevDiff_2D")
+{
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh2D(meshStr);
+
+   REQUIRE( mesh2D.GetNE() == 1 );
+   REQUIRE( mesh2D.GetNodes() != NULL );
+
+   FunctionCoefficient c1(scalar_func, scalar_funcRevDiff);
+   runScalarTest(mesh2D, c1);
+
+   FunctionCoefficient c2(scalar_func2, scalar_func2RevDiff);
+   runScalarTest(mesh2D, c2);
+}
+
+TEST_CASE("CoeffRevDiff::ProductCoefficient::EvalRevDiff_2D 1 Coeff")
+{
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh2D(meshStr);
+
+   REQUIRE( mesh2D.GetNE() == 1 );
+   REQUIRE( mesh2D.GetNodes() != NULL );
+
+   FunctionCoefficient c1(scalar_func, scalar_funcRevDiff);
+   ProductCoefficient prod(2.0, c1);
+
+   runScalarTest(mesh2D, prod);
+}
+
+TEST_CASE("CoeffRevDiff::ProductCoefficient::EvalRevDiff_2D 2 Coeffs")
+{
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh2D(meshStr);
+
+   REQUIRE( mesh2D.GetNE() == 1 );
+   REQUIRE( mesh2D.GetNodes() != NULL );
+
+   FunctionCoefficient c1(scalar_func, scalar_funcRevDiff);
+   FunctionCoefficient c2(scalar_func2, scalar_func2RevDiff);
+
+   ProductCoefficient prod(c1, c2);
+
+   runScalarTest(mesh2D, prod);
+}
+
+
+TEST_CASE("CoeffRevDiff::VectorFunctionCoefficient::EvalRevDiff_2D")
+{
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh2D(meshStr);
+
+   REQUIRE( mesh2D.GetNE() == 1 );
+   REQUIRE( mesh2D.GetNodes() != NULL );
+
+   VectorFunctionCoefficient vc2D(2, func2D, func2DRevDiff);
+
+   runVectorTest(mesh2D, vc2D);
+}
+
+TEST_CASE("CoeffRevDiff::VectorFunctionCoefficient::EvalRevDiff_3D")
+{
+   auto mesh3D = Mesh::MakeCartesian3D(2, 2, 2,
+                                       Element::TETRAHEDRON,
+                                       2.0, 1.0, 3.0, true);
+   mesh3D.EnsureNodes();
+
+   VectorFunctionCoefficient vc3D(3, func3D, func3DRevDiff);
+
+   runVectorTest(mesh3D, vc3D);
+}
+
+TEST_CASE("CoeffRevDiff::ScalarVectorProductCoefficient::EvalRevDiff_3D")
+{
+   auto mesh3D = Mesh::MakeCartesian3D(2, 2, 2,
+                                       Element::TETRAHEDRON,
+                                       2.0, 1.0, 3.0, true);
+   mesh3D.EnsureNodes();
+
+   VectorFunctionCoefficient vfc(3, func3D, func3DRevDiff);
+
+   ScalarVectorProductCoefficient vc(2.0, vfc);
+
+   runVectorTest(mesh3D, vc);
+}
+
+} // namespace coeff_revdiff

tests/unit/fem/test_eltrans_revdiff.cpp

@@ -0,0 +1,268 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include "catch.hpp"
+
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <fstream>
+
+using namespace mfem;
+
+// String used to define a single element mesh, with a c-shaped quad element
+std::string mesh_str =
+   "MFEM mesh v1.0"                    "\n\n"
+   "dimension"                           "\n"
+   "2"                                 "\n\n"
+   "elements"                            "\n"
+   "1"                                   "\n"
+   "1 3 0 1 2 3"                       "\n\n"
+   "boundary"                            "\n"
+   "0"                                 "\n\n"
+   "vertices"                            "\n"
+   "4"                                 "\n\n"
+   "nodes"                               "\n"
+   "FiniteElementSpace"                  "\n"
+   "FiniteElementCollection: Quadratic"  "\n"
+   "VDim: 2"                             "\n"
+   "Ordering: 1"                         "\n"
+   "0 0"                                 "\n"
+   "0 2"                                 "\n"
+   "0 6"                                 "\n"
+   "0 8"                                 "\n"
+   "0 1"                                 "\n"
+   "-6 4"                                "\n"
+   "0 7"                                 "\n"
+   "-8 4"                                "\n"
+   "-7 4"                                "\n";
+
+TEST_CASE("IsoparametricTransformation reverse-mode differentiation",
+          "[IsoparametricTransformation]")
+{
+   constexpr double eps_fd = 1e-5;
+
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh(meshStr);
+
+   REQUIRE( mesh.GetNE() == 1 );
+   REQUIRE( mesh.GetNodes() != NULL );
+
+   bool dumpMesh = false;
+   if (dumpMesh)
+   {
+      std::ofstream mesh_ostream("isoparametric-revdiff-mesh.vtk");
+      mesh_ostream.precision(14);
+      int refine = 10;
+      mesh.PrintVTK(mesh_ostream, refine);
+   }
+
+   // Create the transformation and get integration rule
+   IsoparametricTransformation trans;
+   mesh.GetElementTransformation(0, &trans);
+   const int intorder = 5;
+   const IntegrationRule *ir = &IntRules.Get(mesh.GetElementBaseGeometry(0),
+                                             intorder);
+   DenseMatrix &coords = trans.GetPointMat();
+   DenseMatrix coords_bar(coords.Height(), coords.Width());
+
+   SECTION("TransformRevDiff")
+   {
+      // x_bar(i) is the weight on the (i)th entry of the coordinate x;
+      // the values are not important for this test.
+      double x_bar_data[4] = {2.5, -3.2};
+      Vector x_bar(x_bar_data, 2);
+      Vector x_fd(2), x_pert(2);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint(&ip);
+         // reverse-mode differentiation of coordinate transformation
+         coords_bar = 0.0;
+         trans.TransformRevDiff(ip, x_bar, coords_bar);
+         // get the weighted derivatives using finite difference method
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               trans.Transform(ip, x_fd);
+               coords(di, n) -= 2.0*eps_fd;
+               trans.Transform(ip, x_pert);
+               x_fd -= x_pert;
+               x_fd *= 1.0/(2.0*eps_fd);
+               coords(di, n) += eps_fd;
+               double x_bar_fd = 0.0;
+               for (int j = 0; j < x_bar.Size(); ++j)
+               {
+                  x_bar_fd += x_bar(j)*x_fd(j);
+               }
+               REQUIRE(coords_bar(di, n) == Approx(x_bar_fd));
+            }
+         }
+      }
+   }
+
+   SECTION("JacobianRevDiff")
+   {
+      // dFdx_bar(i,j) is the weight on the (i,j)th entry of the Jacobian;
+      // the values are not important for this test.
+      double dFdx_bar_data[4] = {2.0, -3.0, 4.0, -1.0};
+      DenseMatrix dFdx_bar(dFdx_bar_data, 2, 2);
+      DenseMatrix dFdx_fd(2,2);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint(&ip);
+         // reverse-mode differentiation of Jacobian of mapping
+         coords_bar = 0.0;
+         trans.JacobianRevDiff(dFdx_bar, coords_bar);
+         // get the weighted derivatives using finite difference method
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               trans.SetIntPoint(&ip); // force re-evaluation of Jacobian
+               dFdx_fd = trans.Jacobian();
+               coords(di, n) -= 2.0*eps_fd;
+               trans.SetIntPoint(&ip); // force re-evaluation of Jacobian
+               dFdx_fd -= trans.Jacobian();
+               dFdx_fd *= 1.0/(2.0*eps_fd);
+               coords(di, n) += eps_fd;
+               double dFdx_bar_fd = 0.0;
+               for (int j = 0; j < dFdx_bar.Height(); ++j)
+               {
+                  for (int k = 0; k < dFdx_bar.Width(); ++k)
+                  {
+                     dFdx_bar_fd += dFdx_bar(j,k)*dFdx_fd(j,k);
+                  }
+               }
+               REQUIRE(coords_bar(di, n) == Approx(dFdx_bar_fd));
+            }
+         }
+      }
+   }
+
+   SECTION("AdjugateJacobianRevDiff")
+   {
+      // adjJ_bar(i,j) is the weight on the (i,j)th entry of the Adjugate;
+      // the values are not important for this test.
+      double adjJ_bar_data[4] = {2.0, -3.0, 4.0, -1.0};
+      DenseMatrix adjJ_bar(adjJ_bar_data, 2, 2);
+      DenseMatrix adjJ_fd(2,2);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint(&ip);
+         // reverse-mode differentiation of Adjugate of mapping
+         coords_bar = 0.0;
+         trans.AdjugateJacobianRevDiff(adjJ_bar, coords_bar);
+         // get the weighted derivatives using finite difference method
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               trans.SetIntPoint(&ip); // force re-evaluation of Adjugate
+               adjJ_fd = trans.AdjugateJacobian();
+               coords(di, n) -= 2.0*eps_fd;
+               trans.SetIntPoint(&ip); // force re-evaluation of Adjugate
+               adjJ_fd -= trans.AdjugateJacobian();
+               adjJ_fd *= 1.0/(2.0*eps_fd);
+               coords(di, n) += eps_fd;
+               double adjJ_bar_fd = 0.0;
+               for (int j = 0; j < adjJ_bar.Height(); ++j)
+               {
+                  for (int k = 0; k < adjJ_bar.Width(); ++k)
+                  {
+                     adjJ_bar_fd += adjJ_bar(j,k)*adjJ_fd(j,k);
+                  }
+               }
+               REQUIRE(coords_bar(di, n) == Approx(adjJ_bar_fd));
+            }
+         }
+      }
+   }
+
+   SECTION("InverseJacobianRevDiff")
+   {
+      // invJ_bar(i,j) is the weight on the (i,j)th entry of the Inverse;
+      // the values are not important for this test.
+      double invJ_bar_data[4] = {2.0, -3.0, 4.0, -1.0};
+      DenseMatrix invJ_bar(invJ_bar_data, 2, 2);
+      DenseMatrix invJ_fd(2,2);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint(&ip);
+         // reverse-mode differentiation of Inverse of mapping
+         coords_bar = 0.0;
+         trans.InverseJacobianRevDiff(invJ_bar, coords_bar);
+         // get the weighted derivatives using finite difference method
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               trans.SetIntPoint(&ip); // force re-evaluation of Inverse
+               invJ_fd = trans.InverseJacobian();
+               coords(di, n) -= 2.0*eps_fd;
+               trans.SetIntPoint(&ip); // force re-evaluation of Inverse
+               invJ_fd -= trans.InverseJacobian();
+               invJ_fd *= 1.0/(2.0*eps_fd);
+               coords(di, n) += eps_fd;
+               double invJ_bar_fd = 0.0;
+               for (int j = 0; j < invJ_bar.Height(); ++j)
+               {
+                  for (int k = 0; k < invJ_bar.Width(); ++k)
+                  {
+                     invJ_bar_fd += invJ_bar(j,k)*invJ_fd(j,k);
+                  }
+               }
+               REQUIRE(coords_bar(di, n) == Approx(invJ_bar_fd));
+            }
+         }
+      }
+   }
+
+   SECTION("WeightRevDiff")
+   {
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint(&ip);
+         // get the gradient of the Weight() using reverse mode
+         coords_bar = 0.0;
+         trans.WeightRevDiff(coords_bar);
+         // get the gradient of the Weight() using finite difference method
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               trans.SetIntPoint(&ip); // force re-evaluation of Weight
+               double dWeight_fd = trans.Weight();
+               coords(di, n) -= 2.0*eps_fd;
+               trans.SetIntPoint(&ip); // force re-evaluation of Weight
+               dWeight_fd -= trans.Weight();
+               dWeight_fd /= (2.0*eps_fd);
+               coords(di, n) += eps_fd;
+               REQUIRE(coords_bar(di, n) == Approx(dWeight_fd).margin(1e-10));
+            }
+         }
+      }
+   }
+
+}

tests/unit/fem/test_fe_revdiff.cpp

@@ -0,0 +1,419 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include "catch.hpp"
+
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <fstream>
+#include <random>
+
+using namespace mfem;
+
+namespace
+{
+// String used to define a single element mesh, with a c-shaped quad element
+std::string mesh_str =
+   "MFEM mesh v1.0"                    "\n\n"
+   "dimension"                           "\n"
+   "2"                                 "\n\n"
+   "elements"                            "\n"
+   "1"                                   "\n"
+   "1 3 0 1 2 3"                       "\n\n"
+   "boundary"                            "\n"
+   "0"                                 "\n\n"
+   "vertices"                            "\n"
+   "4"                                 "\n\n"
+   "nodes"                               "\n"
+   "FiniteElementSpace"                  "\n"
+   "FiniteElementCollection: Quadratic"  "\n"
+   "VDim: 2"                             "\n"
+   "Ordering: 1"                         "\n"
+   "0 0"                                 "\n"
+   "0 2"                                 "\n"
+   "0 6"                                 "\n"
+   "0 8"                                 "\n"
+   "0 1"                                 "\n"
+   "-6 4"                                "\n"
+   "0 7"                                 "\n"
+   "-8 4"                                "\n"
+   "-7 4"                                "\n";
+
+void func2D(const Vector &x, Vector &y)
+{
+   y.SetSize(2);
+   y(0) = x(0)*x(0) - x(1);
+   y(1) = x(0) * exp(x(1));
+}
+
+void func2DRevDiff(const Vector &x, const Vector &v_bar, Vector &x_bar)
+{
+   x_bar(0) = v_bar(0) * 2*x(0) + v_bar(1) * exp(x(1));
+   x_bar(1) = -v_bar(0) + v_bar(1) * x(0) * exp(x(1));
+}
+
+void func3D(const Vector &x, Vector &y)
+{
+   y.SetSize(3);
+   y(0) = x(0)*x(0) - x(1);
+   y(1) = x(0) * exp(x(1));
+   y(2) = x(2)*x(0) - x(1);
+}
+
+void func3DRevDiff(const Vector &x, const Vector &v_bar, Vector &x_bar)
+{
+   x_bar(0) = v_bar(0) * 2*x(0) + v_bar(1) * exp(x(1)) + v_bar(2)*x(2);
+   x_bar(1) = -v_bar(0) + v_bar(1) * x(0) * exp(x(1)) - v_bar(2);
+   x_bar(2) = v_bar(2) * x(0);
+}
+
+} // anonymous namespace
+
+namespace fe_revdiff
+{
+
+template<typename T>
+void runProjectRevDiffTest(Mesh &mesh, VectorCoefficient &vc);
+
+template<typename T>
+void runCalcPhysShapeRevDiffTest(Mesh &mesh);
+
+template<typename T>
+void runCalcVShapeRevDiffTest(Mesh &mesh);
+
+void runCalcPhysCurlShapeRevDiffTest(Mesh &mesh);
+
+constexpr double eps_fd = 1e-5;
+std::default_random_engine generator;
+std::uniform_real_distribution<double> distribution(-1.0,1.0);
+
+TEST_CASE("VectorFiniteElement::ProjectRevDiff - 2D")
+{
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh2D(meshStr);
+
+   REQUIRE(mesh2D.GetNE() == 1);
+   REQUIRE(mesh2D.GetNodes() != nullptr);
+
+   VectorFunctionCoefficient vc2D(2, func2D, func2DRevDiff);
+
+   runProjectRevDiffTest<RT_FECollection>(mesh2D, vc2D);
+   runProjectRevDiffTest<ND_FECollection>(mesh2D, vc2D);
+}
+
+TEST_CASE("VectorFiniteElement::ProjectRevDiff - 3D")
+{
+   auto mesh3D = Mesh::MakeCartesian3D(2, 2, 2, Element::TETRAHEDRON,
+                                       2.0, 1.0, 3.0, true);
+   // mesh3D.ReorientTetMesh();
+   mesh3D.EnsureNodes();
+
+   VectorFunctionCoefficient vc3D(3, func3D, func3DRevDiff);
+
+   runProjectRevDiffTest<RT_FECollection>(mesh3D, vc3D);
+   runProjectRevDiffTest<ND_FECollection>(mesh3D, vc3D);
+}
+
+TEST_CASE("FiniteElement::CalcPhysShapeRevDiff")
+{
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh2D(meshStr);
+   REQUIRE(mesh2D.GetNE() == 1);
+   REQUIRE(mesh2D.GetNodes() != nullptr);
+   runCalcPhysShapeRevDiffTest<H1_FECollection>(mesh2D);
+   runCalcPhysShapeRevDiffTest<L2_FECollection>(mesh2D);
+}
+
+TEST_CASE("VectorFiniteElement::CalcVShape_RTRevDiff - 2D")
+{
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh2D(meshStr);
+   REQUIRE(mesh2D.GetNE() == 1);
+   REQUIRE(mesh2D.GetNodes() != nullptr);
+   runCalcVShapeRevDiffTest<RT_FECollection>(mesh2D);
+}
+
+TEST_CASE("VectorFiniteElement::CalcVShape_NDRevDiff - 2D")
+{
+   // Create quadratic mesh with single C-shaped quadrilateral
+   std::stringstream meshStr;
+   meshStr << mesh_str;
+   Mesh mesh2D(meshStr);
+   REQUIRE(mesh2D.GetNE() == 1);
+   REQUIRE(mesh2D.GetNodes() != nullptr);
+   runCalcVShapeRevDiffTest<ND_FECollection>(mesh2D);
+}
+
+TEST_CASE("VectorFiniteElement::CalcVShape_RTRevDiff - 3D")
+{
+   auto mesh3D = Mesh::MakeCartesian3D(2, 2, 2, Element::TETRAHEDRON,
+                                       2.0, 1.0, 3.0, true);
+   // mesh3D.ReorientTetMesh();
+   mesh3D.EnsureNodes();
+   runCalcVShapeRevDiffTest<RT_FECollection>(mesh3D);
+}
+
+TEST_CASE("VectorFiniteElement::CalcVShape_NDRevDiff - 3D")
+{
+   auto mesh3D = Mesh::MakeCartesian3D(2, 2, 2, Element::TETRAHEDRON,
+                                       2.0, 1.0, 3.0, true);
+   // mesh3D.ReorientTetMesh();
+   mesh3D.EnsureNodes();
+   runCalcVShapeRevDiffTest<ND_FECollection>(mesh3D);
+}
+
+TEST_CASE("FiniteElement::CalcPhysCurlShapeRevDiff - 3D")
+{
+   auto mesh3D = Mesh::MakeCartesian3D(2, 2, 2, Element::TETRAHEDRON,
+                                       2.0, 1.0, 3.0, true);
+   // mesh3D.ReorientTetMesh();
+   mesh3D.EnsureNodes();
+   runCalcPhysCurlShapeRevDiffTest(mesh3D);
+}
+
+template<typename T>
+void runProjectRevDiffTest(Mesh &mesh, VectorCoefficient &vc)
+{
+   for (int p = 1; p <= 4; ++p)
+   {
+      const int dim = mesh.Dimension();
+      T fec(p, dim);
+      FiniteElementSpace fes(&mesh, &fec);
+
+      const FiniteElement &el = *fes.GetFE(0);
+      IsoparametricTransformation trans;
+      mesh.GetElementTransformation(0, &trans);
+
+      // P_bar is the vector contracted with the derivative of the projection
+      // the values are not important for this test
+      const int dof = el.GetDof();
+      Vector P_bar(dof);
+      for (int i = 0; i < P_bar.Size(); ++i)
+      {
+         P_bar(i) = distribution(generator);
+      }
+
+      // reverse-mode differentiation of projection
+      DenseMatrix &coords = trans.GetPointMat();
+      DenseMatrix coords_bar(coords.Height(), coords.Width());
+      coords_bar = 0.0;
+      el.ProjectRevDiff(P_bar, vc, trans, coords_bar);
+
+      // get the weighted derivatives using finite difference method
+      Vector dofs_fd(dof), dofs_pert(dof);
+      for (int n = 0; n < coords.Width(); ++n)
+      {
+         for (int di = 0; di < coords.Height(); ++di)
+         {
+            coords(di, n) += eps_fd;
+            trans.Reset();
+            el.Project(vc, trans, dofs_fd);
+            coords(di, n) -= 2.0 * eps_fd;
+            trans.Reset();
+            el.Project(vc, trans, dofs_pert);
+            dofs_fd -= dofs_pert;
+            dofs_fd *= 1.0 / (2.0 * eps_fd);
+            coords(di, n) += eps_fd;
+            double x_bar_fd = P_bar * dofs_fd;
+
+            REQUIRE(coords_bar(di, n) == Approx(x_bar_fd));
+         }
+      }
+   }
+}
+
+template<typename T>
+void runCalcPhysShapeRevDiffTest(Mesh &mesh)
+{
+   for (int p = 1; p <= 4; ++p)
+   {
+      const int dim = mesh.Dimension();
+      T fec(p, dim);
+      FiniteElementSpace fes(&mesh, &fec);
+
+      const FiniteElement &el = *fes.GetFE(0);
+      IsoparametricTransformation trans;
+      mesh.GetElementTransformation(0, &trans);
+
+      int order = trans.OrderW() + 2 * el.GetOrder();
+      const IntegrationRule *ir = &IntRules.Get(el.GetGeomType(), order);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint(&ip);
+
+         const int dof = el.GetDof();
+         Vector shape_bar(dof);
+         for (int k = 0; k < shape_bar.Size(); ++k)
+         {
+            shape_bar(k) = distribution(generator);
+         }
+
+         // reverse-mode differentiation CalcVShape
+         DenseMatrix &coords = trans.GetPointMat();
+         DenseMatrix coords_bar(coords.Height(), coords.Width());
+         coords_bar = 0.0;
+         el.CalcPhysShapeRevDiff(trans, shape_bar, coords_bar);
+
+         // get the weighted derivatives using finite difference method
+         Vector shape_fd(dof), shape_pert(dof);
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               trans.Reset();
+               el.CalcPhysShape(trans, shape_fd);
+               coords(di, n) -= 2.0 * eps_fd;
+               trans.Reset();
+               el.CalcPhysShape(trans, shape_pert);
+               shape_fd -= shape_pert;
+               shape_fd *= 1.0 / (2.0 * eps_fd);
+               coords(di, n) += eps_fd;
+               double x_bar_fd = shape_bar * shape_fd;
+
+               REQUIRE(coords_bar(di, n) == Approx(x_bar_fd));
+            }
+         }
+      }
+   }
+}
+
+template<typename T>
+void runCalcVShapeRevDiffTest(Mesh &mesh)
+{
+   for (int p = 1; p <= 4; ++p)
+   {
+      const int dim = mesh.Dimension();
+      T fec(p, dim);
+      FiniteElementSpace fes(&mesh, &fec);
+
+      const FiniteElement &el = *fes.GetFE(0);
+      IsoparametricTransformation trans;
+      mesh.GetElementTransformation(0, &trans);
+
+      int order = trans.OrderW() + 2 * el.GetOrder();
+      const IntegrationRule *ir = &IntRules.Get(el.GetGeomType(), order);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint(&ip);
+
+         const int dof = el.GetDof();
+         const int el_dim = el.GetDim();
+         DenseMatrix vshape_bar(dof, el_dim);
+         for (int k = 0; k < vshape_bar.Width(); ++k)
+         {
+            for (int j = 0; j < vshape_bar.Height(); ++j)
+            {
+               vshape_bar(j, k) = distribution(generator);
+            }
+         }
+
+         // reverse-mode differentiation CalcVShape
+         DenseMatrix &coords = trans.GetPointMat();
+         DenseMatrix coords_bar(coords.Height(), coords.Width());
+         coords_bar = 0.0;
+         el.CalcVShapeRevDiff(trans, vshape_bar, coords_bar);
+
+         // get the weighted derivatives using finite difference method
+         DenseMatrix vshape_fd(dof, el_dim), vshape_pert(dof, el_dim);
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               trans.Reset();
+               el.CalcVShape(trans, vshape_fd);
+               coords(di, n) -= 2.0 * eps_fd;
+               trans.Reset();
+               el.CalcVShape(trans, vshape_pert);
+               vshape_fd -= vshape_pert;
+               vshape_fd *= 1.0 / (2.0 * eps_fd);
+               coords(di, n) += eps_fd;
+               double x_bar_fd = vshape_bar * vshape_fd;
+
+               REQUIRE(coords_bar(di, n) == Approx(x_bar_fd));
+            }
+         }
+      }
+   }
+}
+
+void runCalcPhysCurlShapeRevDiffTest(Mesh &mesh)
+{
+   for (int p = 1; p <= 4; ++p)
+   {
+      const int dim = mesh.Dimension();
+      ND_FECollection fec(p, dim);
+      FiniteElementSpace fes(&mesh, &fec);
+
+      const FiniteElement &el = *fes.GetFE(0);
+      IsoparametricTransformation trans;
+      mesh.GetElementTransformation(0, &trans);
+
+      int order = trans.OrderW() + 2 * el.GetOrder();
+      const IntegrationRule *ir = &IntRules.Get(el.GetGeomType(), order);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         trans.SetIntPoint(&ip);
+
+         const int dof = el.GetDof();
+         const int el_dim = el.GetDim();
+         DenseMatrix curlshape_bar(dof, el_dim);
+         for (int k = 0; k < curlshape_bar.Width(); ++k)
+         {
+            for (int j = 0; j < curlshape_bar.Height(); ++j)
+            {
+               curlshape_bar(j, k) = distribution(generator);
+            }
+         }
+
+         // reverse-mode differentiation CalcPhysCurlShape
+         DenseMatrix &coords = trans.GetPointMat();
+         DenseMatrix coords_bar(coords.Height(), coords.Width());
+         coords_bar = 0.0;
+         el.CalcPhysCurlShapeRevDiff(trans, curlshape_bar, coords_bar);
+
+         // get the weighted derivatives using finite difference method
+         DenseMatrix curlshape_fd(dof, el_dim), curlshape_pert(dof, el_dim);
+         for (int n = 0; n < coords.Width(); ++n)
+         {
+            for (int di = 0; di < coords.Height(); ++di)
+            {
+               coords(di, n) += eps_fd;
+               trans.Reset();
+               el.CalcPhysCurlShape(trans, curlshape_fd);
+               coords(di, n) -= 2.0 * eps_fd;
+               trans.Reset();
+               el.CalcPhysCurlShape(trans, curlshape_pert);
+               curlshape_fd -= curlshape_pert;
+               curlshape_fd *= 1.0 / (2.0 * eps_fd);
+               coords(di, n) += eps_fd;
+               double x_bar_fd = curlshape_bar * curlshape_fd;
+
+               REQUIRE(coords_bar(di, n) == Approx(x_bar_fd));
+            }
+         }
+      }
+   }
+}
+
+} // namespace fe_revdiff

tests/unit/fem/test_lexicographic_ordering.cpp

@@ -45,7 +45,7 @@ template <typename T> void VerifyOrdering(int order)
    VerifyOrdering(el);
 }
 
-TEST_CASE("Lexicographic Ordering", "[FiniteElement,Geometry]")
+TEST_CASE("Lexicographic Ordering", "[FiniteElement][Geometry]")
 {
    auto order = GENERATE(1, 2, 3, 4, 5, 6);
    VerifyOrdering<H1_SegmentElement>(order);

tests/unit/fem/test_pa_coeff.cpp

@@ -814,7 +814,7 @@ TEST_CASE("Hcurl/Hdiv mixed pa_coeff",
                      y_assembly.SetSize(y_mat.Size());
                      y_pa.SetSize(y_mat.Size());
 
-                     A_explicit.BuildTranspose();
+                     A_explicit.EnsureMultTranspose();
                      paform->MultTranspose(*xin, y_pa);
                      assemblyform->MultTranspose(*xin, y_assembly);
                      A_explicit.MultTranspose(*xin, y_mat);

tests/unit/fem/test_pa_grad.cpp

@@ -78,7 +78,7 @@ double compare_pa_assembly(int dim, int num_elements, int order, bool transpose)
    xv.Randomize();
    if (transpose)
    {
-      assembled_grad_mat.BuildTranspose();
+      assembled_grad_mat.EnsureMultTranspose();
       assembled_grad_mat.MultTranspose(xv, assembled_y);
       pa_grad.MultTranspose(xv, pa_y);
    }

tests/unit/fem/test_pa_idinterp.cpp

@@ -90,7 +90,7 @@ double compare_pa_id_assembly(int dim, int num_elements, int order,
    x.Randomize();
    if (transpose)
    {
-      assembled_id_mat.BuildTranspose();
+      assembled_id_mat.EnsureMultTranspose();
       assembled_id_mat.MultTranspose(x, assembled_y);
       pa_id.MultTranspose(x, pa_y);
    }

tests/unit/fem/test_pa_kernels.cpp

@@ -349,18 +349,25 @@ void AddConvectionIntegrators(BilinearForm &k, Coefficient &rho,
    }
 }
 
-void test_pa_convection(const char *meshname, int order, int prob)
+void test_pa_convection(const std::string &meshname, int order, int prob,
+                        int refinement)
 {
-   INFO("mesh=" << meshname << ", order=" << order << ", prob=" << prob);
-   Mesh mesh(meshname, 1, 1);
+   INFO("mesh=" << meshname << ", order=" << order << ", prob=" << prob
+        << ", refinement=" << refinement );
+   Mesh mesh(meshname.c_str(), 1, 1);
    mesh.EnsureNodes();
    mesh.SetCurvature(mesh.GetNodalFESpace()->GetElementOrder(0));
+   for (int r = 0; r < refinement; r++)
+   {
+      mesh.RandomRefinement(0.6,false,1,4);
+   }
    int dim = mesh.Dimension();
 
    FiniteElementCollection *fec;
    if (prob)
    {
-      fec = new L2_FECollection(order, dim, BasisType::GaussLobatto);
+      auto basis = prob==3 ? BasisType::Positive : BasisType::GaussLobatto;
+      fec = new L2_FECollection(order, dim, basis);
    }
    else
    {
@@ -380,7 +387,7 @@ void test_pa_convection(const char *meshname, int order, int prob)
    Coefficient *rho;
 
    // prob: 0: CG, 1: DG continuous coeff, 2: DG discontinuous coeff
-   if (prob == 2)
+   if (prob >= 2)
    {
       vel_gf.Randomize(1);
       vel_coeff = new VectorGridFunctionCoefficient(&vel_gf);
@@ -420,37 +427,80 @@ void test_pa_convection(const char *meshname, int order, int prob)
 }
 
 // Basic unit test for convection
-TEST_CASE("PA Convection", "[PartialAssembly]")
+TEST_CASE("PA Convection", "[PartialAssembly][MFEMData]")
 {
-   // prob: 0: CG, 1: DG continuous coeff, 2: DG discontinuous coeff
-   auto prob = GENERATE(0, 1, 2);
-   auto order_2d = GENERATE(2, 3, 4);
+   // prob:
+   // - 0: CG,
+   // - 1: DG continuous coeff,
+   // - 2: DG discontinuous coeff,
+   // - 3: DG Bernstein discontinuous coeff.
+   auto prob = GENERATE(0, 1, 2, 3);
+   auto order_2d = GENERATE(2);
    auto order_3d = GENERATE(2);
+   // refinement > 0 => Non-conforming mesh
+   auto refinement_2d = GENERATE(0,1);
+   auto refinement_3d = GENERATE(0,1);
 
    SECTION("2D")
    {
-      test_pa_convection("../../data/periodic-square.mesh", order_2d, prob);
-      test_pa_convection("../../data/periodic-hexagon.mesh", order_2d, prob);
-      test_pa_convection("../../data/star-q3.mesh", order_2d, prob);
+      test_pa_convection("../../data/periodic-square.mesh", order_2d, prob,
+                         refinement_2d);
+      if (launch_all_non_regression_tests)
+      {
+         test_pa_convection("../../data/periodic-hexagon.mesh", order_2d, prob,
+                            refinement_2d);
+         test_pa_convection("../../data/star-q3.mesh", order_2d, prob,
+                            refinement_2d);
+         test_pa_convection(mfem_data_dir+"/gmsh/v22/unstructured_quad.v22.msh",
+                            order_2d, prob, refinement_2d);
+      }
    }
 
    SECTION("3D")
    {
-      test_pa_convection("../../data/periodic-cube.mesh", order_3d, prob);
-      test_pa_convection("../../data/fichera-q3.mesh", order_3d, prob);
-   }
-
-   // Test AMR cases (DG not implemented)
-   SECTION("AMR 2D")
-   {
-      test_pa_convection("../../data/amr-quad.mesh", order_2d, 0);
-   }
-
-   SECTION("AMR 3D")
-   {
-      test_pa_convection("../../data/fichera-amr.mesh", order_3d, 0);
+      test_pa_convection("../../data/periodic-cube.mesh", order_3d, prob,
+                         refinement_3d);
+      if (launch_all_non_regression_tests)
+      {
+         test_pa_convection("../../data/fichera-q3.mesh", order_3d, prob,
+                            refinement_3d);
+         test_pa_convection(mfem_data_dir+"/gmsh/v22/unstructured_hex.v22.msh",
+                            order_3d, prob, refinement_3d);
+      }
    }
 
 } // test case
 
+TEST_CASE("PA Mass", "[PartialAssembly]")
+{
+   auto fname = GENERATE("../../data/star.mesh", "../../data/star-q3.mesh",
+                         "../../data/fichera.mesh", "../../data/fichera-q3.mesh");
+   auto map_type = GENERATE(FiniteElement::VALUE, FiniteElement::INTEGRAL);
+   int order = 2;
+
+   Mesh mesh(fname);
+   int dim = mesh.Dimension();
+   L2_FECollection fec(order, dim, BasisType::GaussLobatto, map_type);
+   FiniteElementSpace fes(&mesh, &fec);
+
+   GridFunction x(&fes), y_fa(&fes), y_pa(&fes);
+   x.Randomize(1);
+
+   BilinearForm blf_fa(&fes);
+   blf_fa.AddDomainIntegrator(new MassIntegrator);
+   blf_fa.Assemble();
+   blf_fa.Finalize();
+   blf_fa.Mult(x, y_fa);
+
+   BilinearForm blf_pa(&fes);
+   blf_pa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   blf_pa.AddDomainIntegrator(new MassIntegrator);
+   blf_pa.Assemble();
+   blf_pa.Mult(x, y_pa);
+
+   y_fa -= y_pa;
+
+   REQUIRE(y_fa.Normlinf() == MFEM_Approx(0.0));
+} // test case
+
 } // namespace pa_kernels

tests/unit/fem/test_transfer.cpp

@@ -14,16 +14,54 @@
 
 using namespace mfem;
 
-int dimension;
-double coeff(const Vector& x)
+int RandomPRefinement(FiniteElementSpace & fes)
 {
+   Mesh *mesh = fes.GetMesh();
+   int maxorder = 0;
+   for (int i = 0; i < mesh->GetNE(); i++)
+   {
+      const int order = fes.GetElementOrder(i);
+      maxorder = std::max(maxorder,order);
+      if ((double) rand() / RAND_MAX < 0.5)
+      {
+         fes.SetElementOrder(i,order+1);
+         maxorder = std::max(maxorder,order+1);
+      }
+   }
+   fes.Update(false);
+   return maxorder;
+}
+
+
+int dimension;
+int coeff_order;
+double coeff(const Vector& X)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
    if (dimension == 2)
    {
-      return 1.1 * x[0] + 2.0 * x[1];
+      if (coeff_order == 1)
+      {
+         return 1.1 * x + 2.0 * y;
+      }
+      else
+      {
+         return (1.-x)*x*(1.-y)*y;
+      }
    }
    else
    {
-      return 1.1 * x[0] + 2.0 * x[1] + 3.0 * x[2];
+      z = X[2];
+      if (coeff_order == 1)
+      {
+         return 1.1 * x + 2.0 * y + 3.0 * z;
+      }
+      else
+      {
+         return (1.-x)*x*(1.-y)*y*(1.-z)*z;
+      }
    }
 }
 
@@ -37,318 +75,475 @@ void vectorcoeff(const Vector& x, Vector& y)
    }
 }
 
+enum class VecSpace { H1, VectorH1, ND, RT };
 
-TEST_CASE("transfer")
+std::string VecSpaceName(VecSpace vectorspace)
 {
-   for (int vectorspace = 0; vectorspace <= 3; ++vectorspace)
+   switch (vectorspace)
    {
-      for (dimension = 2; dimension <= 3; ++dimension)
-      {
-         for (int elementType = 0; elementType <= 1; ++elementType)
-         {
-            for (int ne = 1; ne <= 3; ++ne)
-            {
-               for (int order = 1; order <= 4; order *= 2)
-               {
-                  for (int geometric = 0; geometric <= 1; ++geometric)
-                  {
-                     int fineOrder = (geometric == 1) ? order : 2 * order;
-
-                     std::cout << "Testing transfer:\n"
-                               << "  Vectorspace:  " << vectorspace << "\n"
-                               << "  Dimension:    " << dimension << "\n"
-                               << "  Element type: " << elementType << "\n"
-                               << "  Elements:     " << std::pow(ne, dimension) << "\n"
-                               << "  Coarse order: " << order << "\n"
-                               << "  Fine order:   " << fineOrder << "\n"
-                               << "  Geometric:    " << geometric << "\n";
-
-                     Mesh mesh;
-                     if (dimension == 2)
-                     {
-                        Element::Type type = Element::QUADRILATERAL;
-                        if (elementType != 0)
-                        {
-                           type = Element::TRIANGLE;
-                        }
-                        mesh = Mesh::MakeCartesian2D(ne, ne, type, 1, 1.0, 1.0);
-                     }
-                     else
-                     {
-                        Element::Type type = Element::HEXAHEDRON;
-                        if (elementType != 0)
-                        {
-                           type = Element::TETRAHEDRON;
-                        }
-                        mesh = Mesh::MakeCartesian3D(ne, ne, ne, type, 1.0, 1.0, 1.0);
-                     }
-                     FiniteElementCollection* c_h1_fec = nullptr;
-                     FiniteElementCollection* f_h1_fec = nullptr;
-
-                     if (vectorspace < 2)
-                     {
-                        c_h1_fec = new H1_FECollection(order, dimension);
-                        f_h1_fec = (geometric == 1) ? c_h1_fec : new
-                                   H1_FECollection(fineOrder, dimension);
-                     }
-                     else if (vectorspace == 2)
-                     {
-                        c_h1_fec = new ND_FECollection(order+1, dimension);
-                        f_h1_fec = (geometric == 1) ? c_h1_fec : new
-                                   ND_FECollection(fineOrder, dimension);
-                     }
-                     else
-                     {
-                        c_h1_fec = new RT_FECollection(order, dimension);
-                        f_h1_fec = (geometric == 1) ? c_h1_fec : new
-                                   RT_FECollection(fineOrder, dimension);
-                     }
-
-                     Mesh fineMesh(mesh);
-                     if (geometric)
-                     {
-                        fineMesh.UniformRefinement();
-                     }
-
-                     int spaceDimension = 1;
-
-                     if (vectorspace == 1)
-                     {
-                        spaceDimension = dimension;
-                     }
-
-                     FiniteElementSpace* c_h1_fespace =
-                        new FiniteElementSpace(&mesh, c_h1_fec, spaceDimension);
-                     FiniteElementSpace* f_h1_fespace =
-                        new FiniteElementSpace(&fineMesh, f_h1_fec,spaceDimension);
-
-
-                     Operator* referenceOperator = nullptr;
-
-                     if (geometric == 0)
-                     {
-                        referenceOperator = new PRefinementTransferOperator(*c_h1_fespace,
-                                                                            *f_h1_fespace);
-                     }
-                     else
-                     {
-                        OperatorPtr P(Operator::ANY_TYPE);
-                        f_h1_fespace->GetTransferOperator(*c_h1_fespace, P);
-                        P.SetOperatorOwner(false);
-                        referenceOperator = P.Ptr();
-                     }
-
-                     TransferOperator testTransferOperator(*c_h1_fespace, *f_h1_fespace);
-                     GridFunction X(c_h1_fespace);
-                     GridFunction X_cmp(c_h1_fespace);
-                     GridFunction Y_exact(f_h1_fespace);
-                     GridFunction Y_std(f_h1_fespace);
-                     GridFunction Y_test(f_h1_fespace);
-
-                     if (vectorspace == 0)
-                     {
-                        FunctionCoefficient funcCoeff(&coeff);
-                        X.ProjectCoefficient(funcCoeff);
-                        Y_exact.ProjectCoefficient(funcCoeff);
-                     }
-                     else
-                     {
-                        VectorFunctionCoefficient funcCoeff(dimension, &vectorcoeff);
-                        X.ProjectCoefficient(funcCoeff);
-                        Y_exact.ProjectCoefficient(funcCoeff);
-                     }
-
-                     Y_std = 0.0;
-                     Y_test = 0.0;
-
-                     referenceOperator->Mult(X, Y_std);
-
-                     Y_std -= Y_exact;
-                     REQUIRE(Y_std.Norml2() < 1e-12 * Y_exact.Norml2());
-
-                     if (vectorspace == 0)
-                     {
-                        testTransferOperator.Mult(X, Y_test);
-
-                        Y_test -= Y_exact;
-                        REQUIRE(Y_test.Norml2() < 1e-12 * Y_exact.Norml2());
-                     }
-
-                     if (vectorspace == 0)
-                     {
-                        referenceOperator->MultTranspose(Y_exact, X);
-                        testTransferOperator.MultTranspose(Y_exact, X_cmp);
-
-                        X -= X_cmp;
-                        REQUIRE(X.Norml2() < 1e-12 * X_cmp.Norml2());
-                     }
-
-                     delete referenceOperator;
-                     delete f_h1_fespace;
-                     delete c_h1_fespace;
-                     if (geometric == 0)
-                     {
-                        delete f_h1_fec;
-                     }
-                     delete c_h1_fec;
-                  }
-               }
-            }
-         }
-      }
+      case VecSpace::H1: return "H1";
+      case VecSpace::VectorH1: return "Vector H1";
+      case VecSpace::ND: return "Nedelec";
+      case VecSpace::RT: return "Raviart-Thomas";
    }
+   return "";
+}
+
+TEST_CASE("Transfer", "[Transfer]")
+{
+   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1, VecSpace::ND,
+                               VecSpace::RT);
+   auto geometric = GENERATE(true, false);
+   auto simplex = GENERATE(true, false);
+   dimension = GENERATE(2, 3);
+
+   int order = 2;
+   int ne = 2;
+
+   int fineOrder = geometric ? order : 2*order;
+
+   // Log test case information
+   int total_ne = std::pow(ne, dimension);
+   CAPTURE(VecSpaceName(vectorspace), dimension, simplex, total_ne, order,
+           fineOrder, geometric);
+
+   Mesh mesh;
+   if (dimension == 2)
+   {
+      Element::Type type = simplex ? Element::TRIANGLE : Element::QUADRILATERAL;
+      mesh = Mesh::MakeCartesian2D(ne, ne, type, 1, 1.0, 1.0);
+   }
+   else
+   {
+      Element::Type type = simplex ? Element::TETRAHEDRON : Element::HEXAHEDRON;
+      mesh = Mesh::MakeCartesian3D(ne, ne, ne, type, 1.0, 1.0, 1.0);
+   }
+   FiniteElementCollection *c_fec = nullptr;
+   FiniteElementCollection *f_fec = nullptr;
+
+   switch (vectorspace)
+   {
+      case VecSpace::H1:
+      case VecSpace::VectorH1:
+         c_fec = new H1_FECollection(order, dimension);
+         f_fec = geometric ? c_fec : new H1_FECollection(fineOrder, dimension);
+         break;
+      case VecSpace::ND:
+         c_fec = new ND_FECollection(order+1, dimension);
+         f_fec = geometric ? c_fec : new ND_FECollection(fineOrder, dimension);
+         break;
+      case VecSpace::RT:
+         c_fec = new RT_FECollection(order, dimension);
+         f_fec = geometric ? c_fec : new RT_FECollection(fineOrder, dimension);
+         break;
+   }
+
+   Mesh fineMesh(mesh);
+   if (geometric)
+   {
+      fineMesh.UniformRefinement();
+   }
+
+   int spaceDimension = (vectorspace == VecSpace::VectorH1) ? dimension : 1;
+
+   FiniteElementSpace *c_fespace =
+      new FiniteElementSpace(&mesh, c_fec, spaceDimension);
+   FiniteElementSpace *f_fespace =
+      new FiniteElementSpace(&fineMesh, f_fec,spaceDimension);
+
+   Operator* referenceOperator = nullptr;
+
+   if (!geometric)
+   {
+      referenceOperator = new PRefinementTransferOperator(*c_fespace,
+                                                          *f_fespace);
+   }
+   else
+   {
+      OperatorPtr P(Operator::ANY_TYPE);
+      f_fespace->GetTransferOperator(*c_fespace, P);
+      P.SetOperatorOwner(false);
+      referenceOperator = P.Ptr();
+   }
+
+   TransferOperator testTransferOperator(*c_fespace, *f_fespace);
+   GridFunction X(c_fespace);
+   GridFunction X_cmp(c_fespace);
+   GridFunction Y_exact(f_fespace);
+   GridFunction Y_std(f_fespace);
+   GridFunction Y_test(f_fespace);
+   coeff_order = 1;
+   if (vectorspace == VecSpace::H1)
+   {
+      FunctionCoefficient funcCoeff(&coeff);
+      X.ProjectCoefficient(funcCoeff);
+      Y_exact.ProjectCoefficient(funcCoeff);
+   }
+   else
+   {
+      VectorFunctionCoefficient funcCoeff(dimension, &vectorcoeff);
+      X.ProjectCoefficient(funcCoeff);
+      Y_exact.ProjectCoefficient(funcCoeff);
+   }
+
+   Y_std = 0.0;
+   Y_test = 0.0;
+
+   referenceOperator->Mult(X, Y_std);
+
+   Y_std -= Y_exact;
+   REQUIRE(Y_std.Norml2() < 1e-12 * Y_exact.Norml2());
+
+   testTransferOperator.Mult(X, Y_test);
+
+   Y_test -= Y_exact;
+   REQUIRE(Y_test.Norml2() < 1e-12 * Y_exact.Norml2());
+
+   referenceOperator->MultTranspose(Y_exact, X);
+   testTransferOperator.MultTranspose(Y_exact, X_cmp);
+
+   X -= X_cmp;
+   REQUIRE(X.Norml2() < 1e-12 * X_cmp.Norml2());
+
+   delete referenceOperator;
+   delete f_fespace;
+   delete c_fespace;
+   if (geometric == 0)
+   {
+      delete f_fec;
+   }
+   delete c_fec;
+}
+
+TEST_CASE("Variable Order Transfer", "[Transfer][VariableOrder]")
+{
+   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1, VecSpace::ND,
+                               VecSpace::RT);
+   dimension = GENERATE(2, 3);
+
+   int ne = 2;
+   int order = 2;
+
+   // Log test case information
+   int total_ne = pow(ne, dimension);
+   CAPTURE(VecSpaceName(vectorspace), dimension, total_ne, order);
+
+   Mesh mesh;
+   if (dimension == 2)
+   {
+      Element::Type type = Element::QUADRILATERAL;
+      mesh = Mesh::MakeCartesian2D(ne, ne, type, 1, 1.0, 1.0);
+   }
+   else
+   {
+      Element::Type type = Element::HEXAHEDRON;
+      mesh = Mesh::MakeCartesian3D(ne, ne, ne, type, 1.0, 1.0, 1.0);
+   }
+   FiniteElementCollection* c_fec = nullptr;
+   FiniteElementCollection* f_fec = nullptr;
+   switch (vectorspace)
+   {
+      case VecSpace::H1:
+      case VecSpace::VectorH1:
+         c_fec = new H1_FECollection(order, dimension);
+         f_fec = new H1_FECollection(order, dimension);
+         break;
+      case VecSpace::ND:
+         c_fec = new ND_FECollection(order+1, dimension);
+         f_fec = new ND_FECollection(order+1, dimension);
+         break;
+      case VecSpace::RT:
+         c_fec = new RT_FECollection(order, dimension);
+         f_fec = new RT_FECollection(order, dimension);
+         break;
+   }
+
+   mesh.EnsureNCMesh();
+   mesh.RandomRefinement(0.5);
+
+   int spaceDimension = (vectorspace == VecSpace::VectorH1) ? dimension : 1;
+
+   FiniteElementSpace *c_fespace =
+      new FiniteElementSpace(&mesh, c_fec, spaceDimension);
+   FiniteElementSpace *f_fespace =
+      new FiniteElementSpace(&mesh, f_fec,spaceDimension);
+
+   Operator* referenceOperator = nullptr;
+
+   referenceOperator = new PRefinementTransferOperator(*c_fespace,
+                                                       *f_fespace);
+
+   TransferOperator testTransferOperator(*c_fespace, *f_fespace);
+   GridFunction X(c_fespace); X = 0.;
+   GridFunction X_cmp(c_fespace); X_cmp = 0.;
+   GridFunction Y_exact(f_fespace); Y_exact = 0.;
+   GridFunction Y_std(f_fespace); Y_std = 0.;
+   GridFunction Y_test(f_fespace); Y_test = 0.;
+   coeff_order = std::min(2,order);
+   if (vectorspace == VecSpace::H1)
+   {
+      FunctionCoefficient funcCoeff(&coeff);
+      X.ProjectCoefficient(funcCoeff);
+      Y_exact.ProjectCoefficient(funcCoeff);
+   }
+   else
+   {
+      VectorFunctionCoefficient funcCoeff(dimension, &vectorcoeff);
+      X.ProjectCoefficient(funcCoeff);
+      Y_exact.ProjectCoefficient(funcCoeff);
+   }
+
+   Y_std = 0.0;
+   Y_test = 0.0;
+
+   referenceOperator->Mult(X, Y_std);
+   Y_std -= Y_exact;
+   REQUIRE(Y_std.Norml2() < 1e-12 * Y_exact.Norml2());
+
+   testTransferOperator.Mult(X, Y_test);
+
+   Y_test -= Y_exact;
+   REQUIRE(Y_test.Norml2() < 1e-12 * Y_exact.Norml2());
+
+   referenceOperator->MultTranspose(Y_exact, X);
+   testTransferOperator.MultTranspose(Y_exact, X_cmp);
+
+   X -= X_cmp;
+   REQUIRE(X.Norml2() < 1e-12 * X_cmp.Norml2());
+
+   delete referenceOperator;
+   delete f_fespace;
+   delete c_fespace;
+   delete f_fec;
+   delete c_fec;
+}
+
+TEST_CASE("Variable Order True Transfer", "[Transfer][VariableOrder]")
+{
+   auto vectorspace = GENERATE(VecSpace::H1, VecSpace::VectorH1);
+   dimension = GENERATE(2, 3);
+
+   int ne = 2;
+   int order = 2;
+
+   // Log test case information
+   CAPTURE(VecSpaceName(vectorspace), dimension, order);
+
+   Mesh mesh;
+   if (dimension == 2)
+   {
+      Element::Type type = Element::QUADRILATERAL;
+      mesh = Mesh::MakeCartesian2D(ne, ne, type, 1, 1.0, 1.0);
+   }
+   else
+   {
+      Element::Type type = Element::HEXAHEDRON;
+      mesh = Mesh::MakeCartesian3D(ne, ne, ne, type, 1.0, 1.0, 1.0);
+   }
+   FiniteElementCollection *c_fec = nullptr;
+   FiniteElementCollection *f_fec = nullptr;
+   c_fec = new H1_FECollection(order, dimension);
+   f_fec = new H1_FECollection(order, dimension);
+   mesh.EnsureNCMesh();
+   mesh.RandomRefinement(0.5);
+   int spaceDimension = (vectorspace == VecSpace::VectorH1) ? dimension : 1;
+
+   FiniteElementSpace *c_fespace =
+      new FiniteElementSpace(&mesh, c_fec, spaceDimension);
+   FiniteElementSpace *f_fespace =
+      new FiniteElementSpace(&mesh, f_fec,spaceDimension);
+
+   RandomPRefinement(*f_fespace);
+
+   const SparseMatrix *Rc = c_fespace->GetRestrictionMatrix();
+   TrueTransferOperator T(*c_fespace, *f_fespace);
+   GridFunction xc(c_fespace);
+   Vector Xc(c_fespace->GetTrueVSize());
+   Vector Diff(c_fespace->GetTrueVSize());
+   Vector Yc(c_fespace->GetTrueVSize());
+   Vector Xf(f_fespace->GetTrueVSize());
+   Vector Yf(f_fespace->GetTrueVSize());
+
+   coeff_order = 2;
+
+   BilinearFormIntegrator *massc = nullptr;
+   BilinearFormIntegrator *massf = nullptr;
+
+   if (vectorspace == VecSpace::H1)
+   {
+      FunctionCoefficient funcCoeff(&coeff);
+      xc.ProjectCoefficient(funcCoeff);
+      massc = new MassIntegrator;
+      massf = new MassIntegrator;
+   }
+   else
+   {
+      VectorFunctionCoefficient funcCoeff(dimension, &vectorcoeff);
+      xc.ProjectCoefficient(funcCoeff);
+      massc = new VectorMassIntegrator;
+      massf = new VectorMassIntegrator;
+   }
+   if (Rc)
+   {
+      Rc->Mult(xc,Xc);
+   }
+   else
+   {
+      Xc.MakeRef(xc,0);
+   }
+   T.Mult(Xc, Xf);
+
+   BilinearForm mc(c_fespace);
+   mc.AddDomainIntegrator(massc);
+   mc.Assemble();
+   SparseMatrix Mc;
+   Array<int> empty;
+   mc.FormSystemMatrix(empty, Mc);
+
+   BilinearForm mf(f_fespace);
+   mf.AddDomainIntegrator(massf);
+   mf.Assemble();
+   SparseMatrix Mf;
+   mf.FormSystemMatrix(empty, Mf);
+
+   Mf.Mult(Xf,Yf);
+
+   T.MultTranspose(Yf,Yc);
+
+   GSSmoother M(Mc);
+   Diff = 0.0;
+   PCG(Mc, M, Yc, Diff, 0, 500, 1e-24, 0.0);
+
+   Diff -= Xc;
+   REQUIRE(Diff.Norml2() < 1e-10);
+
+   delete f_fespace;
+   delete c_fespace;
+   delete f_fec;
+   delete c_fec;
 }
 
 #ifdef MFEM_USE_MPI
 
-TEST_CASE("partransfer", "[Parallel]")
+TEST_CASE("Parallel Transfer", "[Transfer][Parallel]")
 {
-   for (dimension = 2; dimension <= 3; ++dimension)
+   auto simplex = GENERATE(true, false);
+   auto geometric = GENERATE(true, false);
+   dimension = GENERATE(2, 3);
+   int ne = 4;
+   int order = 2;
+
+   int fineOrder = geometric ? order : 2 * order;
+   int num_procs, myid;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // Log test case information
+   int total_ne = std::pow(ne, dimension);
+   CAPTURE(dimension, simplex, total_ne, order, fineOrder, geometric);
+
+   coeff_order = 1;
+
+   Mesh mesh;
+   if (dimension == 2)
    {
-      for (int elementType = 0; elementType <= 1; ++elementType)
-      {
-         for (int ne = 4; ne <= 5; ++ne)
-         {
-            for (int order = 1; order <= 4; order *= 2)
-            {
-               for (int geometric = 0; geometric <= 1; ++geometric)
-               {
-                  int fineOrder = (geometric == 1) ? order : 2 * order;
-
-                  int num_procs;
-                  MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-                  int myid;
-                  MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-                  if (myid == 0)
-                  {
-                     std::cout << "Testing parallel transfer:\n"
-                               << "  Dimension:    " << dimension << "\n"
-                               << "  Element type: " << elementType << "\n"
-                               << "  Elements:     " << std::pow(ne, dimension) << "\n"
-                               << "  Coarse order: " << order << "\n"
-                               << "  Fine order:   " << fineOrder << "\n"
-                               << "  Geometric:    " << geometric << "\n";
-                  }
-
-                  Mesh mesh;
-                  if (dimension == 2)
-                  {
-                     Element::Type type = Element::QUADRILATERAL;
-                     if (elementType != 0)
-                     {
-                        type = Element::TRIANGLE;
-                     }
-                     mesh = Mesh::MakeCartesian2D(ne, ne, type, 1, 1.0, 1.0);
-                  }
-                  else
-                  {
-                     Element::Type type = Element::HEXAHEDRON;
-                     if (elementType != 0)
-                     {
-                        type = Element::TETRAHEDRON;
-                     }
-                     mesh =
-                        Mesh::MakeCartesian3D(ne, ne, ne, type, 1.0, 1.0, 1.0);
-                  }
-
-                  Mesh fineMesh(mesh);
-                  if (geometric)
-                  {
-                     fineMesh.UniformRefinement();
-                  }
-
-                  ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, mesh);
-                  ParMesh pfineMesh(MPI_COMM_WORLD, mesh);
-                  if (geometric)
-                  {
-                     pfineMesh.UniformRefinement();
-                  }
-
-                  FiniteElementCollection* c_h1_fec =
-                     new H1_FECollection(order, dimension);
-                  FiniteElementCollection* f_h1_fec = (geometric == 1) ? c_h1_fec : new
-                                                      H1_FECollection(fineOrder, dimension);
-
-                  int spaceDimension = 1;
-
-                  double referenceRestrictionValue = 0.0;
-
-                  // Compute reference values in serial
-                  {
-                     FiniteElementSpace* c_h1_fespace = new FiniteElementSpace(&mesh, c_h1_fec,
-                                                                               spaceDimension);
-                     FiniteElementSpace* f_h1_fespace = new FiniteElementSpace(&fineMesh, f_h1_fec,
-                                                                               spaceDimension);
-
-                     Operator* transferOperator = new TransferOperator(*c_h1_fespace,
-                                                                       *f_h1_fespace);
-                     GridFunction X(c_h1_fespace);
-                     GridFunction Y(f_h1_fespace);
-
-                     FunctionCoefficient funcCoeff(&coeff);
-                     Y.ProjectCoefficient(funcCoeff);
-                     X = 0.0;
-
-                     transferOperator->MultTranspose(Y, X);
-
-                     referenceRestrictionValue = std::sqrt(InnerProduct(X, X));
-
-                     delete transferOperator;
-                     delete f_h1_fespace;
-                     delete c_h1_fespace;
-                  }
-
-                  ParFiniteElementSpace* c_h1_fespace = new ParFiniteElementSpace(pmesh, c_h1_fec,
-                                                                                  spaceDimension);
-                  ParFiniteElementSpace* f_h1_fespace = new ParFiniteElementSpace(&pfineMesh,
-                                                                                  f_h1_fec,
-                                                                                  spaceDimension);
-
-                  Operator* transferOperator = new TrueTransferOperator(*c_h1_fespace,
-                                                                        *f_h1_fespace);
-                  ParGridFunction X(c_h1_fespace);
-                  ParGridFunction Y_exact(f_h1_fespace);
-                  ParGridFunction Y(f_h1_fespace);
-
-                  FunctionCoefficient funcCoeff(&coeff);
-                  X.ProjectCoefficient(funcCoeff);
-                  Y_exact.ProjectCoefficient(funcCoeff);
-
-                  Y = 0.0;
-
-                  Vector X_true(c_h1_fespace->GetTrueVSize());
-                  Vector Y_true(f_h1_fespace->GetTrueVSize());
-
-                  c_h1_fespace->GetRestrictionMatrix()->Mult(X, X_true);
-                  transferOperator->Mult(X_true, Y_true);
-                  f_h1_fespace->GetProlongationMatrix()->Mult(Y_true, Y);
-
-                  Y -= Y_exact;
-                  REQUIRE(Y.Norml2() < 1e-12 * Y_exact.Norml2());
-
-                  f_h1_fespace->GetRestrictionMatrix()->Mult(Y_exact, Y_true);
-                  transferOperator->MultTranspose(Y_true, X_true);
-
-                  double restrictionValue = std::sqrt(InnerProduct(MPI_COMM_WORLD, X_true,
-                                                                   X_true));
-                  REQUIRE(std::abs(restrictionValue - referenceRestrictionValue) < 1e-12 *
-                          std::abs(referenceRestrictionValue));
-
-                  delete transferOperator;
-                  delete f_h1_fespace;
-                  delete c_h1_fespace;
-                  if (geometric == 0)
-                  {
-                     delete f_h1_fec;
-                  }
-                  delete c_h1_fec;
-                  delete pmesh;
-               }
-            }
-         }
-      }
+      Element::Type type = simplex ? Element::TRIANGLE : Element::QUADRILATERAL;
+      mesh = Mesh::MakeCartesian2D(ne, ne, type, 1, 1.0, 1.0);
    }
+   else
+   {
+      Element::Type type = simplex ? Element::TETRAHEDRON : Element::HEXAHEDRON;
+      mesh = Mesh::MakeCartesian3D(ne, ne, ne, type, 1.0, 1.0, 1.0);
+   }
+
+   Mesh fineMesh(mesh);
+   if (geometric)
+   {
+      fineMesh.UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, mesh);
+   ParMesh pfineMesh(MPI_COMM_WORLD, mesh);
+   if (geometric)
+   {
+      pfineMesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *c_h1_fec =
+      new H1_FECollection(order, dimension);
+   FiniteElementCollection *f_h1_fec = geometric ? c_h1_fec : new
+                                       H1_FECollection(fineOrder, dimension);
+
+   int spaceDimension = 1;
+
+   double referenceRestrictionValue = 0.0;
+
+   // Compute reference values in serial
+   {
+      FiniteElementSpace* c_h1_fespace = new FiniteElementSpace(&mesh, c_h1_fec,
+                                                                spaceDimension);
+      FiniteElementSpace* f_h1_fespace = new FiniteElementSpace(&fineMesh, f_h1_fec,
+                                                                spaceDimension);
+
+      Operator* transferOperator = new TransferOperator(*c_h1_fespace,
+                                                        *f_h1_fespace);
+      GridFunction X(c_h1_fespace);
+      GridFunction Y(f_h1_fespace);
+
+      FunctionCoefficient funcCoeff(&coeff);
+      Y.ProjectCoefficient(funcCoeff);
+      X = 0.0;
+
+      transferOperator->MultTranspose(Y, X);
+
+      referenceRestrictionValue = std::sqrt(InnerProduct(X, X));
+
+      delete transferOperator;
+      delete f_h1_fespace;
+      delete c_h1_fespace;
+   }
+
+   ParFiniteElementSpace* c_h1_fespace = new ParFiniteElementSpace(pmesh, c_h1_fec,
+                                                                   spaceDimension);
+   ParFiniteElementSpace* f_h1_fespace = new ParFiniteElementSpace(&pfineMesh,
+                                                                   f_h1_fec,
+                                                                   spaceDimension);
+
+   Operator* transferOperator = new TrueTransferOperator(*c_h1_fespace,
+                                                         *f_h1_fespace);
+   ParGridFunction X(c_h1_fespace);
+   ParGridFunction Y_exact(f_h1_fespace);
+   ParGridFunction Y(f_h1_fespace);
+   FunctionCoefficient funcCoeff(&coeff);
+   X.ProjectCoefficient(funcCoeff);
+   Y_exact.ProjectCoefficient(funcCoeff);
+
+   Y = 0.0;
+
+   Vector X_true(c_h1_fespace->GetTrueVSize());
+   Vector Y_true(f_h1_fespace->GetTrueVSize());
+
+   c_h1_fespace->GetRestrictionMatrix()->Mult(X, X_true);
+   transferOperator->Mult(X_true, Y_true);
+   f_h1_fespace->GetProlongationMatrix()->Mult(Y_true, Y);
+
+   Y -= Y_exact;
+   REQUIRE(Y.Norml2() < 1e-12 * Y_exact.Norml2());
+
+   f_h1_fespace->GetRestrictionMatrix()->Mult(Y_exact, Y_true);
+   transferOperator->MultTranspose(Y_true, X_true);
+
+   double restrictionValue = std::sqrt(InnerProduct(MPI_COMM_WORLD, X_true,
+                                                    X_true));
+   REQUIRE(std::abs(restrictionValue - referenceRestrictionValue) < 1e-12 *
+           std::abs(referenceRestrictionValue));
+
+   delete transferOperator;
+   delete f_h1_fespace;
+   delete c_h1_fespace;
+   if (!geometric) { delete f_h1_fec; }
+   delete c_h1_fec;
+   delete pmesh;
 }
 
 #endif

tests/unit/linalg/test_matrix_dense.cpp

@@ -341,3 +341,517 @@ TEST_CASE("DenseTensor copy", "[DenseMatrix][DenseTensor]")
       REQUIRE(t3.Data()[i] == t1.Data()[i]);
    }
 }
+
+TEST_CASE("DenseMatrix CalcAdjugateRevDiff", "[DenseMatrix]")
+{
+   constexpr double eps_fd = 1e-5; // 2nd-order finite-difference step size
+
+   SECTION("1x1 matrix")
+   {
+      double A_data[1] = { 3.1415926};
+      double adjA_bar_data[1] = {-2.0};
+
+      DenseMatrix A(A_data, 1, 1);
+      DenseMatrix adjA_bar(adjA_bar_data, 1, 1);
+      DenseMatrix A_bar(1, 1), adjA_fd(1, 1);
+      DenseMatrix A_pert(1, 1), adjA_pert(1, 1);
+
+      // Compute the derivative using reverse mode
+      CalcAdjugateRevDiff(A, adjA_bar, A_bar);
+
+      // Compute the derivative using central finite-difference approximation
+      A_pert = A;
+      A_pert(0, 0) += eps_fd;
+      CalcAdjugate(A_pert, adjA_fd);
+      A_pert(0, 0) -= 2.0 * eps_fd;
+      CalcAdjugate(A_pert, adjA_pert);
+      adjA_fd -= adjA_pert;
+      adjA_fd *= 1/(2.0 * eps_fd);
+      // sum up derivative with weights
+      double A_bar_fd = adjA_fd(0, 0) * adjA_bar(0, 0);
+      REQUIRE(A_bar(0, 0) == Approx(A_bar_fd));
+   }
+
+   SECTION("2x1 matrix")
+   {
+      double A_data[2] = {2.0, -3.0};
+      double adjA_bar_data[2] = {-1.5, 4.0};
+
+      DenseMatrix A(A_data, 2, 1);
+      DenseMatrix adjA_bar(adjA_bar_data, 1, 2);
+      DenseMatrix A_bar(2,1), adjA_fd(1,2);
+      DenseMatrix A_pert(2,1), adjA_pert(1,2);
+
+      // Compute the derivatives using reverse mode
+      CalcAdjugateRevDiff(A, adjA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 2; ++i)
+      {
+         // Pertrub A(i,0) and evaluate derivative of adjugate
+         A_pert = A;
+         A_pert(i, 0) += eps_fd;
+         CalcAdjugate(A_pert, adjA_fd);
+         A_pert(i, 0) -= 2.0 * eps_fd;
+         CalcAdjugate(A_pert, adjA_pert);
+         adjA_fd -= adjA_pert;
+         adjA_fd *= 1 / (2.0 * eps_fd);
+         // sum up derivative with weights
+         double A_bar_fd = 0.0;
+         for (int k = 0; k < 2; ++k)
+         {
+            A_bar_fd += adjA_fd(0, k) * adjA_bar(0, k);
+         }
+         REQUIRE(A_bar(i, 0) == Approx(A_bar_fd));
+      }
+   }
+
+   SECTION("2x2 matrix")
+   {
+      double A_data[4] = {2.0, -3.0, 4.0, -1.0};
+      double adjA_bar_data[4] = {1.0, 4.0, 2.0, -3.0};
+
+      DenseMatrix A(A_data, 2, 2);
+      DenseMatrix adjA_bar(adjA_bar_data, 2, 2);
+      DenseMatrix A_bar(2,2), adjA_fd(2,2);
+      DenseMatrix A_pert(2,2), adjA_pert(2,2);
+
+      // Compute the derivatives using reverse mode
+      CalcAdjugateRevDiff(A, adjA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 2; ++i)
+      {
+         for (int j = 0; j < 2; ++j)
+         {
+            // Pertrub A(i,j) and evaluate derivative of adjugate
+            A_pert = A;
+            A_pert(i,j) += eps_fd;
+            CalcAdjugate(A_pert, adjA_fd);
+            A_pert(i,j) -= 2.0*eps_fd;
+            CalcAdjugate(A_pert, adjA_pert);
+            adjA_fd -= adjA_pert;
+            adjA_fd *= 1/(2.0*eps_fd);
+            // sum up derivative with weights
+            double A_bar_fd = 0.0;
+            for (int k = 0; k < 2; ++k)
+            {
+               for (int l = 0; l < 2; ++l)
+               {
+                  A_bar_fd += adjA_fd(k,l)*adjA_bar(k,l);
+               }
+            }
+            REQUIRE(A_bar(i,j) == Approx(A_bar_fd));
+         }
+      }
+   }
+
+   SECTION("3x1 matrix")
+   {
+      double A_data[3] = {2.0, -3.0, 3.1415926};
+      double adjA_bar_data[3] = {-1.5, 4.0, 2.71828};
+
+      DenseMatrix A(A_data, 3, 1);
+      DenseMatrix adjA_bar(adjA_bar_data, 1, 3);
+      DenseMatrix A_bar(3, 1), adjA_fd(1, 3);
+      DenseMatrix A_pert(3, 1), adjA_pert(1, 3);
+
+      // Compute the derivatives using reverse mode
+      CalcAdjugateRevDiff(A, adjA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 3; ++i)
+      {
+         // Pertrub A(i,0) and evaluate derivative of adjugate
+         A_pert = A;
+         A_pert(i, 0) += eps_fd;
+         CalcAdjugate(A_pert, adjA_fd);
+         A_pert(i, 0) -= 2.0 * eps_fd;
+         CalcAdjugate(A_pert, adjA_pert);
+         adjA_fd -= adjA_pert;
+         adjA_fd *= 1 / (2.0 * eps_fd);
+         // sum up derivative with weights
+         double A_bar_fd = 0.0;
+         for (int k = 0; k < 3; ++k)
+         {
+            A_bar_fd += adjA_fd(0, k) * adjA_bar(0, k);
+         }
+         REQUIRE(A_bar(i, 0) == Approx(A_bar_fd));
+      }
+   }
+
+   SECTION("3x3 matrix")
+   {
+      double A_data[9] = {1.0, 5.0, 3.0, -2.0, 6.0, -9.0, 4.0, -7.0, 8.0};
+      double adjA_bar_data[9] = {3.0, 6.0, -8.0, 1.0, -7.0, 5.0, 2.0, 4.0, -9.0};
+
+      DenseMatrix A(A_data, 3, 3);
+      DenseMatrix adjA_bar(adjA_bar_data, 3, 3);
+      DenseMatrix A_bar(3,3), adjA_fd(3,3);
+      DenseMatrix A_pert(3,3), adjA_pert(3,3);
+
+      // Compute the derivatives using reverse mode
+      CalcAdjugateRevDiff(A, adjA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 3; ++i)
+      {
+         for (int j = 0; j < 3; ++j)
+         {
+            // Pertrub A(i,j) and evaluate derivative of adjugate
+            A_pert = A;
+            A_pert(i,j) += eps_fd;
+            CalcAdjugate(A_pert, adjA_fd);
+            A_pert(i,j) -= 2.0*eps_fd;
+            CalcAdjugate(A_pert, adjA_pert);
+            adjA_fd -= adjA_pert;
+            adjA_fd *= 1/(2.0*eps_fd);
+            // sum up derivative with weights
+            double A_bar_fd = 0.0;
+            for (int k = 0; k < 3; ++k)
+            {
+               for (int l = 0; l < 3; ++l)
+               {
+                  A_bar_fd += adjA_fd(k,l)*adjA_bar(k,l);
+               }
+            }
+            REQUIRE(A_bar(i,j) == Approx(A_bar_fd));
+         }
+      }
+   }
+}
+
+TEST_CASE("DenseMatrix CalcInverseRevDiff", "[DenseMatrix]")
+{
+   constexpr double eps_fd = 1e-5; // 2nd-order finite-difference step size
+
+   SECTION("1x1 matrix")
+   {
+      double A_data[1] = { 3.1415926};
+      double invA_bar_data[1] = {-2.0};
+
+      DenseMatrix A(A_data, 1, 1);
+      DenseMatrix invA_bar(invA_bar_data, 1, 1);
+      DenseMatrix A_bar(1, 1), invA_fd(1, 1);
+      DenseMatrix A_pert(1, 1), invA_pert(1, 1);
+
+      // Compute the derivative using reverse mode
+      A_bar = 0.0;
+      CalcInverseRevDiff(A, invA_bar, A_bar);
+
+      // Compute the derivative using central finite-difference approximation
+      A_pert = A;
+      A_pert(0, 0) += eps_fd;
+      CalcInverse(A_pert, invA_fd);
+      A_pert(0, 0) -= 2.0 * eps_fd;
+      CalcInverse(A_pert, invA_pert);
+      invA_fd -= invA_pert;
+      invA_fd *= 1/(2.0 * eps_fd);
+      // sum up derivative with weights
+      double A_bar_fd = invA_fd(0, 0) * invA_bar(0, 0);
+      REQUIRE(A_bar(0, 0) == Approx(A_bar_fd));
+   }
+
+   SECTION("2x1 matrix")
+   {
+      double A_data[2] = {2.0, -3.0};
+      double invA_bar_data[2] = {-1.5, 4.0};
+
+      DenseMatrix A(A_data, 2, 1);
+      DenseMatrix invA_bar(invA_bar_data, 1, 2);
+      DenseMatrix A_bar(2,1), invA_fd(1,2);
+      DenseMatrix A_pert(2,1), invA_pert(1,2);
+
+      // Compute the derivatives using reverse mode
+      A_bar = 0.0;
+      CalcInverseRevDiff(A, invA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 2; ++i)
+      {
+         // Pertrub A(i,0) and evaluate derivative of inverse
+         A_pert = A;
+         A_pert(i, 0) += eps_fd;
+         CalcInverse(A_pert, invA_fd);
+         A_pert(i, 0) -= 2.0 * eps_fd;
+         CalcInverse(A_pert, invA_pert);
+         invA_fd -= invA_pert;
+         invA_fd *= 1 / (2.0 * eps_fd);
+         // sum up derivative with weights
+         double A_bar_fd = 0.0;
+         for (int k = 0; k < 2; ++k)
+         {
+            A_bar_fd += invA_fd(0, k) * invA_bar(0, k);
+         }
+         REQUIRE(A_bar(i, 0) == Approx(A_bar_fd));
+      }
+   }
+
+   SECTION("2x2 matrix")
+   {
+      double A_data[4] = {2.0, -3.0, 4.0, -1.0};
+      double invA_bar_data[4] = {1.0, 4.0, 2.0, -3.0};
+
+      DenseMatrix A(A_data, 2, 2);
+      DenseMatrix invA_bar(invA_bar_data, 2, 2);
+      DenseMatrix A_bar(2,2), invA_fd(2,2);
+      DenseMatrix A_pert(2,2), invA_pert(2,2);
+
+      // Compute the derivatives using reverse mode
+      A_bar = 0.0;
+      CalcInverseRevDiff(A, invA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 2; ++i)
+      {
+         for (int j = 0; j < 2; ++j)
+         {
+            // Pertrub A(i,j) and evaluate derivative of inverse
+            A_pert = A;
+            A_pert(i,j) += eps_fd;
+            CalcInverse(A_pert, invA_fd);
+            A_pert(i,j) -= 2.0*eps_fd;
+            CalcInverse(A_pert, invA_pert);
+            invA_fd -= invA_pert;
+            invA_fd *= 1/(2.0*eps_fd);
+            // sum up derivative with weights
+            double A_bar_fd = 0.0;
+            for (int k = 0; k < 2; ++k)
+            {
+               for (int l = 0; l < 2; ++l)
+               {
+                  A_bar_fd += invA_fd(k,l)*invA_bar(k,l);
+               }
+            }
+            REQUIRE(A_bar(i,j) == Approx(A_bar_fd));
+         }
+      }
+   }
+
+   SECTION("3x1 matrix")
+   {
+      double A_data[3] = {2.0, -3.0, 3.1415926};
+      double invA_bar_data[3] = {-1.5, 4.0, 2.71828};
+
+      DenseMatrix A(A_data, 3, 1);
+      DenseMatrix invA_bar(invA_bar_data, 1, 3);
+      DenseMatrix A_bar(3, 1), invA_fd(1, 3);
+      DenseMatrix A_pert(3, 1), invA_pert(1, 3);
+
+      // Compute the derivatives using reverse mode
+      A_bar = 0.0;
+      CalcInverseRevDiff(A, invA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 3; ++i)
+      {
+         // Pertrub A(i,0) and evaluate derivative of inverse
+         A_pert = A;
+         A_pert(i, 0) += eps_fd;
+         CalcInverse(A_pert, invA_fd);
+         A_pert(i, 0) -= 2.0 * eps_fd;
+         CalcInverse(A_pert, invA_pert);
+         invA_fd -= invA_pert;
+         invA_fd *= 1 / (2.0 * eps_fd);
+         // sum up derivative with weights
+         double A_bar_fd = 0.0;
+         for (int k = 0; k < 3; ++k)
+         {
+            A_bar_fd += invA_fd(0, k) * invA_bar(0, k);
+         }
+         REQUIRE(A_bar(i, 0) == Approx(A_bar_fd));
+      }
+   }
+
+   SECTION("3x2 matrix")
+   {
+      double A_data[6] = {1.0, 5.0, 3.0, -2.0, 6.0, -9.0};
+      double invA_bar_data[6] = {3.0, 6.0, -8.0, 1.0, -7.0, 5.0};
+      // double invA_bar_data[6] = {1.0, 0.0, 0.0, 0.0, 0.0, 0.0};
+      DenseMatrix invA_bar(invA_bar_data, 2, 3);
+      DenseMatrix invA_pert(2, 3), invA_fd(2, 3);
+      DenseMatrix A(A_data, 3, 2);
+      DenseMatrix A_bar(3, 2), A_pert(3, 2);
+
+      // Compute the derivatives using reverse mode
+      A_bar = 0.0;
+      CalcInverseRevDiff(A, invA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 3; ++i)
+      {
+         for (int j = 0; j < 2; ++j)
+         {
+            // Pertrub A(i,j) and evaluate derivative of inverse
+            A_pert = A;
+            A_pert(i,j) += eps_fd;
+            CalcInverse(A_pert, invA_fd);
+            A_pert(i,j) -= 2.0*eps_fd;
+            CalcInverse(A_pert, invA_pert);
+            invA_fd -= invA_pert;
+            invA_fd *= 1/(2.0*eps_fd);
+            // sum up derivative with weights
+            double A_bar_fd = 0.0;
+            for (int k = 0; k < 3; ++k)
+            {
+               for (int l = 0; l < 2; ++l)
+               {
+                  A_bar_fd += invA_fd(l, k)*invA_bar(l, k);
+               }
+            }
+            REQUIRE(A_bar(i,j) == Approx(A_bar_fd));
+         }
+      }
+   }
+
+   SECTION("3x3 matrix")
+   {
+      double A_data[9] = {1.0, 5.0, 3.0, -2.0, 6.0, -9.0, 4.0, -7.0, 8.0};
+      double invA_bar_data[9] = {3.0, 6.0, -8.0, 1.0, -7.0, 5.0, 2.0, 4.0, -9.0};
+
+      DenseMatrix A(A_data, 3, 3);
+      DenseMatrix invA_bar(invA_bar_data, 3, 3);
+      DenseMatrix A_bar(3,3), invA_fd(3,3);
+      DenseMatrix A_pert(3,3), invA_pert(3,3);
+
+      // Compute the derivatives using reverse mode
+      A_bar = 0.0;
+      CalcInverseRevDiff(A, invA_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 3; ++i)
+      {
+         for (int j = 0; j < 3; ++j)
+         {
+            // Pertrub A(i,j) and evaluate derivative of inverse
+            A_pert = A;
+            A_pert(i,j) += eps_fd;
+            CalcInverse(A_pert, invA_fd);
+            A_pert(i,j) -= 2.0*eps_fd;
+            CalcInverse(A_pert, invA_pert);
+            invA_fd -= invA_pert;
+            invA_fd *= 1/(2.0*eps_fd);
+            // sum up derivative with weights
+            double A_bar_fd = 0.0;
+            for (int k = 0; k < 3; ++k)
+            {
+               for (int l = 0; l < 3; ++l)
+               {
+                  A_bar_fd += invA_fd(k,l)*invA_bar(k,l);
+               }
+            }
+            REQUIRE(A_bar(i,j) == Approx(A_bar_fd));
+         }
+      }
+   }
+}
+
+TEST_CASE("DenseMatrix WeightRevDiff", "[DenseMatrix]")
+{
+   // This also tests DenseMatrix::DetRevDiff indirectly
+   constexpr double eps_fd = 1e-5; // 2nd-order finite-difference step size
+   double A_data[9] = {1.0, 5.0, 3.0, -2.0, 6.0, -9.0, 4.0, -7.0, 8.0};
+
+   for (int height = 1; height <= 3; ++height)
+   {
+      for (int width = 1; width <= height; ++width)
+      {
+         DenseMatrix A(A_data, height, width);
+         DenseMatrix weight_bar(height, width);
+         DenseMatrix A_pert(height, width);
+
+         // Compute the gradient of A.Weight() using reverse mode AD
+         A.WeightRevDiff(weight_bar);
+
+         // Compute the gradient of A.Weight using 2nd order finite-difference
+         for (int i = 0; i < height; ++i)
+         {
+            for (int j = 0; j < width; ++j)
+            {
+               // Perturb A(i,j) in + and - directions and evaluate Weight()
+               A_pert = A;
+               A_pert(i,j) += eps_fd;
+               double dweight = A_pert.Weight();
+               A_pert(i,j) -= 2.0*eps_fd;
+               dweight -= A_pert.Weight();
+               dweight /= (2.0*eps_fd);
+               REQUIRE(weight_bar(i,j) == Approx(dweight));
+            }
+         }
+      }
+   }
+}
+
+TEST_CASE("DenseMatrix CalcOrthoRevDiff", "[DenseMatrix]")
+{
+   constexpr double eps_fd = 1e-5; // 2nd-order finite-difference step size
+
+   SECTION("2x1 matrix")
+   {
+      double A_data[2] = {2.0, -3.0};
+      double n_bar_data[2] = {-1.5, 4.0};
+      Vector n_bar(n_bar_data, 2);
+      Vector n_pert(2), n_fd(2);
+      DenseMatrix A(A_data, 2, 1);
+      DenseMatrix A_bar(2,1), A_pert(2,1);
+
+      // Compute the derivatives using reverse mode
+      CalcOrthoRevDiff(A, n_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 2; ++i)
+      {
+         // Pertrub A(i,0) and evaluate derivative of adjugate
+         A_pert = A;
+         A_pert(i, 0) += eps_fd;
+         CalcOrtho(A_pert, n_fd);
+         A_pert(i, 0) -= 2.0 * eps_fd;
+         CalcOrtho(A_pert, n_pert);
+         n_fd -= n_pert;
+         n_fd *= 1 / (2.0 * eps_fd);
+         // sum up derivative with weights
+         double A_bar_fd = 0.0;
+         for (int k = 0; k < 2; ++k)
+         {
+            A_bar_fd += n_fd(k) * n_bar(k);
+         }
+         REQUIRE(A_bar(i, 0) == Approx(A_bar_fd));
+      }
+   }
+
+   SECTION("3x2 matrix")
+   {
+      double A_data[6] = {1.0, 5.0, 3.0, -2.0, 6.0, -9.0};
+      double n_bar_data[3] = {1.0, 4.0, -3.0};
+      Vector n_bar(n_bar_data, 3);
+      Vector n_pert(3), n_fd(3);
+      DenseMatrix A(A_data, 3, 2);
+      DenseMatrix A_bar(3,2), A_pert(3,2);
+
+      // Compute the derivatives using reverse mode
+      CalcOrthoRevDiff(A, n_bar, A_bar);
+
+      // Compute the derivatives using central finite-difference approximation
+      for (int i = 0; i < 3; ++i)
+      {
+         for (int j = 0; j < 2; ++j)
+         {
+            // Pertrub A(i,j) and evaluate derivative of adjugate
+            A_pert = A;
+            A_pert(i,j) += eps_fd;
+            CalcOrtho(A_pert, n_fd);
+            A_pert(i,j) -= 2.0*eps_fd;
+            CalcOrtho(A_pert, n_pert);
+            n_fd -= n_pert;
+            n_fd *= 1/(2.0*eps_fd);
+            // sum up derivative with weights
+            double A_bar_fd = 0.0;
+            for (int k = 0; k < 3; ++k)
+            {
+               A_bar_fd += n_fd(k)*n_bar(k);
+            }
+            REQUIRE(A_bar(i,j) == Approx(A_bar_fd));
+         }
+      }
+   }
+}