including an obstacle problem variant wherein the essential dofs are not given to the optimizer as free variables that it must choose the values for.

Change Array::MakeRef to use MakeAlias

.gitignore

@@ -210,6 +210,7 @@ miniapps/meshing/trimmer
 miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
+miniapps/meshing/pmesh-fitting
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc

.gitlab/quartz-build-and-test.yml

@@ -27,11 +27,6 @@ allocate_resource:
   timeout: 6h
 
 # GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_4_9_3:
-  variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
 debug_ser_gcc_6_1_0:
   variables:
     SPEC: "%gcc@6.1.0 +debug~mpi"

CHANGELOG

@@ -14,6 +14,20 @@ Version 4.5.3 (development)
 - Added new methods in the Mesh class to set and get attributes on NURBS patches
   and patch boundaries.
 
+New and updated examples and miniapps
+-------------------------------------
+- Added a miniapp pmesh-fitting in miniapps/meshing for interface and boundary fitting to implicit domains defined using level-set functions.
+
+- Moved the distance solver methods from miniapps/shifted to miniapps/common. 
+
+Discretization improvements
+---------------------------
+- Face restriction operators for Nedelec and Raviart-Thomas finite element
+  spaces are now supported through the ConformingFaceRestriction class.
+
+- VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
+
+
 Version 4.5.2, released on March 23, 2023
 =========================================
 

config/sample-runs.sh

@@ -138,7 +138,7 @@ groups_parallel=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "pmesh-optimizer.cpp pminimal-surface.cpp"'
+   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
@@ -227,7 +227,7 @@ groups_all=(
    "Meshing miniapps:"
    "miniapps/meshing"
    "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    {,p}mesh-optimizer.cpp {,p}minimal-surface.cpp"'
+    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"

data/disk.mesh

@@ -0,0 +1,109 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+5
+1 3 4 5 6 7
+1 3 0 1 5 4
+1 3 1 2 6 5
+1 3 3 7 6 2
+1 3 0 4 7 3
+
+boundary
+4
+1 1 0 1
+1 1 2 3
+1 1 1 2
+1 1 3 0
+
+edges
+12
+0 0 1
+0 4 5
+0 7 6
+0 3 2
+1 1 2
+1 5 6
+1 4 7
+1 0 3
+2 0 4
+2 1 5
+2 2 6
+2 3 7
+
+vertices
+8
+
+knotvectors
+3
+2 3 0 0 0 1 1 1
+2 3 0 0 0 1 1 1
+2 3 0 0 0 1 1 1
+
+weights
+1
+1
+1
+1
+1
+1
+1
+1
+0.70710678118655
+1
+1
+0.70710678118655
+0.70710678118655
+1
+1
+0.70710678118655
+1
+1
+1
+1
+1
+0.85355339059327
+0.85355339059327
+0.85355339059327
+0.85355339059327
+
+FiniteElementSpace
+FiniteElementCollection: NURBS2
+VDim: 2
+Ordering: 1
+
+-0.70710678118 -0.70710678118
+0.70710678118 -0.70710678118
+0.70710678118 0.70710678118
+-0.70710678118 0.70710678118
+-0.35355339059 -0.35355339059
+0.35355339059 -0.35355339059
+0.35355339059 0.35355339059
+-0.35355339059 0.35355339059
+0 -1.41421356236
+0 -0.35355339059
+0 0.35355339059
+0 1.41421356236
+1.41421356236 0
+0.35355339059 0
+-0.35355339059 0
+-1.41421356236 0
+-0.530330085885 -0.530330085885
+0.530330085885 -0.530330085885
+0.530330085885 0.530330085885
+-0.530330085885 0.530330085885
+0 0
+0 -0.883883476475
+0.883883476475 0
+0 0.883883476475
+-0.883883476475 0

data/segment-nurbs.mesh

@@ -0,0 +1,46 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+elements
+1
+1 1 0 1
+
+boundary
+2
+1 0 0
+2 0 1
+
+edges
+1
+0 0 1
+
+vertices
+2
+
+knotvectors
+1
+1 2 0 0 1 1
+
+weights
+1
+1
+
+FiniteElementSpace
+FiniteElementCollection: NURBS1
+VDim: 1
+Ordering: 1
+
+0
+1
+
+

examples/contact/DirichletObstacleProblem.cpp

@@ -0,0 +1,141 @@
+#include "mfem.hpp"
+#include "Problems.hpp"
+#include "IPsolver.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+
+double fRhs(const Vector &pt);
+double obstacle(const Vector &pt);
+double dmanufacturedFun(const Vector &pt);
+
+int main(int argc, char *argv[])
+{
+  int FEorder = 1; // order of the finite elements
+  int linSolver = 0;
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  int ref_levels = 3; 
+  OptionsParser args(argc, argv);
+  args.AddOption(&FEorder, "-o", "--order",\
+		  "Order of the finite elements.");
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+  
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+
+  const char *meshFile = "../../data/inline-quad.mesh";
+  Mesh *mesh = new Mesh(meshFile, 1, 1);
+  int dim = mesh->Dimension(); // geometric dimension of the domain
+  for (int l = 0; l < ref_levels; l++)
+  {
+      mesh->UniformRefinement();
+  }
+
+  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
+  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
+  Array<int> ess_tdof_list;
+  if (mesh->bdr_attributes.Size())
+  {
+     Array<int> ess_bdr(mesh->bdr_attributes.Max());
+     ess_bdr = 1;
+     Vh->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+  }
+  
+  double DC_val = 0.0;
+  int dimD = Vh->GetTrueVSize();
+  Vector x0(dimD); x0 = DC_val;
+  Vector xf(dimD); xf = 0.0;
+
+  ObstacleProblem problem(Vh, x0, &fRhs, &obstacle, ess_tdof_list);
+  
+
+  InteriorPointSolver optimizer(&problem); 
+  optimizer.SetTol(1.e-7);
+  optimizer.SetLinearSolver(linSolver);
+  optimizer.SetMaxIter(maxIPMiters);
+  optimizer.Mult(x0, xf);
+  
+  double Einitial = problem.E(x0);
+  double Efinal = problem.E(xf);
+  cout << "Energy objective at initial point = " << Einitial << endl;
+  cout << "Energy objective at optimizer = " << Efinal << endl;
+
+  GridFunction d_gf(Vh);
+
+  d_gf = xf;
+  
+  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
+  GridFunction dm_gf(Vh);
+  dm_gf.ProjectCoefficient(dm_fc);
+
+  
+  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
+  paraview_dc.SetPrefixPath("ParaView");
+  paraview_dc.SetLevelsOfDetail(FEorder);
+  paraview_dc.SetDataFormat(VTKFormat::BINARY);
+  paraview_dc.SetHighOrderOutput(true);
+  paraview_dc.SetCycle(0);
+  paraview_dc.SetTime(0.0);
+  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
+  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
+  paraview_dc.Save();
+  
+
+  delete Vh;
+  delete fec;
+  delete mesh;
+  return 0;
+}
+
+
+
+double dmanufacturedFun(const Vector &pt)
+{
+  double alpha = 16.5;
+  return sin(M_PI * pt(1)) * (sin(M_PI * pt(0)) - alpha * pow(pt(0) * (1. - pt(0)), 2));
+}
+
+
+// f(x) forcing term... which enters the objective energy functional
+// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
+// of f(x). f(x) is such that in the absence of bound-constraints then
+// the solution of the optimization problem satisfies the PDE
+// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
+// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
+
+double fRhs(const Vector &pt)
+{
+  double alpha = 16.5;
+  double fx;
+  fx = pow(M_PI, 2) * sin(M_PI * pt(0));
+  fx += alpha * (2. * pow(pt(0), 2) + 2. * pow(1.-pt(0), 2) - 8. * pt(0) * (1.-pt(0)));
+  fx += pow(M_PI, 2) * sin(M_PI * pt(0)) * dmanufacturedFun(pt);
+  fx *= sin(M_PI * pt(1));
+  return fx;
+}
+
+double obstacle(const Vector &pt)
+{
+  return 0.0;
+}

examples/contact/DirichletObstacleProblemVariant.cpp

@@ -0,0 +1,156 @@
+#include "mfem.hpp"
+#include "Problems.hpp"
+#include "IPsolver.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+
+double fRhs(const Vector &pt);
+double obstacle(const Vector &pt);
+double dmanufacturedFun(const Vector &pt);
+
+int main(int argc, char *argv[])
+{
+  int FEorder = 1; // order of the finite elements
+  int linSolver = 0;
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  int ref_levels = 3; 
+  OptionsParser args(argc, argv);
+  args.AddOption(&FEorder, "-o", "--order",\
+		  "Order of the finite elements.");
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+  
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+
+  const char *meshFile = "../../data/inline-quad.mesh";
+  Mesh *mesh = new Mesh(meshFile, 1, 1);
+  int dim = mesh->Dimension(); // geometric dimension of the domain
+  for (int l = 0; l < ref_levels; l++)
+  {
+      mesh->UniformRefinement();
+  }
+
+  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
+  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
+  Array<int> ess_tdof_list;
+  if (mesh->bdr_attributes.Size())
+  {
+     Array<int> ess_bdr(mesh->bdr_attributes.Max());
+     ess_bdr = 1;
+     Vh->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+  }
+  
+  double DC_val = 0.06;
+  Vector x0DC(Vh->GetTrueVSize()); x0DC = DC_val;
+  int dimD = Vh->GetTrueVSize() - ess_tdof_list.Size();
+  Vector x0(dimD); x0 = 0.0;
+  Vector xf(dimD); xf = 0.0;
+
+  ObstacleProblemVariant problem(Vh, x0DC, &fRhs, &obstacle, ess_tdof_list);
+
+  InteriorPointSolver optimizer(&problem); 
+  optimizer.SetTol(1.e-7);
+  optimizer.SetLinearSolver(linSolver);
+  optimizer.SetMaxIter(maxIPMiters);
+  optimizer.Mult(x0, xf);
+  
+  double Einitial = problem.E(x0);
+  double Efinal = problem.E(xf);
+  cout << "Energy objective at initial point = " << Einitial << endl;
+  cout << "Energy objective at optimizer = " << Efinal << endl;
+
+  Array<int> noness_tdof_list;
+  noness_tdof_list.SetSize(dimD);
+  int i = 0;
+  for(int j = 0; j < Vh->GetTrueVSize(); j++)
+  {
+     if(ess_tdof_list.Find(j) == -1)
+     {
+        noness_tdof_list[i] = j;
+        i += 1;
+     }
+  }
+
+
+  GridFunction d_gf(Vh);
+  d_gf.Set(1.0, x0DC);
+  d_gf.SetSubVector(noness_tdof_list, xf);
+
+
+
+  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
+  GridFunction dm_gf(Vh);
+  dm_gf.ProjectCoefficient(dm_fc);
+
+  
+  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
+  paraview_dc.SetPrefixPath("ParaView");
+  paraview_dc.SetLevelsOfDetail(FEorder);
+  paraview_dc.SetDataFormat(VTKFormat::BINARY);
+  paraview_dc.SetHighOrderOutput(true);
+  paraview_dc.SetCycle(0);
+  paraview_dc.SetTime(0.0);
+  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
+  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
+  paraview_dc.Save();
+ 
+
+  delete Vh;
+  delete fec;
+  delete mesh;
+  return 0;
+}
+
+
+
+double dmanufacturedFun(const Vector &pt)
+{
+  double alpha = 16.5;
+  return sin(M_PI * pt(1)) * (sin(M_PI * pt(0)) - alpha * pow(pt(0) * (1. - pt(0)), 2));
+}
+
+
+// f(x) forcing term... which enters the objective energy functional
+// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
+// of f(x). f(x) is such that in the absence of bound-constraints then
+// the solution of the optimization problem satisfies the PDE
+// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
+// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
+
+double fRhs(const Vector &pt)
+{
+  double alpha = 16.5;
+  double fx;
+  fx = pow(M_PI, 2) * sin(M_PI * pt(0));
+  fx += alpha * (2. * pow(pt(0), 2) + 2. * pow(1.-pt(0), 2) - 8. * pt(0) * (1.-pt(0)));
+  fx += pow(M_PI, 2) * sin(M_PI * pt(0)) * dmanufacturedFun(pt);
+  fx *= sin(M_PI * pt(1));
+  return fx;
+}
+
+double obstacle(const Vector &pt)
+{
+  return 0.0;
+}

examples/contact/IPsolver.cpp

@@ -0,0 +1,827 @@
+#include "mfem.hpp"
+#include "IPsolver.hpp"
+#include "Problems.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstdlib>
+
+using namespace std;
+using namespace mfem;
+
+
+
+InteriorPointSolver::InteriorPointSolver(GeneralOptProblem * Problem) : optProblem(Problem), block_offsetsumlz(5), block_offsetsuml(4), block_offsetsx(3),
+Huu(nullptr), Hum(nullptr), Hmu(nullptr), Hmm(nullptr), Wmm(nullptr), D(nullptr), Ju(nullptr), Jm(nullptr), JuT(nullptr), JmT(nullptr), Huucl(nullptr), HLuu(nullptr), saveLogBarrierIterates(false)
+{
+  rel_tol  = 1.e-2;
+  max_iter = 20;
+  mu_k     = 1.0;
+
+  sMax     = 1.e2;
+  kSig     = 1.e10;   // control deviation from primal Hessian
+  tauMin   = 0.8;     // control rate at which iterates can approach the boundary
+  eta      = 1.e-4;   // backtracking constant
+  thetaMin = 1.e-4;   // allowed violation of the equality constraints
+
+  // constants in line-step A-5.4
+  delta    = 1.0;
+  sTheta   = 1.1;
+  sPhi     = 2.3;
+
+  // control the rate at which the penalty parameter is decreased
+  kMu     = 0.2;
+  thetaMu = 1.5;
+
+  // TO DO -- include the filter
+
+  thetaMax = 1.e6; // maximum constraint violation
+  // data for the second order correction
+  kSoc     = 0.99;
+
+  // equation (18)
+  gTheta = 1.e-5;
+  gPhi   = 1.e-5;
+
+  kEps   = 1.e1;
+
+  dimU = optProblem->GetDimU();
+  dimM = optProblem->GetDimM();
+  dimC = optProblem->GetDimC();
+  ckSoc.SetSize(dimC);
+  
+  block_offsetsumlz[0] = 0;
+  block_offsetsumlz[1] = dimU; // u
+  block_offsetsumlz[2] = dimM; // m
+  block_offsetsumlz[3] = dimC; // lambda
+  block_offsetsumlz[4] = dimM; // zl
+  block_offsetsumlz.PartialSum();
+
+  for(int i = 0; i < block_offsetsuml.Size(); i++)  { block_offsetsuml[i] = block_offsetsumlz[i]; }
+  for(int i = 0; i < block_offsetsx.Size(); i++)    { block_offsetsx[i]   = block_offsetsuml[i] ; }
+
+  // lower-bound for the inequality constraint m >= ml
+  ml = optProblem->Getml();
+  
+  lk.SetSize(dimC);  lk  = 0.0;
+  zlk.SetSize(dimM); zlk = 0.0;
+
+  linSolver = 0;
+  MyRank = 0;
+  iAmRoot = MyRank == 0 ? true : false;
+}
+
+double InteriorPointSolver::MaxStepSize(Vector &x, Vector &xl, Vector &xhat, double tau)
+{
+  double alphaMaxloc = 1.0;
+  double alphaTmp;
+  for(int i = 0; i < x.Size(); i++)
+  {
+    if( xhat(i) < 0. )
+    {
+      alphaTmp = -1. * tau * (x(i) - xl(i)) / xhat(i);
+      alphaMaxloc = min(alphaMaxloc, alphaTmp);
+    } 
+  }
+
+  // alphaMaxloc is the local maximum step size which is
+  // distinct on each MPI process. Need to compute
+  // the global maximum step size 
+  double alphaMaxglb;
+  alphaMaxglb = alphaMaxloc;
+  return alphaMaxglb;
+}
+
+double InteriorPointSolver::MaxStepSize(Vector &x, Vector &xhat, double tau)
+{
+  Vector zero(x.Size()); zero = 0.0;
+  return MaxStepSize(x, zero, xhat, tau);
+}
+
+
+void InteriorPointSolver::Mult(const Vector &x0, Vector &xf)
+{
+  BlockVector x0block(block_offsetsx); x0block = 0.0;
+  x0block.GetBlock(0).Set(1.0, x0);
+  // To do: give options for user specificiation of initialization m0
+  x0block.GetBlock(1) = 100.;
+  x0block.GetBlock(1).Add(1.0, ml);
+  BlockVector xfblock(block_offsetsx); xfblock = 0.0;
+  Mult(x0block, xfblock);
+  xf.Set(1.0, xfblock.GetBlock(0));
+}
+
+void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
+{
+  converged = false;
+  
+  BlockVector xk(block_offsetsx), xhat(block_offsetsx); xk = 0; xhat = 0.0;
+  BlockVector Xk(block_offsetsumlz), Xhat(block_offsetsumlz); Xk = 0.0; Xhat = 0.0;
+  BlockVector Xhatuml(block_offsetsuml); Xhatuml = 0.0;
+  Vector zlhat(dimM); zlhat = 0.0;
+
+  xk.GetBlock(0).Set(1.0, x0.GetBlock(0));
+  xk.GetBlock(1).Set(1.0, x0.GetBlock(1));
+  // running estimate of the final values of the Lagrange multipliers
+  lk  = 0.0;
+  zlk = 0.0;
+
+  for(int i = 0; i < dimM; i++)
+  {
+    zlk(i) = 1.e1 * mu_k / (xk(i+dimU) - ml(i));
+  }
+
+  Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+  Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+  Xk.GetBlock(2).Set(1.0, lk);
+  Xk.GetBlock(3).Set(1.0, zlk);
+
+  /* set theta0 = theta(x0)
+   *     thetaMin
+   *     thetaMax
+   * when theta(xk) < thetaMin and the switching condition holds
+   * then we ask for the Armijo sufficient decrease of the barrier
+   * objective to be satisfied, in order to accept the trial step length alphakl
+   * 
+   * thetaMax controls how the filter is initialized for each log-barrier subproblem
+   * F0 = {(th, phi) s.t. th > thetaMax}
+   * that is the filter does not allow for iterates where the constraint violation
+   * is larger than that of thetaMax
+   */
+  double theta0 = theta(xk);
+  thetaMin = 1.e-4 * max(1.0, theta0);
+  thetaMax = 1.e8  * thetaMin;
+
+  double Eeval, maxBarrierSolves, Eevalmu0;
+  bool printOptimalityError; // control optimality error print to console for log-barrier subproblems
+  
+  maxBarrierSolves = 10;
+
+  for(jOpt = 0; jOpt < max_iter; jOpt++)
+  {
+    mfem::out << "interior-point solve step " << jOpt << endl;
+    // A-2. Check convergence of overall optimization problem
+    printOptimalityError = false;
+    Eevalmu0 = E(xk, lk, zlk, printOptimalityError);
+    if(Eevalmu0 < rel_tol)
+    {
+      converged = true;
+      mfem::out << "solved optimization problem :)\n";
+      break;
+    }
+    
+    if(jOpt > 0) { maxBarrierSolves = 1; }
+    
+    for(int i = 0; i < maxBarrierSolves; i++)
+    {
+      // A-3. Check convergence of the barrier subproblem
+      printOptimalityError = true;
+      Eeval = E(xk, lk, zlk, mu_k, printOptimalityError);
+      if(Eeval < kEps * mu_k)
+      {
+        mfem::out << "solved barrier subproblem, for mu = " << mu_k << endl;
+        // A-3.1. Recompute the barrier parameter
+        mu_k  = max(rel_tol / 10., min(kMu * mu_k, pow(mu_k, thetaMu)));
+        // A-3.2. Re-initialize the filter
+        F1.DeleteAll();
+        F2.DeleteAll();
+      }
+      else
+      {
+        break;
+      }
+    }
+    
+    // A-4. Compute the search direction
+    // solve for (uhat, mhat, lhat)
+    mfem::out << "\n** A-4. IP-Newton solve **\n";
+    zlhat = 0.0; Xhatuml = 0.0;
+    // why do we have Xhatuml ....???
+    // TO DO: remove Xhatuml in favor of passing Xhat
+    IPNewtonSolve(xk, lk, zlk, zlhat, Xhatuml, mu_k, false); 
+
+
+    // assign data stack, X = (u, m, l, zl)
+    Xk = 0.0;
+    Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+    Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+    Xk.GetBlock(2).Set(1.0, lk);
+    Xk.GetBlock(3).Set(1.0, zlk);
+
+    // assign data stack, Xhat = (uhat, mhat, lhat, zlhat)
+    Xhat = 0.0;
+    for(int i = 0; i < 3; i++)
+    {
+      Xhat.GetBlock(i).Set(1.0, Xhatuml.GetBlock(i));
+    }
+    Xhat.GetBlock(3).Set(1.0, zlhat);
+
+
+    // A-5. Backtracking line search.
+    mfem::out << "\n** A-5. Linesearch **\n";
+    mfem::out << "mu = " << mu_k << endl;
+    
+    lineSearch(Xk, Xhat, mu_k);
+    
+    if(lineSearchSuccess)
+    {
+      if(!switchCondition || !sufficientDecrease)
+      {
+        F1.Append( (1. - gTheta) * thx0);
+        F2.Append( phx0 - gPhi * thx0);
+      }
+      // ----- A-6: Accept the trial point
+      // print info regarding zl...
+      xk.GetBlock(0).Add(alpha, Xhat.GetBlock(0));
+      xk.GetBlock(1).Add(alpha, Xhat.GetBlock(1));
+      lk.Add(alpha,   Xhat.GetBlock(2));
+      zlk.Add(alphaz, Xhat.GetBlock(3));
+      projectZ(xk, zlk, mu_k);
+    }
+    else
+    {
+      mfem::out << "lineSearch not successful :(\n";
+      mfem::out << "attempting feasibility restoration with theta = " << thx0 << endl;
+      mfem::out << "no feasibility restoration implemented, exiting now \n";
+      break;
+    }
+    //
+    if(jOpt + 1 == max_iter) 
+    { 
+      mfem::out << "maximum optimization iterations :(\n";
+    }
+  }
+  // done with optimization routine, just reassign data to xf reference so
+  // that the application code has access to the optimal point
+  xf = 0.0;
+  xf.GetBlock(0).Set(1.0, xk.GetBlock(0));
+  xf.GetBlock(1).Set(1.0, xk.GetBlock(1));
+}
+
+void InteriorPointSolver::FormIPNewtonMat(BlockVector & x, Vector & l, Vector &zl, BlockOperator &Ak)
+{
+  // WARNING: Huu, Hum, Hmu, Hmm should all be Hessian terms of the Lagrangian, currently we 
+  //          them by Hessian terms of the objective function and neglect the Hessian of l^T c
+
+  Huu = optProblem->Duuf(x); Hum = optProblem->Dumf(x);
+  Hmu = optProblem->Dmuf(x); Hmm = optProblem->Dmmf(x);
+
+  Vector DiagLogBar(dimM); DiagLogBar = 0.0;
+  for(int ii = 0; ii < dimM; ii++)
+  {
+    DiagLogBar(ii) = zl(ii) / (x(ii+dimU) - ml(ii));
+  }
+  if(saveLogBarrierIterates)
+  {
+    std::ofstream diagStream;
+    char diagString[100];
+    snprintf(diagString, 100, "logBarrierHessiandata/D%d.dat", jOpt);
+    diagStream.open(diagString, ios::out | ios::trunc);
+    for(int ii = 0; ii < dimM; ii++)
+    {
+      diagStream << setprecision(30) << DiagLogBar(ii) << endl;
+    }
+    diagStream.close();
+  } 
+
+
+  D = new SparseMatrix(DiagLogBar);
+  
+  if(Hmm != nullptr)
+  {
+    Wmm = new SparseMatrix(*Hmm);
+    Wmm->Add(1.0, *D);
+  }
+  else
+  {
+    Wmm = D;
+  }
+
+  Ju = optProblem->Duc(x); JuT = Transpose(*Ju);
+  Jm = optProblem->Dmc(x); JmT = Transpose(*Jm);
+ 
+  Huucl = optProblem->lDuuc(x, l);
+  if(Huucl != nullptr)
+  {
+    HLuu = Add(*Huucl, *Huu);
+    Ak.SetBlock(0, 0, HLuu);
+  } 
+  else
+  {
+    Ak.SetBlock(0, 0, Huu);
+  }
+
+  //         IP-Newton system matrix
+  //    Ak = [[H_(u,u)  H_(u,m)   J_u^T]
+  //          [H_(m,u)  W_(m,m)   J_m^T]
+  //          [ J_u      J_m       0  ]]
+
+  Ak.SetBlock(0, 2, JuT);
+  Ak.SetBlock(1, 1, Wmm); Ak.SetBlock(1, 2, JmT);
+  Ak.SetBlock(2, 0,  Ju); Ak.SetBlock(2, 1,  Jm);
+
+  if(Hum != nullptr) { Ak.SetBlock(0, 1, Hum); Ak.SetBlock(1, 0, Hmu); }
+}
+
+
+// perturbed KKT system solve
+// determine the search direction
+void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, Vector &zlhat, BlockVector &Xhat, double mu, bool socSolve)
+{
+  // solve A x = b, where A is the IP-Newton matrix
+  BlockOperator A(block_offsetsuml, block_offsetsuml); BlockVector b(block_offsetsuml); b = 0.0;
+  FormIPNewtonMat(x, l, zl, A);
+
+  //       [grad_u phi + Ju^T l]
+  // b = - [grad_m phi + Jm^T l]
+  //       [          c        ]
+  BlockVector gradphi(block_offsetsx); gradphi = 0.0;
+  BlockVector JTl(block_offsetsx); JTl = 0.0;
+  Dxphi(x, mu, gradphi);
+  
+  (A.GetBlock(0,2)).Mult(l, JTl.GetBlock(0));
+  (A.GetBlock(1,2)).Mult(l, JTl.GetBlock(1));
+
+  for(int ii = 0; ii < 2; ii++)
+  {
+    b.GetBlock(ii).Set(1.0, gradphi.GetBlock(ii));
+    b.GetBlock(ii).Add(1.0, JTl.GetBlock(ii));
+  }
+  if(!socSolve) 
+  {
+    optProblem->c(x, b.GetBlock(2));
+  }
+  else
+  {
+    b.GetBlock(2).Set(1.0, ckSoc);
+  }
+  b *= -1.0; 
+  Xhat = 0.0;
+
+
+  #ifdef MFEM_USE_SUITESPARSE
+    // Direct solve for IP-Newton saddle-point system
+    // A = [ [ Huu   0    Ju^T]
+    //       [  0    D     -I ]
+    //       [ Ju   -I      0 ]]
+    if(linSolver == 0)
+    {
+      BlockMatrix ABlockMatrix(block_offsetsuml, block_offsetsuml);
+      for(int ii = 0; ii < 3; ii++)
+      {
+        for(int jj = 0; jj < 3; jj++)
+        {
+          if(!A.IsZeroBlock(ii, jj))
+          {
+            ABlockMatrix.SetBlock(ii, jj, dynamic_cast<SparseMatrix *>(&(A.GetBlock(ii, jj))));
+          }
+        }
+      }
+      /* direct solve of the 3x3 IP-Newton linear system */
+      UMFPackSolver ASolver;
+      SparseMatrix *ASparse = ABlockMatrix.CreateMonolithic();
+      ASolver.SetOperator(*ASparse);
+      ASolver.Mult(b, Xhat);
+
+      Vector residual(Xhat.Size());
+      ASparse->Mult(Xhat, residual);
+      residual.Add(-1.0, b);
+      delete ASparse;
+    }
+    else if(linSolver == 1)
+    {
+      // Direct solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+      // where Wmm = D for contact problems
+      SparseMatrix * Huuloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0)));
+      SparseMatrix * Wmmloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1)));
+      SparseMatrix * Juloc  = dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0)));
+      SparseMatrix * JuTloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2)));
+      Vector DVec(dimM); DVec = 0.0;
+      Vector one(dimM); one = 1.0;
+      D->Mult(one, DVec);
+      SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, DVec);     // Ju^T D Ju
+      SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+
+      
+      /* prepare the reduced rhs */
+      // breduced = bu + Ju^T (bm + Wmm bl)
+      Vector breduced(dimU); breduced = 0.0;
+      Vector tempVec(dimM); tempVec = 0.0;
+      Wmmloc->Mult(b.GetBlock(2), tempVec);
+      tempVec.Add(1.0, b.GetBlock(1));
+      JuTloc->Mult(tempVec, breduced);
+      breduced.Add(1.0, b.GetBlock(0));
+
+      // solve the reduced linear system
+      UMFPackSolver AreducedSolver;
+      AreducedSolver.SetOperator(*Areduced);
+      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+
+      // now propagate solved uhat to obtain mhat and lhat
+      // xm = Ju xu - bl
+      Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+      Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+      // xl = Wmm xm - bm
+      Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+      Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+      delete JuTDJu;
+      delete Areduced;
+    }
+  #else
+    MFEM_VERIFY(linSolver > 1, "linSolver = 0, 1 require MFEM_USE_SUITESPARSE=YES");
+  #endif
+  if(linSolver > 1)
+  { 
+    // Iterative solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+    // where Wmm = D for contact problems
+    // here the iterative solver is a Jacobi-preconditioned CG-solve
+    SparseMatrix * Huuloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0)));
+    SparseMatrix * Wmmloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1)));
+    SparseMatrix * Juloc  = dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0)));
+    SparseMatrix * JuTloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2)));
+    Vector DVec(dimM); DVec = 0.0;
+    Vector one(dimM); one = 1.0;
+    D->Mult(one, DVec);
+    SparseMatrix *JuTDJu   = Mult_AtDA(*Juloc, DVec);     // Ju^T D Ju
+    SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+
+    /* prepare the reduced rhs */
+    // breduced = bu + Ju^T (bm + Wmm bl)
+    Vector breduced(dimU); breduced = 0.0;
+    Vector tempVec(dimM); tempVec = 0.0;
+    Wmmloc->Mult(b.GetBlock(2), tempVec);
+    tempVec.Add(1.0, b.GetBlock(1));
+    JuTloc->Mult(tempVec, breduced);
+    breduced.Add(1.0, b.GetBlock(0));
+    
+    if (linSolver == 2)
+    {
+      /* Jacobi preconditioned conjugate-gradient solve */
+      DSmoother AreducedPrec((SparseMatrix &)(*Areduced));
+      CGSolver AreducedSolver;
+      AreducedSolver.SetOperator(*Areduced);
+      AreducedSolver.SetAbsTol(1.e-12);
+      AreducedSolver.SetRelTol(1.e-8);
+      AreducedSolver.SetMaxIter(500);
+      AreducedSolver.SetPreconditioner(AreducedPrec);
+      AreducedSolver.SetPrintLevel(1);
+      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+    }
+    else
+    {
+      /* Gauss-Seidel preconditioned GMRES solve */
+      GSSmoother AreducedPrec((SparseMatrix &)(*Areduced));
+      GMRESSolver AreducedSolver;
+      AreducedSolver.SetOperator(*Areduced);
+      AreducedSolver.SetAbsTol(1.e-12);
+      AreducedSolver.SetRelTol(1.e-8);
+      AreducedSolver.SetMaxIter(500);
+      AreducedSolver.SetPreconditioner(AreducedPrec);
+      AreducedSolver.SetPrintLevel(1);
+      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+    }
+    
+    // now propagate solved uhat to obtain mhat and lhat
+    // xm = Ju xu - bl
+    Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+    Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+    // xl = Wmm xm - bm
+    Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+    Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+    delete JuTDJu;
+    delete Areduced;
+  }
+    
+  
+  /* backsolve to determine zlhat */
+  for(int ii = 0; ii < dimM; ii++)
+  {
+    zlhat(ii) = -1.*(zl(ii) + (zl(ii) * Xhat(ii + dimU) - mu) / (x(ii + dimU) - ml(ii)) );
+  }
+
+  // free memory
+  if(Hmm != nullptr)
+  {
+    delete Wmm;
+  }
+  if( Huucl != nullptr)
+  {
+    delete HLuu; HLuu = nullptr;
+  }
+  delete D;
+  delete JuT;
+  delete JmT;
+}
+
+// here Xhat, X will be BlockVectors w.r.t. the 4 partitioning X = (u, m, l, zl)
+
+void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double mu)
+{
+  double tau  = max(tauMin, 1.0 - mu);
+  Vector u0   = X0.GetBlock(0);
+  Vector m0   = X0.GetBlock(1);
+  Vector l0   = X0.GetBlock(2);
+  Vector z0   = X0.GetBlock(3);
+  Vector uhat = Xhat.GetBlock(0);
+  Vector mhat = Xhat.GetBlock(1);
+  Vector lhat = Xhat.GetBlock(2);
+  Vector zhat = Xhat.GetBlock(3);
+  double alphaMax  = MaxStepSize(m0, ml, mhat, tau);
+  double alphaMaxz = MaxStepSize(z0, zhat, tau);
+  alphaz = alphaMaxz;
+
+
+  BlockVector x0(block_offsetsx); x0 = 0.0;
+  x0.GetBlock(0).Set(1.0, u0);
+  x0.GetBlock(1).Set(1.0, m0);
+  
+  BlockVector xhat(block_offsetsx); xhat = 0.0;
+  xhat.GetBlock(0).Set(1.0, uhat);
+  xhat.GetBlock(1).Set(1.0, mhat);
+  
+  BlockVector xtrial(block_offsetsx); xtrial = 0.0;
+  BlockVector Dxphi0(block_offsetsx); Dxphi0 = 0.0;
+  int maxBacktrack = 20;
+  alpha = alphaMax;
+
+
+  Vector ck0(dimC); ck0 = 0.0;
+  Vector zhatsoc(dimM); zhatsoc = 0.0;
+  BlockVector Xhatumlsoc(block_offsetsuml); Xhatumlsoc = 0.0;
+  BlockVector xhatsoc(block_offsetsx); xhatsoc = 0.0;
+  Vector uhatsoc(dimU); uhatsoc = 0.0;
+  Vector mhatsoc(dimM); mhatsoc = 0.0;
+
+  Dxphi(x0, mu, Dxphi0);
+  Dxphi0_xhat = InnerProduct(Dxphi0, xhat);
+  descentDirection = Dxphi0_xhat < 0. ? true : false;
+  if(descentDirection)
+  {
+    mfem::out << "is a descent direction for the log-barrier objective\n";
+  }
+  else
+  {
+    mfem::out << "is not a descent direction for the log-barrier objective\n";
+  }
+  mfem::out << "Dxphi^T xhat / (|| Dxphi||_2 * || xhat ||_2) = " << Dxphi0_xhat / (xhat.Norml2() * Dxphi0.Norml2()) << endl;
+  thx0 = theta(x0);
+  phx0 = phi(x0, mu);
+
+  lineSearchSuccess = false;
+  for(int i = 0; i < maxBacktrack; i++)
+  {
+    mfem::out << "\n--------- alpha = " << alpha << " ---------\n";
+
+    // ----- A-5.2. Compute trial point: xtrial = x0 + alpha_i xhat
+    xtrial.Set(1.0, x0);
+    xtrial.Add(alpha, xhat);
+
+    // ------ A-5.3. if not in filter region go to A.5.4 otherwise go to A-5.5.
+    thxtrial = theta(xtrial);
+    phxtrial = phi(xtrial, mu);
+
+    filterCheck(thxtrial, phxtrial);    
+    if(!inFilterRegion)
+    {
+      mfem::out << "not in filter region :)\n";
+      // ------ A.5.4: Check sufficient decrease
+      if(!descentDirection)
+      {
+        switchCondition = false;
+      }
+      else
+      {
+        switchCondition = (alpha * pow(abs(Dxphi0_xhat), sPhi) > delta * pow(thx0, sTheta)) ? true : false;
+      }
+      mfem::out << "theta(x0) = "     << thx0     << ", thetaMin = "                  << thetaMin             << endl;
+      mfem::out << "theta(xtrial) = " << thxtrial << ", (1-gTheta) *theta(x0) = "     << (1. - gTheta) * thx0 << endl;
+      mfem::out << "phi(xtrial) = "   << phxtrial << ", phi(x0) - gPhi *theta(x0) = " << phx0 - gPhi * thx0   << endl;
+      
+      // Case I      
+      if(thx0 <= thetaMin && switchCondition)
+      {
+        sufficientDecrease = phxtrial <= phx0 + eta * alpha * Dxphi0_xhat ? true : false;
+        if(sufficientDecrease)
+        {
+          mfem::out << "Accepted step length -- sufficient decrease in log-barrier objective.\n"; 
+          // accept the trial step
+          lineSearchSuccess = true;
+          break;
+        }
+      }
+      else
+      {
+        if(thxtrial <= (1. - gTheta) * thx0 || phxtrial <= phx0 - gPhi * thx0)
+        {
+          mfem::out << "Accepted step length -- decrease in either constraint violation or log-barrier objective.\n"; 
+          // accept the trial step
+          lineSearchSuccess = true;
+          break;
+        }
+      }
+      // A-5.5: Initialize the second-order correction
+      if((!(thx0 < thxtrial)) && i == 0)
+      {
+        mfem::out << "second order correction\n";
+        optProblem->c(xtrial, ckSoc);
+        optProblem->c(x0, ck0);
+        ckSoc.Add(alphaMax, ck0);
+        // A-5.6 Compute the second-order correction.
+        IPNewtonSolve(x0, l0, z0, zhatsoc, Xhatumlsoc, mu, true);
+        mhatsoc.Set(1.0, Xhatumlsoc.GetBlock(1));
+        // alphasoc = MaxStepSize(m0, ml, mhatsoc, tau);
+        //WARNING: not complete but currently solver isn't entering this region
+      }
+    }
+    else
+    {
+      mfem::out << "in filter region\n";
+    }
+
+    // include more if needed
+    alpha *= 0.5;
+
+  } 
+}
+
+
+void InteriorPointSolver::projectZ(const Vector &x, Vector &z, double mu)
+{
+  double zi;
+  double mudivmml;
+  for(int i = 0; i < dimM; i++)
+  {
+    zi = z(i);
+    mudivmml = mu / (x(i + dimU) - ml(i));
+    z(i) = max(min(zi, kSig * mudivmml), mudivmml / kSig);
+  }
+}
+
+void InteriorPointSolver::filterCheck(double th, double ph)
+{
+  inFilterRegion = false;
+  if(th > thetaMax)
+  {
+    inFilterRegion = true;
+  }
+  else
+  {
+    for(int i = 0; i < F1.Size(); i++)
+    {
+      if(th >= F1[i] && ph >= F2[i])
+      {
+        inFilterRegion = true;
+        break;
+      }
+    }
+  }
+}
+
+double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, double mu, bool print)
+{
+  double E1, E2, E3;
+  double sc, sd;
+  BlockVector gradL(block_offsetsx); gradL = 0.0; // stationarity grad L = grad f + J^T l - z
+  Vector cx(dimC); cx = 0.0;     // feasibility c = c(x)
+  Vector comp(dimM); comp = 0.0; // complementarity M Z - mu 1
+
+  DxL(x, l, zl, gradL);
+  E1 = gradL.Normlinf();
+
+  optProblem->c(x, cx);
+  E2 = cx.Normlinf();
+
+  for(int ii = 0; ii < dimM; ii++) 
+  { 
+    comp(ii) = x(dimU + ii) * zl(ii) - mu;
+  }
+  E3 = comp.Normlinf();
+
+  double ll1, zl1;
+  zl1 = zl.Norml1() / double(dimC + dimM);
+  ll1 =  l.Norml1();
+  sc = max(sMax, zl1 / (double(dimM)) ) / sMax;
+  sd = max(sMax, (ll1 + zl1) / (double(dimC + dimM))) / sMax;
+  if(print)
+  {
+    mfem::out << "evaluating optimality error for mu = " << mu << endl;
+    mfem::out << "stationarity measure = "    << E1 / sd << endl;
+    mfem::out << "feasibility measure  = "    << E2      << endl;
+    mfem::out << "complimentarity measure = " << E3 / sc << endl;
+  }
+  return max(max(E1 / sd, E2), E3 / sc);
+}
+
+double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, bool print)
+{
+  return E(x, l, zl, 0.0, print);
+}
+
+double InteriorPointSolver::theta(const BlockVector &x)
+{
+  Vector cx(dimC); cx = 0.0;
+  optProblem->c(x, cx);
+  return cx.Norml2();
+}
+
+// log-barrier objective
+double InteriorPointSolver::phi(const BlockVector &x, double mu)
+{
+  double fx = optProblem->CalcObjective(x); 
+  double logBarrierLoc = 0.0;
+  for(int i = 0; i < dimM; i++) 
+  { 
+    logBarrierLoc += log(x(dimU+i)-ml(i));
+  }
+  double logBarrierGlb = 0.0;
+  logBarrierGlb = logBarrierLoc;
+  return fx - mu * logBarrierGlb;
+}
+
+
+
+
+// gradient of log-barrier objective with respect to x = (u, m)
+void InteriorPointSolver::Dxphi(const BlockVector &x, double mu, BlockVector &y)
+{
+  optProblem->CalcObjectiveGrad(x, y);
+  for(int i = 0; i < dimM; i++) 
+  { 
+    y(dimU + i) -= mu / (x(dimU + i));
+  } 
+}
+
+
+// Lagrangian function evaluation
+// L(x, l, zl) = f(x) + l^T c(x) - zl^T m
+double InteriorPointSolver::L(const BlockVector &x, const Vector &l, const Vector &zl)
+{
+  double fx = optProblem->CalcObjective(x);
+  Vector cx(dimC); optProblem->c(x, cx);
+  return (fx + InnerProduct(cx, l) - InnerProduct(x.GetBlock(1), zl));
+}
+
+void InteriorPointSolver::DxL(const BlockVector &x, const Vector &l, const Vector &zl, BlockVector &y)
+{
+  // evaluate the gradient of the objective with respect to the primal variables x = (u, m)
+  BlockVector gradxf(block_offsetsx); gradxf = 0.0;
+  optProblem->CalcObjectiveGrad(x, gradxf);
+  
+  SparseMatrix *Jacu, *Jacm, *JacuT, *JacmT;
+  Jacu = optProblem->Duc(x); Jacm = optProblem->Dmc(x);
+  JacuT = Transpose(*Jacu);
+  JacmT = Transpose(*Jacm);
+  JacuT->Mult(l, y.GetBlock(0));
+  JacmT->Mult(l, y.GetBlock(1));
+  delete JacuT; 
+  delete JacmT;
+  y.Add(1.0, gradxf);
+  (y.GetBlock(1)).Add(-1.0, zl);
+}
+
+
+bool InteriorPointSolver::GetConverged() const
+{
+  return converged;
+}
+
+void InteriorPointSolver::SetTol(double Tol)
+{
+  rel_tol = Tol;
+}
+
+void InteriorPointSolver::SetMaxIter(int max_it)
+{
+  max_iter = max_it;
+}
+
+void InteriorPointSolver::SetBarrierParameter(double mu_0)
+{
+  mu_k = mu_0;
+}
+
+void InteriorPointSolver::SaveLogBarrierHessianIterates(bool save)
+{
+  MFEM_ASSERT(MyRank == 0 || save == false, "currently can only save logbarrier hessian in serial codes");
+  saveLogBarrierIterates = save;
+}
+
+void InteriorPointSolver::SetLinearSolver(int LinSolver)
+{
+  linSolver = LinSolver;
+}
+
+
+
+InteriorPointSolver::~InteriorPointSolver() 
+{
+  F1.DeleteAll();
+  F2.DeleteAll();
+  block_offsetsx.DeleteAll();
+  block_offsetsumlz.DeleteAll();
+  block_offsetsuml.DeleteAll();
+  ml.SetSize(0);
+}

examples/contact/IPsolver.hpp

@@ -0,0 +1,80 @@
+#include "mfem.hpp"
+#include "Problems.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+#ifndef IPSOLVER 
+#define IPSOLVER
+
+class InteriorPointSolver
+{
+protected:
+    GeneralOptProblem* optProblem;
+    double rel_tol;
+    int  max_iter;
+    double mu_k; // \mu_k
+    Vector lk, zlk;
+
+    double sMax, kSig, tauMin, eta, thetaMin, delta, sTheta, sPhi, kMu, thetaMu;
+    double thetaMax, kSoc, gTheta, gPhi, kEps;
+	
+    // filter
+    Array<double> F1, F2;
+	
+    // quantities computed in lineSearch
+    double alpha, alphaz;
+    double thx0, thxtrial;
+    double phx0, phxtrial;
+    bool descentDirection, switchCondition, sufficientDecrease, lineSearchSuccess, inFilterRegion;
+    double Dxphi0_xhat;
+
+    int dimU, dimM, dimC;
+    Array<int> block_offsetsumlz, block_offsetsuml, block_offsetsx;
+    Vector ml;
+
+    Vector ckSoc;
+    SparseMatrix * Huu, * Hum, * Hmu, * Hmm, * Wmm, *D, * Ju, * Jm, * JuT, * JmT;
+    SparseMatrix * Huucl, *HLuu;    
+    int jOpt;
+    bool converged;
+    
+    int MyRank;
+    bool iAmRoot;
+
+    bool saveLogBarrierIterates;
+
+    int linSolver;
+
+public:
+    InteriorPointSolver(GeneralOptProblem*);
+    void Mult(const BlockVector& , BlockVector&); // used when the user wants to be aware of bound-constrained variable m >= ml
+    void Mult(const Vector&, Vector &); // useful when the user doesn't need to know about bound-constrained variable   m >= ml, e.g., when m is a slack variable
+    double MaxStepSize(Vector& , Vector& , Vector& , double);
+    double MaxStepSize(Vector& , Vector& , double);
+    void FormIPNewtonMat(BlockVector& , Vector& , Vector& , BlockOperator &);
+    void IPNewtonSolve(BlockVector& , Vector& , Vector& , Vector&, BlockVector& , double, bool);
+    void lineSearch(BlockVector& , BlockVector& , double);
+    void projectZ(const Vector & , Vector &, double);
+    void filterCheck(double, double);
+    double E(const BlockVector &, const Vector &, const Vector &, double, bool);
+    double E(const BlockVector &, const Vector &, const Vector &, bool);
+    bool GetConverged() const;
+    // TO DO: include Hessian of Lagrangian
+    double theta(const BlockVector &);
+    double phi(const BlockVector &, double);
+    void Dxphi(const BlockVector &, double, BlockVector &);
+    double L(const BlockVector &, const Vector &, const Vector &);
+    void DxL(const BlockVector &, const Vector &, const Vector &, BlockVector &);
+    void SetTol(double);
+    void SetMaxIter(int);
+    void SetBarrierParameter(double);    
+    void SaveLogBarrierHessianIterates(bool);
+    void SetLinearSolver(int);
+    virtual ~InteriorPointSolver();
+};
+
+#endif

examples/contact/ObstacleProblem.cpp

@@ -0,0 +1,125 @@
+#include "mfem.hpp"
+#include "Problems.hpp"
+#include "IPsolver.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+
+double dmanufacturedFun(const Vector &);
+double fRhs(const Vector &);
+double obstacle(const Vector &);
+
+int main(int argc, char *argv[])
+{
+  int FEorder = 1; // order of the finite elements
+  int linSolver = 0;
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  
+  OptionsParser args(argc, argv);
+  args.AddOption(&FEorder, "-o", "--order",\
+		  "Order of the finite elements.");
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+
+  const char *meshFile = "../../data/inline-quad.mesh";
+  Mesh *mesh = new Mesh(meshFile, 1, 1);
+  int dim = mesh->Dimension(); // geometric dimension of the domain
+  {
+     int ref_levels = 3;
+     for (int l = 0; l < ref_levels; l++)
+     {
+        mesh->UniformRefinement();
+     }
+  }
+
+  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
+  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
+  ObstacleProblem problem(Vh, &fRhs, &obstacle);
+  
+  int dimD = problem.GetDimD();
+  Vector x0(dimD); x0 = 0.0;
+  Vector xf(dimD); xf = 0.0;
+
+  InteriorPointSolver optimizer(&problem); 
+  optimizer.SetTol(1.e-7);
+  optimizer.SetLinearSolver(linSolver);
+  optimizer.SetMaxIter(maxIPMiters);
+  optimizer.Mult(x0, xf);
+
+  double Einitial = problem->E(x0);
+  double Efinal = problem->E(xf);
+  cout << "Energy objective at initial point = " << Einitial << endl;
+  cout << "Energy objective at QP optimizer = " << Efinal << endl;
+
+
+  GridFunction d_gf(Vh);
+
+  d_gf = xf;
+
+  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
+  GridFunction dm_gf(Vh);
+  dm_gf.ProjectCoefficient(dm_fc);
+  
+  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
+  paraview_dc.SetPrefixPath("ParaView");
+  paraview_dc.SetLevelsOfDetail(FEorder);
+  paraview_dc.SetDataFormat(VTKFormat::BINARY);
+  paraview_dc.SetHighOrderOutput(true);
+  paraview_dc.SetCycle(0);
+  paraview_dc.SetTime(0.0);
+  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
+  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
+  paraview_dc.Save();
+  
+  
+  delete Vh;
+  delete fec;
+  delete mesh;
+  return 0;
+}
+
+
+double dmanufacturedFun(const Vector &x)
+{
+  return cos(2*M_PI*x(0)) + 0.2 - 2.0*(pow(x(0),3) - 1.5*pow(x(0),2));
+}
+
+
+// f(x) forcing term... which enters the objective energy functional
+// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
+// of f(x). f(x) is such that in the absence of bound-constraints then
+// the solution of the optimization problem satisfies the PDE
+// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
+// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
+double fRhs(const Vector &x)
+{
+  double fx = 0.;
+  fx = 0.2 - 2.0 * (pow(x(0),3)- 1.5*pow(x(0),2.) - 6 * x(0) + 3.) + (1. + pow(2.*M_PI,2))*cos(2.*M_PI*x(0));
+  return fx;
+}
+
+double obstacle(const Vector &x)
+{
+  return 0.0;
+}

examples/contact/ParIPsolver.cpp

@@ -0,0 +1,834 @@
+#include "mfem.hpp"
+#include "ParIPsolver.hpp"
+#include "ParProblems.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstdlib>
+
+using namespace std;
+using namespace mfem;
+
+
+ParInteriorPointSolver::ParInteriorPointSolver(ParGeneralOptProblem * problem_) 
+                     : problem(problem_), 
+                       block_offsetsumlz(5), block_offsetsuml(4), block_offsetsx(3),
+                       Huu(nullptr), Hum(nullptr), Hmu(nullptr), 
+                       Hmm(nullptr), Wmm(nullptr), D(nullptr), 
+                       Ju(nullptr), Jm(nullptr), JuT(nullptr), JmT(nullptr), 
+                       saveLogBarrierIterates(false)
+{
+   OptTol  = 1.e-2;
+   max_iter = 20;
+   mu_k     = 1.0;
+
+   sMax     = 1.e2;
+   kSig     = 1.e10;   // control deviation from primal Hessian
+   tauMin   = 0.8;     // control rate at which iterates can approach the boundary
+   eta      = 1.e-4;   // backtracking constant
+   thetaMin = 1.e-4;   // allowed violation of the equality constraints
+
+   // constants in line-step A-5.4
+   delta    = 1.0;
+   sTheta   = 1.1;
+   sPhi     = 2.3;
+
+   // control the rate at which the penalty parameter is decreased
+   kMu     = 0.2;
+   thetaMu = 1.5;
+
+   thetaMax = 1.e6; // maximum constraint violation
+   // data for the second order correction
+   kSoc     = 0.99;
+
+   // equation (18)
+   gTheta = 1.e-5;
+   gPhi   = 1.e-5;
+
+   kEps   = 1.e1;
+
+   dimU = problem->GetDimU();
+   dimM = problem->GetDimM();
+   dimC = problem->GetDimC();
+   MPI_Allreduce(&dimU, &dimUglb, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+   MPI_Allreduce(&dimM, &dimMglb, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+   MPI_Allreduce(&dimC, &dimCglb, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+   ckSoc.SetSize(dimC);
+  
+   block_offsetsumlz[0] = 0;
+   block_offsetsumlz[1] = dimU; // u
+   block_offsetsumlz[2] = dimM; // m
+   block_offsetsumlz[3] = dimC; // lambda
+   block_offsetsumlz[4] = dimM; // zl
+   block_offsetsumlz.PartialSum();
+
+   for(int i = 0; i < block_offsetsuml.Size(); i++)  
+   { 
+      block_offsetsuml[i] = block_offsetsumlz[i]; 
+   }
+   for(int i = 0; i < block_offsetsx.Size(); i++)    
+   { 
+      block_offsetsx[i] = block_offsetsuml[i] ; 
+   }
+
+  
+   ml = problem->Getml();
+  
+   lk.SetSize(dimC);  lk  = 0.0;
+   zlk.SetSize(dimM); zlk = 0.0;
+
+   linSolver = 0;
+   linSolveTol = 1.e-8;
+   MyRank = Mpi::WorldRank();
+   iAmRoot = MyRank == 0 ? true : false;
+}
+
+double ParInteriorPointSolver::MaxStepSize(Vector &x, Vector &xl, Vector &xhat, double tau)
+{
+   double alphaMaxloc = 1.0;
+   double alphaTmp;
+   for(int i = 0; i < x.Size(); i++)
+   {   
+      if( xhat(i) < 0. )
+      {
+         alphaTmp = -1. * tau * (x(i) - xl(i)) / xhat(i);
+         alphaMaxloc = min(alphaMaxloc, alphaTmp);
+      } 
+   }
+
+   // alphaMaxloc is the local maximum step size which is
+   // distinct on each MPI process. Need to compute
+   // the global maximum step size 
+   double alphaMaxglb;
+   MPI_Allreduce(&alphaMaxloc, &alphaMaxglb, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
+   return alphaMaxglb;
+}
+
+double ParInteriorPointSolver::MaxStepSize(Vector &x, Vector &xhat, double tau)
+{
+   Vector zero(x.Size()); zero = 0.0;
+   return MaxStepSize(x, zero, xhat, tau);
+}
+
+
+void ParInteriorPointSolver::Mult(const Vector &x0, Vector &xf)
+{
+   BlockVector x0block(block_offsetsx); x0block = 0.0;
+   x0block.GetBlock(0).Set(1.0, x0);
+   x0block.GetBlock(1) = 100.;
+   x0block.GetBlock(1).Add(1.0, ml);
+   BlockVector xfblock(block_offsetsx); xfblock = 0.0;
+   Mult(x0block, xfblock);
+   xf.Set(1.0, xfblock.GetBlock(0));
+}
+
+
+void ParInteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
+{
+   converged = false;
+   
+   BlockVector xk(block_offsetsx), xhat(block_offsetsx); xk = 0; xhat = 0.0;
+   BlockVector Xk(block_offsetsumlz), Xhat(block_offsetsumlz); Xk = 0.0; Xhat = 0.0;
+   BlockVector Xhatuml(block_offsetsuml); Xhatuml = 0.0;
+   Vector zlhat(dimM); zlhat = 0.0;
+
+   xk.GetBlock(0).Set(1.0, x0.GetBlock(0));
+   xk.GetBlock(1).Set(1.0, x0.GetBlock(1));
+   // running estimate of the final values of the Lagrange multipliers
+   lk  = 0.0;
+   zlk = 0.0;
+
+   for(int i = 0; i < dimM; i++)
+   {
+      zlk(i) = 1.e1 * mu_k / (xk(i+dimU) - ml(i));
+   }
+
+   Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+   Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+   Xk.GetBlock(2).Set(1.0, lk);
+   Xk.GetBlock(3).Set(1.0, zlk);
+
+  /* set theta0 = theta(x0)
+   *     thetaMin
+   *     thetaMax
+   * when theta(xk) < thetaMin and the switching condition holds
+   * then we ask for the Armijo sufficient decrease of the barrier
+   * objective to be satisfied, in order to accept the trial step length alphakl
+   * 
+   * thetaMax controls how the filter is initialized for each log-barrier subproblem
+   * F0 = {(th, phi) s.t. th > thetaMax}
+   * that is the filter does not allow for iterates where the constraint violation
+   * is larger than that of thetaMax
+   */
+   double theta0 = theta(xk);
+   thetaMin = 1.e-4 * max(1.0, theta0);
+   thetaMax = 1.e8  * thetaMin; // 1.e4 * max(1.0, theta0)
+
+   double Eeval, maxBarrierSolves, Eevalmu0;
+   bool printOptimalityError; // control optimality error print to console for log-barrier subproblems
+   
+   maxBarrierSolves = 10;
+
+   for(jOpt = 0; jOpt < max_iter; jOpt++)
+   {
+      if(iAmRoot)
+      {
+         cout << "interior-point solve step " << jOpt << endl;
+      }
+      // A-2. Check convergence of overall optimization problem
+      printOptimalityError = false;
+      Eevalmu0 = E(xk, lk, zlk, printOptimalityError);
+      if(Eevalmu0 < OptTol)
+      {
+         converged = true;
+         if(iAmRoot)
+         {
+            cout << "solved optimization problem :)\n";
+         }
+         break;
+      }
+      
+      if(jOpt > 0) { maxBarrierSolves = 1; }
+      
+      for(int i = 0; i < maxBarrierSolves; i++)
+      {
+         // A-3. Check convergence of the barrier subproblem
+         printOptimalityError = true;
+         Eeval = E(xk, lk, zlk, mu_k, printOptimalityError);
+         if(iAmRoot)
+         {
+            cout << "E = " << Eeval << endl;
+         }
+         if(Eeval < kEps * mu_k)
+         {
+            if(iAmRoot)
+            {
+               cout << "solved barrier subproblem :), for mu = " << mu_k << endl;
+            }
+            // A-3.1. Recompute the barrier parameter
+            mu_k  = max(OptTol / 10., min(kMu * mu_k, pow(mu_k, thetaMu)));
+            // A-3.2. Re-initialize the filter
+            F1.DeleteAll();
+            F2.DeleteAll();
+         }
+         else
+         {
+            break;
+         }
+      }
+    
+      // A-4. Compute the search direction
+      // solve for (uhat, mhat, lhat)
+      if(iAmRoot)
+      {
+         cout << "\n** A-4. IP-Newton solve **\n";
+      }
+      zlhat = 0.0; Xhatuml = 0.0;
+      // why do we have Xhatuml ....???
+      // TO DO: remove Xhatuml in favor of passing Xhat
+      IPNewtonSolve(xk, lk, zlk, zlhat, Xhatuml, mu_k, false); 
+
+      // assign data stack, X = (u, m, l, zl)
+      Xk = 0.0;
+      Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+      Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+      Xk.GetBlock(2).Set(1.0, lk);
+      Xk.GetBlock(3).Set(1.0, zlk);
+
+      // assign data stack, Xhat = (uhat, mhat, lhat, zlhat)
+      Xhat = 0.0;
+      for(int i = 0; i < 3; i++)
+      {
+         Xhat.GetBlock(i).Set(1.0, Xhatuml.GetBlock(i));
+      }
+      Xhat.GetBlock(3).Set(1.0, zlhat);
+
+      // A-5. Backtracking line search.
+      if(iAmRoot)
+      {
+         cout << "\n** A-5. Linesearch **\n";
+         cout << "mu = " << mu_k << endl;
+      }
+      lineSearch(Xk, Xhat, mu_k);
+
+      if(lineSearchSuccess)
+      {
+         if(iAmRoot)
+         {
+            cout << "lineSearch successful :)\n";
+         }
+         if(!switchCondition || !sufficientDecrease)
+         {
+            F1.Append( (1. - gTheta) * thx0);
+            F2.Append( phx0 - gPhi * thx0);
+         }
+         // ----- A-6: Accept the trial point
+         // print info regarding zl...
+         xk.GetBlock(0).Add(alpha, Xhat.GetBlock(0));
+         xk.GetBlock(1).Add(alpha, Xhat.GetBlock(1));
+         lk.Add(alpha,   Xhat.GetBlock(2));
+         zlk.Add(alphaz, Xhat.GetBlock(3));
+         projectZ(xk, zlk, mu_k);
+      }
+      else
+      {
+         if(iAmRoot)
+         {
+            cout << "lineSearch not successful :(\n";
+            cout << "attempting feasibility restoration with theta = " << thx0 << endl;
+            cout << "no feasibility restoration implemented, exiting now \n";
+         }
+         break;
+      }
+      if(jOpt + 1 == max_iter && iAmRoot) 
+      {  
+         cout << "maximum optimization iterations :(\n";
+      }
+   }
+   // done with optimization routine, just reassign data to xf reference so
+   // that the application code has access to the optimal point
+   xf = 0.0;
+   xf.GetBlock(0).Set(1.0, xk.GetBlock(0));
+   xf.GetBlock(1).Set(1.0, xk.GetBlock(1));
+}
+
+void ParInteriorPointSolver::FormIPNewtonMat(BlockVector & x, Vector & l, Vector &zl, BlockOperator &Ak)
+{
+   // WARNING: Huu, Hum, Hmu, Hmm should all be Hessian terms of the Lagrangian, currently we 
+   //          them by Hessian terms of the objective function and neglect the Hessian of l^T c
+
+   Huu = problem->Duuf(x); 
+   Hum = problem->Dumf(x);
+   Hmu = problem->Dmuf(x);
+   Hmm = problem->Dmmf(x);
+
+   Vector DiagLogBar(dimM); DiagLogBar = 0.0;
+   for(int ii = 0; ii < dimM; ii++)
+   {
+      DiagLogBar(ii) = zl(ii) / (x(ii+dimU) - ml(ii));
+   }
+   if(saveLogBarrierIterates)
+   {
+      std::ofstream diagStream;
+      char diagString[100];
+      snprintf(diagString, 100, "logBarrierHessiandata/D%d.dat", jOpt);
+      diagStream.open(diagString, ios::out | ios::trunc);
+      for(int ii = 0; ii < dimM; ii++)
+      {
+         diagStream << setprecision(30) << DiagLogBar(ii) << endl;
+      }
+      diagStream.close();
+   } 
+
+   SparseMatrix * Ds = new SparseMatrix(DiagLogBar);
+   ParFiniteElementSpace * fes = problem->GetfesM();
+   D = new HypreParMatrix(fes->GetComm(), fes->GlobalTrueVSize(), fes->GetTrueDofOffsets(), Ds);
+   HypreStealOwnership(*D,*Ds);
+   delete Ds;
+  
+   if(Hmm != nullptr)
+   {
+      Wmm = Hmm;
+      Wmm->Add(1.0, *D);
+   }
+   else
+   {
+      Wmm = D;
+   }
+
+   Ju = problem->Duc(x); JuT = Ju->Transpose();
+   Jm = problem->Dmc(x); JmT = Jm->Transpose();
+  
+   //         IP-Newton system matrix
+   //    Ak = [[H_(u,u)  H_(u,m)   J_u^T]
+   //          [H_(m,u)  W_(m,m)   J_m^T]
+   //          [ J_u      J_m       0  ]]
+
+   Ak.SetBlock(0, 0, Huu);                         Ak.SetBlock(0, 2, JuT);
+                           Ak.SetBlock(1, 1, Wmm); Ak.SetBlock(1, 2, JmT);
+   Ak.SetBlock(2, 0,  Ju); Ak.SetBlock(2, 1,  Jm);
+
+   if(Hum != nullptr) { Ak.SetBlock(0, 1, Hum); Ak.SetBlock(1, 0, Hmu); }
+}
+
+// perturbed KKT system solve
+// determine the search direction
+void ParInteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, Vector &zlhat, BlockVector &Xhat, double mu, bool socSolve)
+{
+   // solve A x = b, where A is the IP-Newton matrix
+   BlockOperator A(block_offsetsuml, block_offsetsuml); BlockVector b(block_offsetsuml); b = 0.0;
+   FormIPNewtonMat(x, l, zl, A);
+
+   //       [grad_u phi + Ju^T l]
+   // b = - [grad_m phi + Jm^T l]
+   //       [          c        ]
+   BlockVector gradphi(block_offsetsx); gradphi = 0.0;
+   BlockVector JTl(block_offsetsx); JTl = 0.0;
+   Dxphi(x, mu, gradphi);
+   
+   (A.GetBlock(0,2)).Mult(l, JTl.GetBlock(0));
+   (A.GetBlock(1,2)).Mult(l, JTl.GetBlock(1));
+
+   for(int ii = 0; ii < 2; ii++)
+   {
+      b.GetBlock(ii).Set(1.0, gradphi.GetBlock(ii));
+      b.GetBlock(ii).Add(1.0, JTl.GetBlock(ii));
+   }
+   if(!socSolve) 
+   {
+      problem->c(x, b.GetBlock(2));
+   }
+   else
+   {
+      b.GetBlock(2).Set(1.0, ckSoc);
+   }
+   b *= -1.0; 
+   Xhat = 0.0;
+
+
+   // Direct solver (default)
+   if(linSolver == 0)
+   {
+      Array2D<HypreParMatrix *> ABlockMatrix(3,3);
+      for(int ii = 0; ii < 3; ii++)
+      {
+      for(int jj = 0; jj < 3; jj++)
+      {
+         if(!A.IsZeroBlock(ii, jj))
+         {
+            ABlockMatrix(ii, jj) = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(ii, jj)));
+	 }
+	 else
+	 {
+	    ABlockMatrix(ii, jj) = nullptr;
+	 }
+      }
+      }
+      
+      HypreParMatrix * Ah = HypreParMatrixFromBlocks(ABlockMatrix);   
+      
+      /* direct solve of the 3x3 IP-Newton linear system */
+      #ifdef MFEM_USE_MUMPS
+        MUMPSSolver ASolver;
+        ASolver.SetPrintLevel(0);
+        ASolver.SetMatrixSymType(MUMPSSolver::MatType::SYMMETRIC_INDEFINITE);
+        ASolver.SetOperator(*Ah);
+        ASolver.Mult(b, Xhat);
+      #else 
+        #ifdef MFEM_USE_MKL_CPARDISO
+          CPardisoSolver ASolver(MPI_COMM_WORLD);
+          ASolver.SetOperator(*Ah);
+          ASolver.Mult(b, Xhat);
+	#else
+	  MFEM_VERIFY(false, "linSolver 0 will not work unless compiled with MUMPS or MKL");
+        #endif
+      #endif
+
+      delete Ah;
+   }
+   else if(linSolver == 1 || linSolver == 2)
+   {
+      // form A = Huu + Ju^T D Ju, Wmm = D for contact
+      HypreParMatrix * Huuloc = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(0, 0)));
+      HypreParMatrix * Wmmloc = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(1, 1)));
+      HypreParMatrix * Juloc  = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(2, 0)));
+      HypreParMatrix * JuTloc = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(0, 2)));
+      
+      
+      HypreParMatrix *JuTDJu   = RAP(Wmmloc, Juloc);     // Ju^T D Ju
+      HypreParMatrix *Areduced = ParAdd(Huuloc, JuTDJu);  // Huu + Ju^T D Ju
+      /* prepare the reduced rhs */
+      // breduced = bu + Ju^T (bm + Wmm bl)
+      Vector breduced(dimU); breduced = 0.0;
+      Vector tempVec(dimM); tempVec = 0.0;
+      Wmmloc->Mult(b.GetBlock(2), tempVec);
+      tempVec.Add(1.0, b.GetBlock(1));
+      JuTloc->Mult(tempVec, breduced);
+      breduced.Add(1.0, b.GetBlock(0));
+      
+      if(linSolver == 1)
+      {
+         // setup the solver for the reduced linear system
+         #ifdef MFEM_USE_MUMPS
+	   MUMPSSolver AreducedSolver;   
+           AreducedSolver.SetPrintLevel(0);
+           AreducedSolver.SetMatrixSymType(MUMPSSolver::MatType::SYMMETRIC_INDEFINITE);
+	   AreducedSolver.SetOperator(*Areduced);
+	   AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+         #else 
+           #ifdef MFEM_USE_MKL_CPARDISO
+	     CPardisoSolver AreducedSolver(MPI_COMM_WORLD);
+	     AreducedSolver.SetOperator(*Areduced);
+	     AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+           #else
+	     MFEM_VERIFY(false, "linSolver 1 will not work unless compiled with MUMPS or MKL");
+           #endif
+         #endif
+      }
+      else
+      {
+         HyprePCG AreducedSolver(MPI_COMM_WORLD);
+	 AreducedSolver.SetOperator(*Areduced);
+	 HypreBoomerAMG AreducedPrec;
+         AreducedSolver.SetTol(linSolveTol);
+         AreducedSolver.SetMaxIter(500);
+         AreducedSolver.SetPreconditioner(AreducedPrec);
+         AreducedSolver.SetResidualConvergenceOptions(); // convergence criteria based on residual norm
+	 AreducedSolver.SetPrintLevel(2);
+         AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+      }
+
+      // now propagate solved uhat to obtain mhat and lhat
+      // xm = Ju xu - bl
+      Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+      Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+      // xl = Wmm xm - bm
+      Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+      Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+      delete JuTDJu;
+      delete Areduced;
+   }
+
+   /* backsolve to determine zlhat */
+   for(int ii = 0; ii < dimM; ii++)
+   {
+      zlhat(ii) = -1.*(zl(ii) + (zl(ii) * Xhat(ii + dimU) - mu) / (x(ii + dimU) - ml(ii)) );
+   }
+
+   // free memory
+   delete D;
+   delete JuT;
+   delete JmT;
+   if(Hmm != nullptr)
+   {
+      delete Wmm;
+   }
+}
+
+// here Xhat, X will be BlockVectors w.r.t. the 4 partitioning X = (u, m, l, zl)
+
+void ParInteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double mu)
+{
+   double tau  = max(tauMin, 1.0 - mu);
+   Vector u0   = X0.GetBlock(0);
+   Vector m0   = X0.GetBlock(1);
+   Vector l0   = X0.GetBlock(2);
+   Vector z0   = X0.GetBlock(3);
+   Vector uhat = Xhat.GetBlock(0);
+   Vector mhat = Xhat.GetBlock(1);
+   Vector lhat = Xhat.GetBlock(2);
+   Vector zhat = Xhat.GetBlock(3);
+   double alphaMax  = MaxStepSize(m0, ml, mhat, tau);
+   double alphaMaxz = MaxStepSize(z0, zhat, tau);
+   alphaz = alphaMaxz;
+
+   BlockVector x0(block_offsetsx); x0 = 0.0;
+   x0.GetBlock(0).Set(1.0, u0);
+   x0.GetBlock(1).Set(1.0, m0);
+   
+   BlockVector xhat(block_offsetsx); xhat = 0.0;
+   xhat.GetBlock(0).Set(1.0, uhat);
+   xhat.GetBlock(1).Set(1.0, mhat);
+   
+   BlockVector xtrial(block_offsetsx); xtrial = 0.0;
+   BlockVector Dxphi0(block_offsetsx); Dxphi0 = 0.0;
+   int maxBacktrack = 20;
+   alpha = alphaMax;
+
+   Vector ck0(dimC); ck0 = 0.0;
+   Vector zhatsoc(dimM); zhatsoc = 0.0;
+   BlockVector Xhatumlsoc(block_offsetsuml); Xhatumlsoc = 0.0;
+   BlockVector xhatsoc(block_offsetsx); xhatsoc = 0.0;
+   Vector uhatsoc(dimU); uhatsoc = 0.0;
+   Vector mhatsoc(dimM); mhatsoc = 0.0;
+
+   Dxphi(x0, mu, Dxphi0);
+
+   Dxphi0_xhat = InnerProduct(MPI_COMM_WORLD, Dxphi0, xhat);
+   descentDirection = Dxphi0_xhat < 0. ? true : false;
+   if(descentDirection)
+   {
+      if (iAmRoot)
+      {
+         cout << "is a descent direction for the log-barrier objective\n";
+      }
+   }
+   else
+   {
+      cout << "is not a descent direction for the log-barrier objective\n";
+   }
+   thx0 = theta(x0);
+   phx0 = phi(x0, mu);
+
+   lineSearchSuccess = false;
+   for(int i = 0; i < maxBacktrack; i++)
+   {
+      if (iAmRoot)
+      {
+         cout << "\n--------- alpha = " << alpha << " ---------\n";
+      }
+      // ----- A-5.2. Compute trial point: xtrial = x0 + alpha_i xhat
+      xtrial.Set(1.0, x0);
+      xtrial.Add(alpha, xhat);
+
+      // ------ A-5.3. if not in filter region go to A.5.4 otherwise go to A-5.5.
+      thxtrial = theta(xtrial);
+      phxtrial = phi(xtrial, mu);
+      filterCheck(thxtrial, phxtrial);    
+      if(!inFilterRegion)
+      {
+         if (iAmRoot)
+         {
+            cout << "not in filter region :)\n";
+         }
+         // ------ A.5.4: Check sufficient decrease
+         if(!descentDirection)
+         {
+            switchCondition = false;
+         }
+         else
+         {
+            switchCondition = (alpha * pow(abs(Dxphi0_xhat), sPhi) > delta * pow(thx0, sTheta)) ? true : false;
+         }
+         if (iAmRoot)
+         {
+            cout << "theta(x0) = "     << thx0     << ", thetaMin = "                  << thetaMin             << endl;
+            cout << "theta(xtrial) = " << thxtrial << ", (1-gTheta) *theta(x0) = "     << (1. - gTheta) * thx0 << endl;
+            cout << "phi(xtrial) = "   << phxtrial << ", phi(x0) - gPhi *theta(x0) = " << phx0 - gPhi * thx0   << endl;
+         }      
+         // Case I      
+         if(thx0 <= thetaMin && switchCondition)
+         {
+            sufficientDecrease = (phxtrial <= phx0 + eta * alpha * Dxphi0_xhat) ? true : false;
+            if(sufficientDecrease)
+            {
+               if(iAmRoot) { cout << "Line search successful: sufficient decrease in log-barrier objective.\n"; } 
+               // accept the trial step
+               lineSearchSuccess = true;
+               break;
+            }
+         }
+         else
+         {
+            if(thxtrial <= (1. - gTheta) * thx0 || phxtrial <= phx0 - gPhi * thx0)
+            {
+               if(iAmRoot) { cout << "Line search successful: infeasibility or log-barrier objective decreased.\n"; } 
+               // accept the trial step
+               lineSearchSuccess = true;
+               break;
+            }
+         }
+         // A-5.5: Initialize the second-order correction
+         if((!(thx0 < thxtrial)) && i == 0)
+         {
+            if (iAmRoot)
+            {
+               cout << "second order correction\n";
+            }
+            problem->c(xtrial, ckSoc);
+            problem->c(x0, ck0);
+            ckSoc.Add(alphaMax, ck0);
+            // A-5.6 Compute the second-order correction.
+            IPNewtonSolve(x0, l0, z0, zhatsoc, Xhatumlsoc, mu, true);
+            mhatsoc.Set(1.0, Xhatumlsoc.GetBlock(1));
+            //WARNING: not complete but currently solver isn't entering this region
+         }
+      }
+      else
+      {
+         if (iAmRoot)
+         {
+            cout << "in filter region :(\n"; 
+         }
+      }
+      // include more if needed
+      alpha *= 0.5;
+
+   } 
+}
+
+
+void ParInteriorPointSolver::projectZ(const Vector &x, Vector &z, double mu)
+{
+   double zi;
+   double mudivmml;
+   for(int i = 0; i < dimM; i++)
+   {
+      zi = z(i);
+      mudivmml = mu / (x(i + dimU) - ml(i));
+      z(i) = max(min(zi, kSig * mudivmml), mudivmml / kSig);
+   }
+}
+
+void ParInteriorPointSolver::filterCheck(double th, double ph)
+{
+   inFilterRegion = false;
+   if(th > thetaMax)
+   {
+      inFilterRegion = true;
+   }
+   else
+   {
+      for(int i = 0; i < F1.Size(); i++)
+      {
+         if(th >= F1[i] && ph >= F2[i])
+         {
+            inFilterRegion = true;
+            break;
+         }
+      }
+   }
+}
+
+double ParInteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, double mu, bool printEeval)
+{
+   double E1, E2, E3;
+   double sc, sd;
+   BlockVector gradL(block_offsetsx); gradL = 0.0; // stationarity grad L = grad f + J^T l - z
+   Vector cx(dimC); cx = 0.0;     // feasibility c = c(x)
+   Vector comp(dimM); comp = 0.0; // complementarity M Z - mu 1
+
+   DxL(x, l, zl, gradL);
+   E1 = GlobalLpNorm(infinity(), gradL.Normlinf(), MPI_COMM_WORLD); 
+
+   problem->c(x, cx);
+   E2 = GlobalLpNorm(infinity(), cx.Normlinf(), MPI_COMM_WORLD); 
+
+
+   for(int ii = 0; ii < dimM; ii++) 
+   { 
+      comp(ii) = x(dimU + ii) * zl(ii) - mu;
+   }
+   E3 = GlobalLpNorm(infinity(), comp.Normlinf(), MPI_COMM_WORLD); 
+
+   double ll1, zl1;
+
+   zl1 = GlobalLpNorm(1, zl.Norml1(), MPI_COMM_WORLD)/ double(dimCglb + dimMglb); 
+   ll1 = GlobalLpNorm(1, l.Norml1(), MPI_COMM_WORLD);
+   sc = max(sMax, zl1 / (double(dimMglb)) ) / sMax;
+   sd = max(sMax, (ll1 + zl1) / (double(dimCglb + dimMglb))) / sMax;
+   if(iAmRoot && printEeval)
+   {
+      cout << "evaluating optimality error for mu = " << mu << endl;
+      cout << "stationarity measure = "    << E1 / sd << endl;
+      cout << "feasibility measure  = "    << E2      << endl;
+      cout << "complimentarity measure = " << E3 / sc << endl;
+   }
+   return max(max(E1 / sd, E2), E3 / sc);
+}
+
+double ParInteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, bool printEeval)
+{
+  return E(x, l, zl, 0.0, printEeval);
+}
+
+double ParInteriorPointSolver::theta(const BlockVector &x)
+{
+  Vector cx(dimC); cx = 0.0;
+  problem->c(x, cx);
+  return sqrt(InnerProduct(MPI_COMM_WORLD,cx, cx));
+}
+
+// log-barrier objective
+double ParInteriorPointSolver::phi(const BlockVector &x, double mu)
+{
+   double fx = problem->CalcObjective(x); 
+   double logBarrierLoc = 0.0;
+   for(int i = 0; i < dimM; i++) 
+   { 
+     logBarrierLoc += log(x(dimU+i)-ml(i));
+   }
+   double logBarrierGlb;
+   MPI_Allreduce(&logBarrierLoc, &logBarrierGlb, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+   return fx - mu * logBarrierGlb;
+}
+
+// gradient of log-barrier objective with respect to x = (u, m)
+void ParInteriorPointSolver::Dxphi(const BlockVector &x, double mu, BlockVector &y)
+{
+   problem->CalcObjectiveGrad(x, y);
+   
+   for(int i = 0; i < dimM; i++) 
+   { 
+      y(dimU + i) -= mu / (x(dimU + i));
+   } 
+}
+
+// Lagrangian function evaluation
+// L(x, l, zl) = f(x) + l^T c(x) - zl^T m
+double ParInteriorPointSolver::L(const BlockVector &x, const Vector &l, const Vector &zl)
+{
+   double fx = problem->CalcObjective(x);
+   Vector cx(dimC); problem->c(x, cx);
+   return (fx + InnerProduct(MPI_COMM_WORLD,cx, l) - InnerProduct(MPI_COMM_WORLD, x.GetBlock(1), zl));
+}
+
+void ParInteriorPointSolver::DxL(const BlockVector &x, const Vector &l, const Vector &zl, BlockVector &y)
+{
+   // evaluate the gradient of the objective with respect to the primal variables x = (u, m)
+   BlockVector gradxf(block_offsetsx); gradxf = 0.0;
+   problem->CalcObjectiveGrad(x, gradxf);
+   
+   HypreParMatrix *Jacu, *Jacm, *JacuT, *JacmT;
+   Jacu = problem->Duc(x); 
+   Jacm = problem->Dmc(x);
+   JacuT = Jacu->Transpose();
+   JacmT = Jacm->Transpose();
+   
+   JacuT->Mult(l, y.GetBlock(0));
+   JacmT->Mult(l, y.GetBlock(1));
+   
+   delete JacuT;
+   delete JacmT;
+   
+   y.Add(1.0, gradxf);
+   (y.GetBlock(1)).Add(-1.0, zl);
+}
+
+bool ParInteriorPointSolver::GetConverged() const
+{
+   return converged;
+}
+
+void ParInteriorPointSolver::SetTol(double Tol)
+{
+   OptTol = Tol;
+}
+
+void ParInteriorPointSolver::SetMaxIter(int max_it)
+{
+   max_iter = max_it;
+}
+
+void ParInteriorPointSolver::SetBarrierParameter(double mu_0)
+{
+   mu_k = mu_0;
+}
+
+void ParInteriorPointSolver::SaveLogBarrierHessianIterates(bool save)
+{
+   MFEM_ASSERT(MyRank == 0 || save == false, "currently can only save logbarrier hessian in serial codes");
+   saveLogBarrierIterates = save;
+}
+
+void ParInteriorPointSolver::SetLinearSolver(int LinSolver)
+{
+   linSolver = LinSolver;
+}
+
+void ParInteriorPointSolver::SetLinearSolveTol(double Tol)
+{
+  linSolveTol = Tol;
+}
+
+
+ParInteriorPointSolver::~ParInteriorPointSolver() 
+{
+   F1.DeleteAll();
+   F2.DeleteAll();
+   block_offsetsx.DeleteAll();
+   block_offsetsumlz.DeleteAll();
+   block_offsetsuml.DeleteAll();
+   ml.SetSize(0);
+}

examples/contact/ParIPsolver.hpp

@@ -0,0 +1,82 @@
+#include "mfem.hpp"
+#include "ParProblems.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+#ifndef PARIPSOLVER 
+#define PARIPSOLVER
+
+class ParInteriorPointSolver
+{
+protected:
+    ParGeneralOptProblem* problem;
+    double OptTol;
+    int  max_iter;
+    double mu_k; // \mu_k
+    Vector lk, zlk;
+
+    double sMax, kSig, tauMin, eta, thetaMin, delta, sTheta, sPhi, kMu, thetaMu;
+    double thetaMax, kSoc, gTheta, gPhi, kEps;
+	
+    // filter
+    Array<double> F1, F2;
+	
+    // quantities computed in lineSearch
+    double alpha, alphaz;
+    double thx0, thxtrial;
+    double phx0, phxtrial;
+    bool descentDirection, switchCondition, sufficientDecrease, lineSearchSuccess, inFilterRegion;
+    double Dxphi0_xhat;
+
+    int dimU, dimM, dimC;
+    int dimUglb, dimMglb, dimCglb;
+    Array<int> block_offsetsumlz, block_offsetsuml, block_offsetsx;
+    Vector ml;
+
+    Vector ckSoc;
+    HypreParMatrix * Huu, * Hum, * Hmu, * Hmm, * Wmm, *D, * Ju, * Jm, * JuT, * JmT;
+    
+    int jOpt;
+    bool converged;
+    
+    int MyRank;
+    bool iAmRoot;
+
+    bool saveLogBarrierIterates;
+
+    int linSolver;
+    double linSolveTol;
+public:
+    ParInteriorPointSolver(ParGeneralOptProblem*);
+    double MaxStepSize(Vector& , Vector& , Vector& , double);
+    double MaxStepSize(Vector& , Vector& , double);
+    void Mult(const BlockVector& , BlockVector&);
+    void Mult(const Vector&, Vector &); 
+    void FormIPNewtonMat(BlockVector& , Vector& , Vector& , BlockOperator &);
+    void IPNewtonSolve(BlockVector& , Vector& , Vector& , Vector&, BlockVector& , double, bool);
+    void lineSearch(BlockVector& , BlockVector& , double);
+    void projectZ(const Vector & , Vector &, double);
+    void filterCheck(double, double);
+    double E(const BlockVector &, const Vector &, const Vector &, double, bool);
+    double E(const BlockVector &, const Vector &, const Vector &, bool);
+    bool GetConverged() const;
+    // TO DO: include Hessian of Lagrangian
+    double theta(const BlockVector &);
+    double phi(const BlockVector &, double);
+    void Dxphi(const BlockVector &, double, BlockVector &);
+    double L(const BlockVector &, const Vector &, const Vector &);
+    void DxL(const BlockVector &, const Vector &, const Vector &, BlockVector &);
+    void SetTol(double);
+    void SetMaxIter(int);
+    void SetBarrierParameter(double);    
+    void SaveLogBarrierHessianIterates(bool);
+    void SetLinearSolver(int);
+    void SetLinearSolveTol(double);
+    virtual ~ParInteriorPointSolver();
+};
+
+#endif

examples/contact/ParObstacleProblem.cpp

@@ -0,0 +1,129 @@
+//                         Obstacle Problem
+//
+//
+// Compile with: make ParObstacleProblem
+//
+// Sample runs: mpirun -np 4 ./ParObstacleProblem
+//
+//
+// Description: This example code demonstrates the use of MFEM to solve the
+//              bound-constrained energy minimization problem
+//
+//                      minimize (||∇u||² + ||u||²) subject to u ≥ ϕ in H¹.
+
+#include "mfem.hpp"
+#include "ParProblems.hpp"
+#include "ParIPsolver.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double dmanufacturedFun(const Vector &);
+double fRhs(const Vector &);
+
+int main(int argc, char *argv[])
+{
+  // Initialize MPI
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   int FEorder = 1; // order of the finite elements
+   int linSolver = 2;
+   int maxIPMiters = 30;
+   int ref_levels = 3;
+   OptionsParser args(argc, argv);
+   args.AddOption(&FEorder, "-o", "--order",\
+	 	  "Order of the finite elements.");
+   args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+        "IP-Newton linear system solution strategy.");
+   args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+	 	  "Maximum number of IPM iterations");
+   args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+  
+   args.Parse();
+   if(!args.Good())
+   {
+     args.PrintUsage(cout);
+     return 1;
+   }
+   else
+   {
+     if(Mpi::Root())
+     {  
+       args.PrintOptions(cout);
+     }
+   }
+
+   const char *meshFile = "../../data/inline-quad.mesh";
+   Mesh mesh(meshFile, 1, 1);
+   int dim = mesh.Dimension(); // geometric dimension of the meshed domain
+   {
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
+   ParFiniteElementSpace   *Vh  = new ParFiniteElementSpace(&pmesh, fec);
+
+   ParObstacleProblem problem(Vh,Vh,&fRhs);
+  
+   int dimD = problem.GetDimD();
+   Vector x0(dimD); x0 = 100.0;
+   Vector xf(dimD); xf = 0.0;
+
+   ParInteriorPointSolver optimizer(&problem); 
+   optimizer.SetTol(1.e-8);
+   optimizer.SetLinearSolveTol(1.e-10);
+   optimizer.SetLinearSolver(linSolver);
+   optimizer.SetMaxIter(maxIPMiters);
+   optimizer.Mult(x0, xf);
+
+   ParGridFunction d_gf(Vh);
+
+   d_gf.SetFromTrueDofs(xf);
+
+
+   FunctionCoefficient dm_fc(dmanufacturedFun); // manufactured solution
+   ParGridFunction dm_gf(Vh);
+   dm_gf.ProjectCoefficient(dm_fc);
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream exact_sock(vishost, visport);
+   exact_sock.precision(8);
+   exact_sock << "parallel " << num_procs << " " << myid << "\n";
+   exact_sock << "solution\n" << pmesh << dm_gf
+              << "window_title 'Manufactured solution'" << flush;
+
+   socketstream sol_sock(vishost, visport);
+   sol_sock.precision(8);
+   sol_sock << "parallel " << num_procs << " " << myid << "\n";
+   sol_sock << "solution\n" << pmesh << d_gf 
+            << "window_title 'Numerical solution'" << flush;
+
+   delete Vh;
+   delete fec;
+   return 0;
+}
+
+
+double dmanufacturedFun(const Vector &x)
+{
+  return cos(2*M_PI*x(0)) + 0.2 - 2.0*(pow(x(0),3) - 1.5*pow(x(0),2));
+}
+
+double fRhs(const Vector &x)
+{
+  double fx = 0.;
+  fx = 0.2 - 2.0 * (pow(x(0),3)- 1.5*pow(x(0),2.) - 6 * x(0) + 3.) + (1. + pow(2.*M_PI,2))*cos(2.*M_PI*x(0));
+  return fx;
+}

examples/contact/ParProblems.cpp

@@ -0,0 +1,212 @@
+#include "mfem.hpp"
+#include "ParProblems.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+ParGeneralOptProblem::ParGeneralOptProblem(ParFiniteElementSpace * fesU_, ParFiniteElementSpace * fesM_) 
+: fesU(fesU_), fesM(fesM_) 
+{ 
+   dimU = fesU->GetTrueVSize();
+   dimM = fesM->GetTrueVSize();
+   dimC = fesM->GetTrueVSize();
+}
+
+void ParGeneralOptProblem::CalcObjectiveGrad(const BlockVector &x, BlockVector &y) const
+{
+   Duf(x, y.GetBlock(0));
+   Dmf(x, y.GetBlock(1));
+}
+
+ParGeneralOptProblem::~ParGeneralOptProblem()
+{
+   block_offsetsx.DeleteAll();
+}
+
+
+// min E(d) s.t. g(d) >= 0
+// min_(d,s) E(d) s.t. c(d,s) := g(d) - s = 0, s >= 0
+ParOptProblem::ParOptProblem(ParFiniteElementSpace * fesU_, 
+                                     ParFiniteElementSpace * fesM_)
+ : ParGeneralOptProblem(fesU_, fesM_), block_offsetsx(3)
+{
+   block_offsetsx[0] = 0;
+   block_offsetsx[1] = dimU;
+   block_offsetsx[2] = dimM;
+   block_offsetsx.PartialSum();
+   ml.SetSize(dimM); ml = 0.0;
+   Vector negIdentDiag(dimM);
+   negIdentDiag = -1.0;
+   SparseMatrix * diag = new SparseMatrix(negIdentDiag);
+   Ih = new HypreParMatrix(fesM->GetComm(), fesM->GlobalTrueVSize(),
+                                             fesM->GetTrueDofOffsets(), diag);
+   HypreStealOwnership(*Ih, *diag);
+   delete diag;
+}
+
+double ParOptProblem::CalcObjective(const BlockVector &x) const { return E(x.GetBlock(0)); }
+
+void ParOptProblem::Duf(const BlockVector &x, Vector &y) const { DdE(x.GetBlock(0), y); }
+
+void ParOptProblem::Dmf(const BlockVector &x, Vector &y) const { y = 0.0; }
+
+HypreParMatrix * ParOptProblem::Duuf(const BlockVector &x) 
+{ 
+   return DddE(x.GetBlock(0)); 
+}
+
+HypreParMatrix * ParOptProblem::Dumf(const BlockVector &x) { return nullptr; }
+
+HypreParMatrix * ParOptProblem::Dmuf(const BlockVector &x) { return nullptr; }
+
+HypreParMatrix * ParOptProblem::Dmmf(const BlockVector &x) { return nullptr; }
+
+void ParOptProblem::c(const BlockVector &x, Vector &y) const // c(u,m) = g(u) - m 
+{
+   g(x.GetBlock(0), y);
+   y.Add(-1.0, x.GetBlock(1));  
+}
+
+HypreParMatrix * ParOptProblem::Duc(const BlockVector &x) 
+{ 
+   return Ddg(x.GetBlock(0)); 
+}
+
+HypreParMatrix * ParOptProblem::Dmc(const BlockVector &x) 
+{ 
+   return Ih;
+} 
+
+ParOptProblem::~ParOptProblem() 
+{
+   delete Ih;
+}
+
+
+// Obstacle Problem, no essential boundary conditions enforced
+// Hessian of energy term is K + M (stiffness + mass)
+ParObstacleProblem::ParObstacleProblem(ParFiniteElementSpace *fesU_, 
+                                       ParFiniteElementSpace *fesM_, 
+                                       double (*fSource)(const Vector &)) : 
+                                       ParOptProblem(fesU_,fesM_), f(dimU), psi(dimU), J(nullptr)
+{
+   Kform = new ParBilinearForm(fesU);
+   Kform->AddDomainIntegrator(new MassIntegrator);
+   Kform->AddDomainIntegrator(new DiffusionIntegrator);
+   Kform->Assemble();
+   Kform->Finalize();
+   Kform->FormSystemMatrix(ess_tdof_list, K);
+
+   FunctionCoefficient fcoeff(fSource);
+   fform = new ParLinearForm(fesU);
+   fform->AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
+   fform->Assemble();
+   Vector F(dimU);
+   fform->ParallelAssemble(F);
+   f.SetSize(dimU);
+   f.Set(1.0, F);
+
+   psi = 0.0;
+   
+   Vector iDiag(dimU); iDiag = 1.0;
+   SparseMatrix * Jacg = new SparseMatrix(iDiag);
+   
+   J = new HypreParMatrix(fesU->GetComm(),fesU->GlobalTrueVSize(),fesU->GetTrueDofOffsets(),Jacg);
+   HypreStealOwnership(*J, *Jacg);
+   delete Jacg;
+}
+
+// Obstacle Problem, essential boundary conditions enforced
+// Hessian of energy term is K (stiffness)
+ParObstacleProblem::ParObstacleProblem(ParFiniteElementSpace *fesU_, 
+                                       ParFiniteElementSpace *fesM_, 
+				       double (*fSource)(const Vector &),
+				       double (*obstacleSource)(const Vector &),
+				       Array<int> tdof_list, Vector &xDC) : ParOptProblem(fesU_,fesM_), f(dimU), psi(dimU), J(nullptr)
+{
+   // elastic energy functional terms	
+   ess_tdof_list = tdof_list;
+   Kform = new ParBilinearForm(fesU);
+   Kform->AddDomainIntegrator(new DiffusionIntegrator);
+   Kform->Assemble();
+   Kform->Finalize();
+   Kform->FormSystemMatrix(ess_tdof_list, K);
+
+   FunctionCoefficient fcoeff(fSource);
+   fform = new ParLinearForm(fesU);
+   fform->AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
+   fform->Assemble();
+   Vector F(dimU);
+   fform->ParallelAssemble(F);
+   f.SetSize(dimU);
+   f.Set(1.0, F);
+   Kform->EliminateVDofsInRHS(ess_tdof_list, xDC, f);
+   
+   // obstacle constraints --  
+   Vector iDiag(dimU); iDiag = 1.0;
+   for(int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+     iDiag(ess_tdof_list[i]) = 0.0;
+   }
+   SparseMatrix * Jacg = new SparseMatrix(iDiag);
+
+   J = new HypreParMatrix(fesU->GetComm(),fesU->GlobalTrueVSize(),fesU->GetTrueDofOffsets(),Jacg);
+   HypreStealOwnership(*J, *Jacg);
+   delete Jacg;
+
+   FunctionCoefficient psi_fc(obstacleSource);
+   ParGridFunction psi_gf(fesU);
+   psi_gf.ProjectCoefficient(psi_fc);
+   psi.Set(1.0, (*psi_gf.GetTrueDofs()));
+   for(int i = 0; i < ess_tdof_list.Size(); i++)
+   {
+     psi(ess_tdof_list[i]) -= 1.e-8;
+   }
+}
+
+
+
+double ParObstacleProblem::E(const Vector &d) const
+{
+   Vector Kd(K.Height()); Kd = 0.0;
+   MFEM_VERIFY(d.Size() == K.Width(), "ParObstacleProblem::E - Inconsistent dimensions");
+   K.Mult(d, Kd);
+   return 0.5 * InnerProduct(MPI_COMM_WORLD, d, Kd) - InnerProduct(MPI_COMM_WORLD, f, d);
+}
+
+void ParObstacleProblem::DdE(const Vector &d, Vector &gradE) const
+{
+   gradE.SetSize(K.Height());
+   MFEM_VERIFY(d.Size() == K.Width(), "ParObstacleProblem::DdE - Inconsistent dimensions");
+   K.Mult(d, gradE);
+   MFEM_VERIFY(f.Size() == K.Height(), "ParObstacleProblem::DdE - Inconsistent dimensions");
+   gradE.Add(-1.0, f);
+}
+
+HypreParMatrix * ParObstacleProblem::DddE(const Vector &d)
+{
+   return &K; 
+}
+
+// g(d) = d >= \psi
+void ParObstacleProblem::g(const Vector &d, Vector &gd) const
+{
+   MFEM_VERIFY(d.Size() == J->Width(), "ParObstacleProblem::g - Inconsistent dimensions");
+   J->Mult(d, gd);
+   MFEM_VERIFY(gd.Size() == J->Height(), "ParObstacleProblem::g - Inconsistent dimensions");
+   gd.Add(-1.0, psi);
+}
+
+HypreParMatrix * ParObstacleProblem::Ddg(const Vector &d)
+{
+   return J;
+}
+
+ParObstacleProblem::~ParObstacleProblem()
+{
+   delete Kform;
+   delete fform;
+   delete J;
+}

examples/contact/ParProblems.hpp

@@ -0,0 +1,107 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+#ifndef PARPROBLEM_DEFS
+#define PARPROBLEM_DEFS
+
+// abstract ParGeneralOptProblem class
+// of the form
+// min_(u,m) f(u,m) s.t. c(u,m)=0 and m>=ml
+// the primal variable (u, m) is represented as a BlockVector
+// think about supporting general lower and upper bounds (see HiOP user manual) 
+
+class ParGeneralOptProblem
+{
+protected:
+    int dimU, dimM, dimC;
+    ParFiniteElementSpace * fesU = nullptr;
+    ParFiniteElementSpace * fesM = nullptr;
+    Array<int> block_offsetsx;
+    Vector ml;
+public:
+    ParGeneralOptProblem(ParFiniteElementSpace * fesU_, ParFiniteElementSpace * fesM_); // constructor
+    virtual double CalcObjective(const BlockVector &) const = 0;
+    virtual void Duf(const BlockVector &, Vector &) const = 0;
+    virtual void Dmf(const BlockVector &, Vector &) const = 0;
+    void CalcObjectiveGrad(const BlockVector &, BlockVector &) const;
+    virtual HypreParMatrix * Duuf(const BlockVector &) = 0;
+    virtual HypreParMatrix * Dumf(const BlockVector &) = 0;
+    virtual HypreParMatrix * Dmuf(const BlockVector &) = 0;
+    virtual HypreParMatrix * Dmmf(const BlockVector &) = 0;
+    virtual HypreParMatrix * Duc(const BlockVector &) = 0;
+    virtual HypreParMatrix * Dmc(const BlockVector &) = 0;
+    // TO DO: include Hessian terms of constraint c
+    virtual void c(const BlockVector &, Vector &) const = 0;
+    int GetDimU() const { return dimU; };
+    int GetDimM() const { return dimM; }; 
+    int GetDimC() const { return dimC; };
+    ParFiniteElementSpace * GetfesU() {return fesU;}
+    ParFiniteElementSpace * GetfesM() {return fesM;}
+    Vector Getml() const { return ml; };
+    ~ParGeneralOptProblem(); // destructor
+};
+
+
+// abstract ContactProblem class
+// of the form
+// min_d e(d) s.t. g(d) >= 0
+class ParOptProblem : public ParGeneralOptProblem
+{
+protected:
+    Array<int> block_offsetsx;
+    HypreParMatrix * Ih;
+public:
+    ParOptProblem(ParFiniteElementSpace * fesU_, ParFiniteElementSpace * fesM_);        // constructor
+    double CalcObjective(const BlockVector &) const; // objective e
+    void Duf(const BlockVector &, Vector &) const;
+    void Dmf(const BlockVector &, Vector &) const;
+
+    HypreParMatrix * Duuf(const BlockVector &);
+    HypreParMatrix * Dumf(const BlockVector &);
+    HypreParMatrix * Dmuf(const BlockVector &);
+    HypreParMatrix * Dmmf(const BlockVector &);
+    HypreParMatrix * Duc(const BlockVector &);
+    HypreParMatrix * Dmc(const BlockVector &);
+
+    void c(const BlockVector &, Vector &) const;
+    virtual double E(const Vector &) const = 0;               // objective e(d) (energy function)
+    virtual void DdE(const Vector &, Vector &) const = 0;      // gradient of objective De / Dd
+    virtual HypreParMatrix * DddE(const Vector &) = 0;
+    // Hessian of objective D^2 e / D d^2
+    virtual HypreParMatrix * Ddg(const Vector &) = 0;
+    // Jacobian of inequality constraint Dg / Dd
+    virtual void g(const Vector &, Vector &) const = 0;       // inequality constraint g(d) >= 0 (gap function)
+    int GetDimD() const { return fesU->GetTrueVSize(); };
+    int GetDimS() const { return fesM->GetTrueVSize(); };
+    virtual ~ParOptProblem();
+};
+
+class ParObstacleProblem : public ParOptProblem
+{
+protected:
+   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d >= \psi
+   // stiffness matrix used to define objective
+   ParBilinearForm *Kform;
+   ParLinearForm   *fform;
+   Array<int> ess_tdof_list; // needed for calls to FormSystemMatrix
+   HypreParMatrix  K;
+   HypreParMatrix  *J;
+   ParFiniteElementSpace *Vh;
+   Vector f;
+   Vector psi;
+public :
+   ParObstacleProblem(ParFiniteElementSpace*, ParFiniteElementSpace*, double (*fSource)(const Vector &));
+   ParObstacleProblem(ParFiniteElementSpace*, ParFiniteElementSpace*, double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list, Vector &);
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   HypreParMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   HypreParMatrix* Ddg(const Vector &);
+   virtual ~ParObstacleProblem();
+};
+
+#endif

examples/contact/ParSphericalObstacleProblem.cpp

@@ -0,0 +1,173 @@
+//                       Spherical Obstacle Problem
+//
+//
+// Compile with: make ParSphericalObstacleProblem
+//
+// Sample runs: mpirun -np 4 ./ParSphericalObstacleProblem -linSolver 0
+//              mpirun -np 4 ./ParSphericalObstacleProblem -linSolver 1
+//              mpirun -np 4 ./ParSphericalObstacleProblem -linSolver 2
+//
+//
+// Description: This example code demonstrates the use of MFEM to solve the
+//              bound-constrained energy minimization problem
+//
+//                      minimize ||∇u||² subject to u ≥ ϕ in H¹₀.
+
+#include "mfem.hpp"
+#include "ParProblems.hpp"
+#include "ParIPsolver.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double fRhs(const Vector &);
+double spherical_obstacle(const Vector &);
+double exact_solution_obstacle(const Vector &);
+
+
+int main(int argc, char *argv[])
+{
+  // Initialize MPI
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   int FEorder = 1; // order of the finite elements
+   int linSolver = 2;
+   int maxIPMiters = 30;
+   int ref_levels = 3;
+   OptionsParser args(argc, argv);
+   args.AddOption(&FEorder, "-o", "--order",\
+	 	  "Order of the finite elements.");
+   args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+        "IP-Newton linear system solution strategy.");
+   args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+	 	  "Maximum number of IPM iterations");
+   args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+  
+   args.Parse();
+   if(!args.Good())
+   {
+     args.PrintUsage(cout);
+     return 1;
+   }
+   else
+   {
+     if(myid == 0)
+     {  
+       args.PrintOptions(cout);
+     }
+   }
+
+   const char *meshFile = "../../data/disk.mesh";
+   Mesh mesh(meshFile, 1, 1);
+   int dim = mesh.Dimension(); // geometric dimension of the meshed domain
+   {
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   
+   FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
+   ParFiniteElementSpace   *Vh  = new ParFiniteElementSpace(&pmesh, fec);
+   Array<int> boundary_dofs;
+   Vh->GetBoundaryTrueDofs(boundary_dofs);
+   int dimD = Vh->GetTrueVSize();
+   Vector xDC(dimD); xDC = 0.0;
+
+   ParObstacleProblem problem(Vh, Vh, &fRhs, &spherical_obstacle, boundary_dofs, xDC);
+   Vector x0(dimD); x0.Set(1.0, xDC);
+   Vector xf(dimD); xf = 0.0;
+
+   ParInteriorPointSolver optimizer(&problem); 
+   optimizer.SetTol(1.e-7);
+   optimizer.SetLinearSolveTol(1.e-10);
+   optimizer.SetLinearSolver(linSolver);
+   optimizer.SetMaxIter(maxIPMiters);
+   optimizer.Mult(x0, xf);
+
+   ParGridFunction d_gf(Vh);
+
+   d_gf.SetFromTrueDofs(xf);
+
+
+   FunctionCoefficient dtrue_fc(exact_solution_obstacle); // analytic solution
+   ParGridFunction dtrue_gf(Vh);
+   dtrue_gf.ProjectCoefficient(dtrue_fc);
+
+   double L2error = d_gf.ComputeL2Error(dtrue_fc);
+   if (myid == 0)
+   {
+     cout << "\n|| u_h - u ||_{L^2} = " << L2error << '\n' << endl;
+   } 
+   
+   ParaViewDataCollection paraview_dc("SphericalObstacleProblem", &pmesh);
+   paraview_dc.SetPrefixPath("ParaView");
+   paraview_dc.SetLevelsOfDetail(FEorder);
+   paraview_dc.SetDataFormat(VTKFormat::BINARY);
+   paraview_dc.SetHighOrderOutput(true);
+   paraview_dc.SetCycle(0);
+   paraview_dc.SetTime(0.0);
+   paraview_dc.RegisterField("u(x,y) (analytic)", &dtrue_gf);
+   paraview_dc.RegisterField("u(x,y) (numerical)", &d_gf);
+   paraview_dc.Save();
+
+   delete Vh;
+   delete fec;
+   return 0;
+}
+
+
+double fRhs(const Vector &x)
+{
+  return 0.;
+}
+
+
+double spherical_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double beta = 0.9;
+
+   double b = r0*beta;
+   double tmp = sqrt(r0*r0 - b*b);
+   double B = tmp + b*b/tmp;
+   double C = -b/tmp;
+
+   if (r > b)
+   {
+      return B + r * C;
+   }
+   else
+   {
+      return sqrt(r0*r0 - r*r);
+   }
+}
+
+double exact_solution_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      return A * log(r);
+   }
+   else
+   {
+      return sqrt(r0*r0-r*r);
+   }
+} 
+

examples/contact/Problems.hpp

@@ -0,0 +1,300 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <set>
+
+using namespace std;
+using namespace mfem;
+
+#ifndef PROBLEM_DEFS
+#define PROBLEM_DEFS
+
+
+
+// abstract GeneralOptProblem class
+// for the problem
+// min_(u,m) f(u,m) 
+// such that c(u,m)=0 and m >= ml
+class GeneralOptProblem
+{
+protected:
+    int dimU, dimM, dimC;
+    Array<int> block_offsetsx;
+    Vector ml;
+public:
+    GeneralOptProblem();
+    virtual double CalcObjective(const BlockVector &) const = 0;
+    virtual void Duf(const BlockVector &, Vector &) const = 0;
+    virtual void Dmf(const BlockVector &, Vector &) const = 0;
+    void CalcObjectiveGrad(const BlockVector &, BlockVector &) const;
+    virtual SparseMatrix* Duuf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dumf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmuf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmmf(const BlockVector &) = 0;
+    virtual void c(const BlockVector &, Vector &) const = 0;
+    virtual SparseMatrix* Duc(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmc(const BlockVector &) = 0;
+    virtual SparseMatrix* lDuuc(const BlockVector &, const Vector &) = 0;
+    virtual SparseMatrix* lDumc(const BlockVector &, const Vector &) = 0;
+    virtual SparseMatrix* lDmuc(const BlockVector &, const Vector &) = 0;
+    virtual SparseMatrix* lDmmc(const BlockVector &, const Vector &) = 0;
+    // TO DO: include log-barrier lumped-mass and pass that
+    // to the optimizer
+    //virtual SparseMatrix* GetLogBarrierLumpedMass() = 0;
+    int GetDimU() const { return dimU; };
+    int GetDimM() const { return dimM; }; 
+    int GetDimC() const { return dimC; };
+    Vector Getml() const { return ml; };
+    ~GeneralOptProblem();
+};
+
+
+// abstract OptProblem class
+// of the form
+// min_d e(d) s.t. g(d) >= 0
+class OptProblem : public GeneralOptProblem
+{
+protected:
+    int dimD;
+    int dimS;
+    Array<int> block_offsetsx;
+    SparseMatrix * negIdentity;
+    SparseMatrix * zeroMatum;
+    SparseMatrix * zeroMatmu;
+    SparseMatrix * zeroMatmm;
+public:
+    //OptProblem(int, int);        // constructor
+    OptProblem();
+    void InitializeParentData(int, int);
+    double CalcObjective(const BlockVector &) const; // objective e
+    void Duf(const BlockVector &, Vector &) const;
+    void Dmf(const BlockVector &, Vector &) const;
+    SparseMatrix* Duuf(const BlockVector &);
+    SparseMatrix* Dumf(const BlockVector &);
+    SparseMatrix* Dmuf(const BlockVector &);
+    SparseMatrix* Dmmf(const BlockVector &);
+    void c(const BlockVector &, Vector &) const;
+    SparseMatrix* Duc(const BlockVector &);
+    SparseMatrix* Dmc(const BlockVector &);
+    SparseMatrix* lDuuc(const BlockVector &, const Vector &);
+    SparseMatrix* lDumc(const BlockVector &, const Vector &);
+    SparseMatrix* lDmuc(const BlockVector &, const Vector &);
+    SparseMatrix* lDmmc(const BlockVector &, const Vector &);
+    virtual double E(const Vector &) const = 0;           // objective e(d) (energy function)
+    virtual void DdE(const Vector &, Vector &) const = 0; // gradient of objective De / Dd
+    virtual SparseMatrix* DddE(const Vector &) = 0;       // Hessian of objective D^2 e / D d^2
+    virtual void g(const Vector &, Vector &) const = 0;   // inequality constraint g(d) >= 0 (gap function)
+    virtual SparseMatrix* Ddg(const Vector &) = 0;        // Jacobian of inequality constraint Dg / Dd
+    virtual SparseMatrix* lDddg(const Vector &, const Vector &) = 0;
+    int GetDimD() const { return dimD; };
+    int GetDimS() const { return dimS; };
+    virtual ~OptProblem();
+};
+
+
+class ObstacleProblem : public OptProblem
+{
+protected:
+   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d + \psi >= 0
+   // stiffness matrix used to define objective
+   BilinearForm *Kform;
+   LinearForm   *fform;
+   Array<int> ess_tdof_list;
+   SparseMatrix  *K;
+   SparseMatrix  *J;
+   SparseMatrix  *Hcl;
+   FiniteElementSpace *Vh;
+   Vector f;
+   Vector psil;
+   Vector psiu;
+   bool twoBounds;
+   Vector xDC;
+   double Ce;
+public : 
+   ObstacleProblem(FiniteElementSpace*, double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &));
+   ObstacleProblem(FiniteElementSpace*, Vector&,  double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list);
+   ObstacleProblem(FiniteElementSpace*, Vector &, double (*fSource)(const Vector &), double (*obstacleSourcel)(const Vector &), double (*obstacleSourceu)(const Vector &), Array<int> tdof_list);
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   SparseMatrix * lDddg(const Vector &, const Vector &);
+   virtual ~ObstacleProblem();
+};
+
+
+SparseMatrix * GenerateProjector(int n, Array<int> ess_tdof_list);
+
+
+class ObstacleProblemVariant : public OptProblem
+{
+protected:
+   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d + \psi >= 0
+   // stiffness matrix used to define objective
+   BilinearForm *Kform;
+   LinearForm   *fform;
+   Array<int> ess_tdof_list;
+   Array<int> noness_tdof_list;
+   SparseMatrix  *K;
+   SparseMatrix  *RKP; // R K P = R K R^T
+   SparseMatrix  *J;
+   SparseMatrix  *Hcl;
+   SparseMatrix *R;
+   FiniteElementSpace *Vh;
+   Vector f;
+   Vector psil;
+   Vector xDC;
+   double Ce;
+public : 
+   ObstacleProblemVariant(FiniteElementSpace*, Vector&,  double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list);
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   SparseMatrix * lDddg(const Vector &, const Vector &);
+   virtual ~ObstacleProblemVariant();
+};
+
+
+
+
+class QPOptProblem : public OptProblem
+{
+protected:
+  SparseMatrix *K;
+  SparseMatrix *J;
+  SparseMatrix *zeroMatdd;
+  Vector f;
+  Vector g0;
+public:
+  QPOptProblem(const SparseMatrix, const SparseMatrix, const Vector, const Vector);
+  double E(const Vector &) const;
+  void DdE(const Vector &, Vector &) const;
+  SparseMatrix* DddE(const Vector &);
+  void g(const Vector &, Vector &) const;
+  SparseMatrix* Ddg(const Vector &);
+  SparseMatrix * lDddg(const Vector &, const Vector &);
+  virtual ~QPOptProblem();
+};
+
+
+class ExContactBlockTL : public OptProblem
+{
+public:
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   SparseMatrix * lDddg(const Vector &, const Vector &);
+   FiniteElementSpace GetVh1();
+   FiniteElementSpace GetVh2();
+   SparseMatrix *zeroMatdd;
+public:
+   /** default constructor */
+   ExContactBlockTL(Mesh *, Mesh *, int);
+   
+
+   /** default destructor */
+   virtual ~ExContactBlockTL();
+
+private:
+   void update_g() const;
+
+private:
+   /**@name Methods to block default compiler methods.
+    *
+    * The compiler automatically generates the following three methods.
+    *  Since the default compiler implementation is generally not what
+    *  you want (for all but the most simple classes), we usually
+    *  put the declarations of these methods in the private section
+    *  and never implement them. This prevents the compiler from
+    *  implementing an incorrect "default" behavior without us
+    *  knowing. (See Scott Meyers book, "Effective C++")
+    */
+   ExContactBlockTL(
+      const ExContactBlockTL&
+   );
+
+   ExContactBlockTL& operator=(
+      const ExContactBlockTL&
+   );
+
+   Array<int> attr;
+   Array<int> m_attr;
+   Array<int> s_conn; // connectivity of the second/slave mesh
+   std::string mesh_file1;
+   std::string mesh_file2;
+   Mesh* mesh1;
+   Mesh* mesh2;
+   FiniteElementCollection* fec1;
+   FiniteElementCollection* fec2;
+   FiniteElementSpace* fespace1;
+   FiniteElementSpace* fespace2;
+   Array<int> ess_tdof_list1;
+   Array<int> ess_tdof_list2;
+   GridFunction nodes0;
+   GridFunction* nodes1;
+   GridFunction* nodes2;
+   mutable GridFunction* x1;
+   mutable GridFunction* x2;
+   PWConstCoefficient* lambda1_func;
+   PWConstCoefficient* lambda2_func;
+   PWConstCoefficient* mu1_func;
+   PWConstCoefficient* mu2_func;
+   BilinearForm* a1;
+   BilinearForm* a2;
+
+   mfem::Vector lambda1;
+   mfem::Vector lambda2;
+   mfem::Vector mu1;
+   mfem::Vector mu2;
+   mutable mfem::Vector xyz;
+
+   std::set<int> bdryVerts2;
+
+   int dim;
+   // degrees of freedom of both meshes
+   int ndof_1;
+   int ndof_2;
+   int ndofs;
+   // number of nodes for each mesh
+   int nnd_1;
+   int nnd_2;
+   int nnd;
+
+   int npoints;
+
+   SparseMatrix A1;
+   mfem::Vector B1, X1;
+   SparseMatrix A2;
+   mfem::Vector B2, X2;
+   BlockVector *B;
+   SparseMatrix* K;
+   mutable mfem::Vector gapv;
+   mutable mfem::Vector m_xi;
+   mutable mfem::Vector xs;
+
+   mutable Array<int> m_conn; // only works for linear elements that have 4 vertices!
+   mutable DenseMatrix* coordsm;
+   mutable SparseMatrix* M;
+
+   mutable std::vector<SparseMatrix>* dM;
+
+   Array<int> Dirichlet_dof;
+   Array<double> Dirichlet_val;
+   Array<int> block_offsets;
+public: 
+   Mesh * GetMesh1() {return mesh1;}
+   Mesh * GetMesh2() {return mesh2;}
+   GridFunction & GetMesh1GridFunction() {return *x1;}
+   GridFunction & GetMesh2GridFunction() {return *x2;}
+   Array<int> & GetMesh1DirichletDofs() {return ess_tdof_list1;}
+   Array<int> & GetMesh2DirichletDofs() {return ess_tdof_list2;}
+};
+
+
+
+#endif

examples/contact/SphericalObstacleProblem.cpp

@@ -0,0 +1,175 @@
+//                          Spherical Obstacle Problem
+//
+//
+// Compile with: make SphericalobstacleProblem
+//
+// Sample runs: ./SphericalobstacleProblem
+//
+//
+// Description: This example code demonstrates the use of MFEM to solve the
+//              bound-constrained energy minimization problem
+//
+//                      minimize ||∇u||² subject to u ≥ ϕ in H¹₀.
+
+
+#include "mfem.hpp"
+#include "Problems.hpp"
+#include "IPsolver.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+double fRhs(const Vector &);
+double spherical_obstacle(const Vector &);
+double exact_solution_obstacle(const Vector &);
+
+int main(int argc, char *argv[])
+{
+  int FEorder = 1;     // finite element order
+  int linSolver = 0;   // linear solver 0 (direct), 1 (iterative) or 2 (iterative)
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  int ref_levels = 3;
+  
+  OptionsParser args(argc, argv);
+  args.AddOption(&FEorder, "-o", "--order",\
+		  "Order of the finite elements.");
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+  
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+
+  const char *meshFile = "../../data/disk.mesh";
+  Mesh *mesh = new Mesh(meshFile, 1, 1);
+  int dim = mesh->Dimension(); // geometric dimension of the domain
+  {
+     for (int l = 0; l < ref_levels; l++)
+     {
+        mesh->UniformRefinement();
+     }
+  }
+  double h_min, h_max, kappa_min, kappa_max;
+  mesh->GetCharacteristics(h_min, h_max, kappa_min, kappa_max);
+
+  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
+  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
+  Array<int> ess_tdof_list;
+  if (mesh->bdr_attributes.Size())
+  {
+     Array<int> ess_bdr(mesh->bdr_attributes.Max());
+     ess_bdr = 1;
+     Vh->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+  }
+  
+  int dimD = Vh->GetTrueVSize();
+  Vector x0(dimD); x0 = 0.0;
+  Vector xf(dimD); xf = 0.0;
+  
+  ObstacleProblem problem(Vh, x0, &fRhs, &spherical_obstacle, ess_tdof_list);
+  InteriorPointSolver optimizer(&problem); 
+  
+  optimizer.SetTol(1.e-7);
+  optimizer.SetLinearSolver(linSolver);
+  optimizer.SetMaxIter(maxIPMiters);
+  optimizer.Mult(x0, xf);
+  
+  
+  double Einitial = problem.E(x0);
+  double Efinal = problem.E(xf);
+  cout << "Energy objective at initial point = " << Einitial << endl;
+  cout << "Energy objective at optimizer = " << Efinal << endl;
+
+  GridFunction d_gf(Vh);
+  d_gf = xf;
+
+  FunctionCoefficient dtrue_fc(exact_solution_obstacle); // exact solution
+  GridFunction dtrue_gf(Vh);
+  dtrue_gf.ProjectCoefficient(dtrue_fc);
+  
+  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
+  paraview_dc.SetPrefixPath("ParaView");
+  paraview_dc.SetLevelsOfDetail(FEorder);
+  paraview_dc.SetDataFormat(VTKFormat::BINARY);
+  paraview_dc.SetHighOrderOutput(true);
+  paraview_dc.SetCycle(0);
+  paraview_dc.SetTime(0.0);
+  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
+  paraview_dc.RegisterField("d(x) (true)", &dtrue_gf);
+  paraview_dc.Save();
+  
+  FunctionCoefficient exact_coef(exact_solution_obstacle);
+  double L2_error = d_gf.ComputeL2Error(exact_coef);
+  cout << "||u - u_true||_L^2(Omega) = " << L2_error << ", hmax = " << h_max << ", hmin = " << h_min << endl;
+  
+  delete Vh;
+  delete fec;
+  delete mesh;
+  return 0;
+}
+
+
+double fRhs(const Vector &x)
+{
+  return 0.;
+}
+
+
+double spherical_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double beta = 0.9;
+
+   double b = r0*beta;
+   double tmp = sqrt(r0*r0 - b*b);
+   double B = tmp + b*b/tmp;
+   double C = -b/tmp;
+
+   if (r > b)
+   {
+      return B + r * C;
+   }
+   else
+   {
+      return sqrt(r0*r0 - r*r);
+   }
+}
+
+double exact_solution_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      return A * log(r);
+   }
+   else
+   {
+      return sqrt(r0*r0-r*r);
+   }
+} 
+

examples/contact/TwoSidedDirichletObstacleProblem.cpp

@@ -0,0 +1,143 @@
+#include "mfem.hpp"
+#include "Problems.hpp"
+#include "IPsolver.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+
+double fRhs(const Vector &pt);
+double obstaclel(const Vector &pt);
+double obstacleu(const Vector &pt);
+double dmanufacturedFun(const Vector &pt);
+
+int main(int argc, char *argv[])
+{
+  int FEorder = 1; // order of the finite elements
+  int linSolver = 0;
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  int ref_levels = 1; 
+  OptionsParser args(argc, argv);
+  args.AddOption(&FEorder, "-o", "--order",\
+		  "Order of the finite elements.");
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+  
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+
+  const char *meshFile = "../../data/inline-quad.mesh";
+  Mesh *mesh = new Mesh(meshFile, 1, 1);
+  int dim = mesh->Dimension(); // geometric dimension of the domain
+  for (int l = 0; l < ref_levels; l++)
+  {
+      mesh->UniformRefinement();
+  }
+
+  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
+  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
+  Array<int> ess_tdof_list;
+  if (mesh->bdr_attributes.Size())
+  {
+     Array<int> ess_bdr(mesh->bdr_attributes.Max());
+     ess_bdr = 1;
+     Vh->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+  }
+  
+  double DC_val = 0.0;
+  int dimD = Vh->GetTrueVSize();
+
+  Vector x0(dimD); x0 = DC_val;
+  Vector xf(dimD); xf = 0.0;
+
+  ObstacleProblem problem(Vh, x0, &fRhs, &obstaclel, &obstacleu, ess_tdof_list);
+
+  InteriorPointSolver optimizer(&problem); 
+  optimizer.SetTol(1.e-7);
+  optimizer.SetLinearSolver(linSolver);
+  optimizer.SetMaxIter(maxIPMiters);
+  optimizer.Mult(x0, xf);
+  
+
+  GridFunction d_gf(Vh);
+
+  d_gf = xf;
+  
+  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
+  GridFunction dm_gf(Vh);
+  dm_gf.ProjectCoefficient(dm_fc);
+
+  
+  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
+  paraview_dc.SetPrefixPath("ParaView");
+  paraview_dc.SetLevelsOfDetail(FEorder);
+  paraview_dc.SetDataFormat(VTKFormat::BINARY);
+  paraview_dc.SetHighOrderOutput(true);
+  paraview_dc.SetCycle(0);
+  paraview_dc.SetTime(0.0);
+  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
+  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
+  paraview_dc.Save();
+  
+
+  delete Vh;
+  delete fec;
+  delete mesh;
+  return 0;
+}
+
+
+
+double dmanufacturedFun(const Vector &pt)
+{
+  double alpha = 16.5;
+  return sin(M_PI * pt(1)) * (sin(M_PI * pt(0)) - alpha * pow(pt(0) * (1. - pt(0)), 2));
+}
+
+
+// f(x) forcing term... which enters the objective energy functional
+// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
+// of f(x). f(x) is such that in the absence of bound-constraints then
+// the solution of the optimization problem satisfies the PDE
+// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
+// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
+
+double fRhs(const Vector &pt)
+{
+  double alpha = 16.5;
+  double fx;
+  fx = pow(M_PI, 2) * sin(M_PI * pt(0));
+  fx += alpha * (2. * pow(pt(0), 2) + 2. * pow(1.-pt(0), 2) - 8. * pt(0) * (1.-pt(0)));
+  fx += pow(M_PI, 2) * sin(M_PI * pt(0)) * dmanufacturedFun(pt);
+  fx *= sin(M_PI * pt(1));
+  return fx;
+}
+
+double obstaclel(const Vector &pt)
+{
+  return 0.0;
+}
+
+double obstacleu(const Vector &pt)
+{
+  return 0.08;
+}

examples/contact/block1.mesh

@@ -0,0 +1,103 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+9
+1   5     0      1     3     2     8    9     11    10
+1   5     2      3     5     4    10    11    13    12
+1   5     4      5     7     6    12    13    15    14
+1   5     8      9    11    10    16    17    19    18
+1   5     10    11    13    12    18    19    21    20
+1   5     12    13    15    14    20    21    23    22
+1   5     16    17    19    18    24    25    27    26
+1   5     18    19    21    20    26    27    29    28
+1   5     20    21    23    22    28    29    31    30
+
+
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+30
+1   3    1     0     2     3
+1   3    3     2     4     5
+1   3    5     4     6     7
+1   3    24    25    27    26
+1   3    26    27    29    28
+1   3    28    29    31    30
+2   3    2     0     8    10
+2   3    4     2    10    12
+2   3    6     4    12    14
+2  3    10    8    16    18
+2  3    12    10    18    20
+2  3    14    12    20    22
+2  3    18    16    24    26
+2  3    20    18    26    28
+2  3    22    20    28    30
+3  3    1     3    11    9
+3  3     3     5    13    11
+3  3     5     7    15    13
+3  3    9    11    19    17
+3  3    11    13    21    19
+3  3    13    15    23    21
+3  3    17    19    27    25
+3  3    19    21    29    27
+3  3    21    23    31    29
+1  3     8     0     1    9
+1  3    16     8    9    17
+1  3    24    16    17    25
+1  3     6    14    15     7
+1  3    14    22    23    15
+1  3    22    30    31    23
+
+
+vertices
+32
+3
+   -1.0000         0         0
+         0         0         0
+   -1.0000    0.3000         0
+         0    0.3000         0
+   -1.0000    0.6500         0
+         0    0.6500         0
+   -1.0000    1.0000         0
+         0    1.0000         0
+   -1.0000         0    0.3000
+         0         0    0.3000
+   -1.0000    0.3000    0.3500
+         0    0.3000    0.3500
+   -1.0000    0.6500    0.3000
+         0    0.6500    0.3000
+   -1.0000    1.0000    0.3000
+         0    1.0000    0.3000
+   -1.0000         0    0.6500
+         0         0    0.6500
+   -1.0000    0.3000    0.6500
+         0    0.3000    0.6500
+   -1.0000    0.6500    0.6500
+         0    0.6500    0.6500
+   -1.0000    1.0000    0.6500
+         0    1.0000    0.6500
+   -1.0000         0    1.0000
+         0         0    1.0000
+   -1.0000    0.3000    1.0000
+         0    0.3000    1.0000
+   -1.0000    0.6500    1.0000
+         0    0.6500    1.0000
+   -1.0000    1.0000    1.0000
+         0    1.0000    1.0000

examples/contact/exQPContactBlockTL.cpp

@@ -0,0 +1,246 @@
+//                                Quadratic-Programming (QP) Contact example
+//
+// Compile with: make exQPContactBlockTL
+//
+// Sample runs:  ./exQPContactBlockTL
+
+
+#include <fstream>
+#include <iostream>
+#include <array>
+
+#include "mfem.hpp"
+#include "Problems.hpp"
+#include "IPsolver.hpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+  int linSolver = 0;
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  int ref_levels = 0;
+
+  OptionsParser args(argc, argv);
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+
+   
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+	
+  Mesh * mesh1 = new Mesh("block1.mesh", 1, 1);
+  Mesh * mesh2 = new Mesh("rotatedblock2.mesh", 1, 1);
+  for(int i = 0; i < ref_levels; i++)
+  {
+     mesh1->UniformRefinement();
+     mesh2->UniformRefinement(); 
+  }
+
+  // Create an instance of the nlp
+  ExContactBlockTL * contact = new ExContactBlockTL(mesh1, mesh2, 1);
+  int ndofs = contact->GetDimD();
+  int nconstraints = contact->GetDimS();
+  
+  // set up a QP-problem
+  // E(d) = 1 / 2 d^T K d + f^T d
+  // g(d) = J d + g0
+  // where K, J, f and g0 are evaluated at d0 (a valid configuration)
+  
+  // to do: seems more appropriate to evaluate at a valid configuration...
+  //        that is one where the Dirichlet conditions hold... need to pull
+  //        this data from contactBlockTL...
+  Vector d0(ndofs); d0 = 0.0;
+  Array<int> ess_tdofs1 = contact->GetMesh1DirichletDofs();
+  Array<int> ess_tdofs2 = contact->GetMesh2DirichletDofs();
+  int sz1 = ess_tdofs1.Size();
+  int sz2 = ess_tdofs2.Size();
+  Array<int> DirichletDofs(sz1+sz2);
+  for (int i = 0; i<sz1; i++)
+  {
+    DirichletDofs[i] = ess_tdofs1[i];
+  }
+  for (int i = 0; i<sz2; i++)
+  {
+    DirichletDofs[i+sz1] = ess_tdofs2[i]+contact->GetVh1().GetTrueVSize();
+  }
+  GridFunction x1 = contact->GetMesh1GridFunction();
+  GridFunction x2 = contact->GetMesh2GridFunction();
+
+  SparseMatrix *K;
+  Vector f(ndofs); f = 0.0;
+  contact->DdE(d0, f); K = contact->DddE(d0);
+  d0.SetVector(x1,0);
+  d0.SetVector(x2,x1.Size());
+  SparseMatrix *J;
+  Vector g0(nconstraints); g0 = 0.0;
+  contact->g(d0, g0); J = contact->Ddg(d0);
+  Vector temp(nconstraints);
+  J->Mult(d0, temp);
+  g0.Add(-1.0, temp);
+
+  // check which rows of the Jacobian are zero!
+  Vector ei(nconstraints); ei = 0.0;
+  Vector JTei(ndofs); JTei = 0.0;
+
+  double normJTei;
+
+  Array<int> nonZeroRows;
+  for(int i = 0; i < nconstraints; i++)
+  {
+    Array<int> col_tmp;
+    Vector v_tmp; v_tmp = 0.0;
+    J->GetRow(i, col_tmp, v_tmp);
+    normJTei = v_tmp.Norml2();
+    if (normJTei > 1.e-12)
+    {
+      nonZeroRows.Append(i);
+    }
+  }
+  mfem::out << J->Height() << " linearized constraints\n";
+  mfem::out << nonZeroRows.Size() << " (reduced) linearized constraints\n"; 
+  
+  // remove zero rows of the gap function Jacobian and corresponding gap function entries
+  SparseMatrix * Jreduced = new SparseMatrix(nonZeroRows.Size(), ndofs);
+  Vector g0reduced(nonZeroRows.Size()); g0reduced = 0.0; 
+  
+  
+  for(int i = 0; i < nonZeroRows.Size(); i++)
+  {
+    Array<int> col_tmp;
+    Vector v_tmp; v_tmp = 0.0;
+    J->GetRow(nonZeroRows[i], col_tmp, v_tmp);
+    
+    /* obtain subset of columns of the given nonZero Jacobian row that are not Dirichlet constrained */
+    bool freeDof;
+    Array<int> free_col_indicies;
+    for(int j = 0; j < col_tmp.Size(); j++)
+    {
+      freeDof = true;
+      for(int k = 0; k < DirichletDofs.Size(); k++)
+      {
+        if(col_tmp[j] == DirichletDofs[k])
+        {
+          freeDof = false;
+        }
+      }
+      if(freeDof)
+      {
+        free_col_indicies.Append(j);
+      }
+    }
+
+    Array<int> col_tmp_reduced(free_col_indicies.Size());
+    Vector v_tmp_reduced(free_col_indicies.Size());
+    for(int j = 0; j < free_col_indicies.Size(); j++)
+    {
+      col_tmp_reduced[j] = col_tmp[free_col_indicies[j]];
+      v_tmp_reduced(j)   = v_tmp(free_col_indicies[j]);
+    }
+
+    Jreduced->SetRow(i, col_tmp_reduced, v_tmp_reduced);
+    g0reduced(i) = g0(nonZeroRows[i]);
+  }
+
+  QPOptProblem *QPContact = new QPOptProblem(*K, *Jreduced, f, g0reduced);
+  
+  InteriorPointSolver * QPContactOptimizer = new InteriorPointSolver(QPContact);
+  QPContactOptimizer->SetTol(1.e-6);
+  QPContactOptimizer->SetLinearSolver(linSolver);
+  Vector x0(ndofs); x0 = 0.0;
+  x0.SetVector(x1,0);
+  x0.SetVector(x2,x1.Size());
+
+  Vector xf(ndofs); xf = 0.0;
+  QPContactOptimizer->Mult(x0, xf);
+  
+  MFEM_VERIFY(QPContactOptimizer->GetConverged(), "Interior point solver did not converge.");
+  double Einitial = QPContact->E(x0);
+  double Efinal = QPContact->E(xf);
+  cout << "Energy objective at initial point = " << Einitial << endl;
+  cout << "Energy objective at QP optimizer = " << Efinal << endl;
+ 
+  
+  
+  int gdim = mesh1->Dimension();
+  FiniteElementCollection * fec = new H1_FECollection(1, gdim);
+  FiniteElementSpace * fespace1 = new FiniteElementSpace(mesh1, fec, gdim, Ordering::byVDIM);
+  FiniteElementSpace * fespace2 = new FiniteElementSpace(mesh2, fec, gdim, Ordering::byVDIM);
+   
+  GridFunction x1_gf(fespace1);
+  GridFunction x2_gf(fespace2);
+
+  int ndof1 = fespace1->GetTrueVSize();
+  int ndof2 = fespace2->GetTrueVSize();
+  int ndof  = ndof1 + ndof2;
+  for(int i = 0; i < ndof1; i++)
+  {
+    x1_gf(i) = xf(i);
+  }
+  for(int i = ndof1; i < ndof; i++)
+  {
+    x2_gf(i - ndof1) = xf(i);
+  }
+
+  mesh1->SetNodalFESpace(fespace1);
+  mesh2->SetNodalFESpace(fespace2);
+  GridFunction *nodes1 = mesh1->GetNodes();
+  GridFunction *nodes2 = mesh2->GetNodes();
+
+  {
+    *nodes1 += x1_gf;
+    *nodes2 += x2_gf;
+  }
+  
+
+  ParaViewDataCollection paraview_dc1("QPContactBody1", mesh1);
+  paraview_dc1.SetPrefixPath("ParaView");
+  paraview_dc1.SetLevelsOfDetail(1);
+  paraview_dc1.SetDataFormat(VTKFormat::BINARY);
+  paraview_dc1.SetHighOrderOutput(true);
+  paraview_dc1.SetCycle(0);
+  paraview_dc1.SetTime(0.0);
+  paraview_dc1.RegisterField("Body1", &x1_gf);
+  paraview_dc1.Save();
+  
+  ParaViewDataCollection paraview_dc2("QPContactBody2", mesh2);
+  paraview_dc2.SetPrefixPath("ParaView");
+  paraview_dc2.SetLevelsOfDetail(1);
+  paraview_dc2.SetDataFormat(VTKFormat::BINARY);
+  paraview_dc2.SetHighOrderOutput(true);
+  paraview_dc2.SetCycle(0);
+  paraview_dc2.SetTime(0.0);
+  paraview_dc2.RegisterField("Body2", &x2_gf);
+  paraview_dc2.Save();
+
+  delete fespace1;
+  delete fespace2;
+  delete fec;
+  delete mesh1;
+  delete mesh2;
+  
+  delete QPContact;
+  delete QPContactOptimizer;
+  
+  delete Jreduced;
+  delete contact;
+  return 0;
+}

examples/contact/makefile

@@ -0,0 +1,125 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../../
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/contact/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = ObstacleProblem SphericalObstacleProblem DirichletObstacleProblem exQPContactBlockTL
+PAR_EXAMPLES = ParObstacleProblem
+EXAMPLES = $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+
+ifeq ($(MFEM_USE_SUITESPARSE),NO)
+$(SEQ_EXAMPLES):
+	$(error MFEM is not configured with SUITESPARSE)
+endif
+
+ifeq ($(MFEM_USE_MUMPS),NO)
+ifeq ($(MFEM_USE_MKL_CPARDISO), NO)
+$(PAR_EXAMPLES):
+	$(error MFEM is not configured with MUMPS or CPARDISO)
+endif
+endif
+
+all: $(EXAMPLES)
+
+ObstacleProblem: ObstacleProblem.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) ObstacleProblem.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+
+SphericalObstacleProblem: SphericalObstacleProblem.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) SphericalObstacleProblem.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+
+DirichletObstacleProblem: DirichletObstacleProblem.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) DirichletObstacleProblem.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+
+DirichletObstacleProblemVariant: DirichletObstacleProblemVariant.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) DirichletObstacleProblemVariant.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+
+TwoSidedDirichletObstacleProblem: TwoSidedDirichletObstacleProblem.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) TwoSidedDirichletObstacleProblem.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+
+
+exQPContactBlockTL: exQPContactBlockTL.o Problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) exQPContactBlockTL.o Problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+
+ParTest: ParTest.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) ParTest.o -o $@ $(MFEM_LIBS)
+
+
+ObstacleProblem.o: $(SRC)ObstacleProblem.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
+
+SphericalObstacleProblem.o: $(SRC)SphericalObstacleProblem.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
+
+DirichletObstacleProblem.o: $(SRC)DirichletObstacleProblem.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+DirichletObstacleProblemVariant.o: $(SRC)DirichletObstacleProblemVariant.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+TwoSidedDirichletObstacleProblem.o: $(SRC)TwoSidedDirichletObstacleProblem.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+exQPContactBlockTL.o: $(SRC)exQPContactBlockTL.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+Problems.o: $(SRC)Problems.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
+
+IPsolver.o: $(SRC)IPsolver.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+ParObstacleProblem: ParObstacleProblem.o ParProblems.o ParIPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) ParObstacleProblem.o ParProblems.o ParIPsolver.o -o $@ $(MFEM_LIBS)
+
+ParSphericalObstacleProblem: ParSphericalObstacleProblem.o ParProblems.o ParIPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) ParSphericalObstacleProblem.o ParProblems.o ParIPsolver.o -o $@ $(MFEM_LIBS)
+
+ParObstacleProblem.o: $(SRC)ParObstacleProblem.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
+
+ParSphericalObstacleProblem.o: $(SRC)ParSphericalObstacleProblem.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
+
+ParProblems.o: $(SRC)ParProblems.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
+
+ParIPsolver.o: $(SRC)ParIPsolver.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+ParTest.o: $(SRC)ParTest.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+clean: clean-build 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+# For out-of-source builds, link the data files from the source tree:
+ifneq ($(SRC),)
+DATA_FILES = block1.mesh rotatedblock2.mesh
+$(DATA_FILES): %: $(SRC)%
+	ln -sf $(<) .
+copy-data: | $(DATA_FILES)
+# For out-of-source builds, the test and sample runs for 'field-interp' need
+# data from the meshing miniapps directory:
+exQPContactBlockTL: | mesh-data
+.PHONY: mesh-data
+mesh-data:
+	$(MAKE) -C ./ copy-data
+endif

examples/contact/nodepair.hpp

@@ -0,0 +1,896 @@
+
+using namespace std;
+using namespace mfem;
+
+void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]);
+   dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);
+   dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]);
+   dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);
+   dNdxi(1,3) = 0.25*(1-xi[0]);
+}
+
+
+void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
+                     DenseMatrix& dN2dxi)
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(2,4); dNdxi = 0.0;
+   dN2dxi.SetSize(3,4);
+   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
+
+   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
+   dN2dxi(1,3) = -0.25;
+}
+
+// returns the vector and matrix form of the shape functions and its derivative
+void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
+                       DenseMatrix& ddNdxi)
+{
+   N.SetSize(3,12); N = 0.0;
+   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
+   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
+   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
+   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
+   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
+   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
+   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
+
+   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
+   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
+   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
+   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
+   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
+   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
+
+   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
+   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
+   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
+   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
+   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
+   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
+   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
+
+   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
+   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
+   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
+   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
+   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
+   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
+}
+
+
+void cross(const Vector a, const Vector b, Vector& c)
+{
+   assert(a.Size()==3);
+   c.SetSize(3);
+   c[0] = a[1]*b[2] - a[2]*b[1];
+   c[1] = -a[0]*b[2] + b[0]*a[2];
+   c[2] = a[0]*b[1] - a[1]*b[0];
+
+}
+// a outer b
+void outer(const Vector a, const Vector b, DenseMatrix& c)
+{
+   int m = a.Size();
+   int n = b.Size();
+   assert(c.Height()==m);
+   assert(c.Width() ==n);
+   for (int i=0; i<m; i++)
+   {
+      for (int j=0; j<n; j++)
+      {
+         c(i,j) = a[i]*b[j];
+      }
+   }
+}
+// dphidxi 2*4
+// coords 4*3
+void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
+                   Vector& normal, double& nnorm)
+{
+
+   DenseMatrix dxdxi(2,3);
+   Mult(dphidxi, coords, dxdxi);
+   Vector dxdxi1(3);
+   Vector dxdxi2(3);
+
+   dxdxi.GetRow(0,dxdxi1);
+   dxdxi.GetRow(1,dxdxi2);
+
+   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
+   // VectorCrossProductCoefficient::Eval has hard-coded cross product
+   nnorm = normal.Norml2( );
+   normal /=  nnorm;
+}
+
+void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
+{
+   bool converged = false;
+   bool pt_on_elem = false;
+   int dim = 3;
+   xi.SetSize(dim-1);
+   xi = 0.0;
+   int max_iter = 15;
+   double off_el_xi = 1e-2;
+   double proj_newton_tol = 1e-13;
+   double proj_max_gap = 0.5;
+   Vector gap_v(dim);
+   // warm start from linear solution
+
+   for (int it=0; it<max_iter; it++)
+   {
+      //cout<<it<<endl;
+      Vector m_N(4);
+      m_N = 0.;
+      DenseMatrix m_dN(2,4);
+      m_dN = 0.;
+      DenseMatrix m_dN2(3,4);
+      m_dN2 = 0.;
+      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+      Vector x_c(dim);
+      m_coords.MultTranspose(m_N, x_c);
+
+      gap_v = s_x;
+      gap_v -= x_c;
+
+      DenseMatrix m_dx(2,3);
+      m_dx = 0.;
+      Mult(m_dN, m_coords, m_dx);
+
+      Vector r(dim-1);
+      r = 0.0;
+      m_dx.Mult(gap_v, r);
+
+      if (r.Normlinf() < proj_newton_tol)
+      {
+         converged = true;
+         break;
+      }
+
+      DenseMatrix drdxi(dim-1,dim-1);
+      drdxi = 0.;
+      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
+      drdxi *= -1.0;
+
+      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+      Mult(m_dN2,m_coords, m_dx2);
+
+      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
+      for (int d=0; d<3; d++)
+      {
+         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+         drdxi.Add(gap_v[d], Mtemp);
+      }
+
+      //cond_num = rcond(drdxi);  condition number?
+      //drdxi.TestInversion();
+      DenseMatrixInverse drdxi_inv(drdxi);
+      Vector xi_tmp(dim-1);
+
+      drdxi_inv.Mult(r,xi_tmp);
+      xi -= xi_tmp;
+   }
+   if (!converged)
+   {
+      xi = 0.0;
+   }
+   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
+
+   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
+   //
+   // Discuss with Frank... what is happening here
+   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
+   {
+      pt_on_elem = true;
+   }
+
+   if (pt_on_elem)
+   {
+     //cout << "convergence of node to segment projection? " << converged << endl;
+     //for(int i = 0; i < 2; i++)
+     //{
+     //  cout << "xi_" << i << " = " << xi(i) << endl;
+     //}
+   }
+   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
+   MFEM_VERIFY(converged == true, "projection didn't converge");
+}
+
+
+
+// m_coords is expected to be 4 * 3
+void  ComputeGapJacobian(const Vector x_s, const Vector xi,
+                         const DenseMatrix m_coords,
+                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N, x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   double nnorm = 0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   gap = gap_v * normal; // gap function value, dot product between vectors
+
+   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
+
+   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
+   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
+
+   Vector m_dxrow1(3);
+   m_dx.GetRow(0, m_dxrow1);
+   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
+   dr_dx_res1 *= -1.0;
+
+   Vector m_dxrow2(3);
+   m_dx.GetRow(1, m_dxrow2);
+   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
+   dr_dx_res2 *= -1.0;
+
+   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
+   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
+
+   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
+   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
+   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
+   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
+
+   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
+   dr_dx_res2 += dr_dx_res2_tmp;
+
+
+   DenseMatrix K_dxidx1(2,2); // 2*2
+   K_dxidx1 = 0.;
+   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   // how to get 2nd order? multidimensional matrix?
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= K_dxidx1;
+   K_dxidx += K_dxidx2;
+
+   // resize the vectors and matrices
+   Vector dxidx(24); dxidx = 0.0;
+   Vector drdx_r(24); drdx_r = 0.0;
+
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         drdx_r[4*j+i] = dr_dx_res1(i,j);
+         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
+
+      }
+   }
+   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
+   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
+   DenseMatrix drdx_K(24,24); drdx_K = 0.;
+   for (int i =0; i<12; i++)
+   {
+      drdx_K(i,i) = K_dxidx(0,0);
+      drdx_K(i,12+i) = K_dxidx(0,1);
+      drdx_K(12+i,i) = K_dxidx(1,0);
+      drdx_K(12+i,12+i) = K_dxidx(1,1);
+   }
+
+   DenseMatrixInverse drdxK_inv(drdx_K);
+   drdxK_inv.Mult(drdx_r,dxidx);
+   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
+   dxidx *= -1.0;
+
+
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+
+   Vector dxidxs(6); dxidxs = 0.0;
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
+
+   dgdxm.SetSize(12); dgdxm = 0.;
+   DenseMatrix dgdxm_tmp(4,3);
+   outer(m_N, normal,dgdxm_tmp);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
+      }
+   }
+   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
+
+   dgdxs.SetSize(3);
+   dgdxs += normal;
+   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
+};
+
+void ComputeGapHessian(const Vector x_s, const Vector xi,
+                       const DenseMatrix m_coords,
+                       DenseMatrix& dg2dx)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   int dim = 3;
+   int num_dofs1 = dim;
+   int num_dofs2 = 4*dim;
+   int num_dofs = num_dofs1 + num_dofs2;
+   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N,x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+   Mult(m_dN2,m_coords, m_dx2);
+   double nnorm = 0.0;
+   Vector normal(3); normal = 0.0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   double gap = gap_v * normal; // gap function value, dot product between vectors
+
+   DenseMatrix M(2,2); M = 0.0;
+   MultABt(m_dx, m_dx, M);
+
+   DenseMatrix f(2, num_dofs2); f = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+      M.Add(-gap_v[d], Mtemp);
+
+      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
+      DenseMatrix ftmp(2,4);
+      outer(m_dxcol, m_N, ftmp);
+      ftmp *= -1;
+      ftmp.Add( gap_v[d], m_dN);   // 2*4
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         f(0,d+j*3) = ftmp(0,j);
+         f(1,d+j*3) = ftmp(1,j);
+      }
+   }
+   //fprintf('hess dxidxm\n');
+   DenseMatrixInverse Minv(M);
+   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
+   Minv.Mult(f, dxidxm);
+   //LinearSolve??
+   //dxidxm = M\f;
+
+   DenseMatrix nde2(2,2); nde2 = 0.0;
+   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
+      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
+      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
+
+      nde2 += ndetmp;
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
+         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
+      }
+
+   }
+
+   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
+   Ndn += Nndx2;
+   AddMult(nde2, dxidxm, Ndn);
+
+
+   DenseMatrix M2(2,2); M2 = 0.0;
+   MultABt(m_dx, m_dx, M2);
+   DenseMatrixInverse M2inv(M2);
+   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
+   DenseMatrix m_con(2,2); m_con = 0.0;
+
+   M2inv.Mult(diag2, m_con);
+
+   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
+
+   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
+   MultAtB(Ndn, m_con, dg2dxm_tmp);
+   Mult(dg2dxm_tmp, Ndn, dg2dxm);
+   dg2dxm *= gap;
+
+   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
+   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   dg2dxm_tmp = 0.0;
+   MultAtB(dxidxm, nde2, dg2dxm_tmp);
+
+   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
+
+   dg2dxm_tmp2 = 0.0;
+   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= M2;
+   K_dxidx += K_dxidx2;
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+   Vector dxidxs(6);
+
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
+   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
+   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
+
+   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
+   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
+
+   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
+   dxidxs_row2 = 0.0;
+   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
+   Vector mdx2_row3(3); mdx2_row3 = 0.0;
+   dxidxs_m.GetRow(0,dxidxs_row1);
+   dxidxs_m.GetRow(1,dxidxs_row2);
+   m_dx2.GetRow(0,mdx2_row1);
+   m_dx2.GetRow(1,mdx2_row2);
+   m_dx2.GetRow(2,mdx2_row3);
+
+   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
+   outer(mdx2_row1, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row2, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+
+   outer(mdx2_row2, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row3, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+
+   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
+
+   for (int d=0; d<3; d++)
+   {
+      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
+      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
+      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
+
+      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
+      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
+
+      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
+      m_dx.GetRow(0, m_dxrow);
+      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
+      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
+      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
+
+      dtaodxs_tmp2 += dtaodxs_tmp;
+      dtaodxs.SetCol(d, dtaodxs_tmp2);
+   }
+
+   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
+   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
+   outer(normal, normal, dndxs_tmp);
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
+
+   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
+   MultAtB(m_dx, dxidxs_m, dgvdxs);
+   dgvdxs *= -1;
+   for (int d=0; d<3; d++)
+   {
+      dgvdxs(d,d) += 1.0;
+   }
+   //dxidxs: 2*3
+
+   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
+   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
+   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
+   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
+   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
+   dg2dxs += dg2dxs_tmp2;
+   dg2dxs_tmp2 = 0.0;
+   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
+   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
+
+   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
+   BasisVectorDerivs(xi, Ne, Be, dBe);
+
+   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
+   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
+
+   Vector m_coords_v(12);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         m_coords_v[i*3+j] = m_coords(i,j);
+      }
+   }
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix dBe_tmp(3,12);
+      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
+
+      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
+
+      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
+      dBe_tmp = 0.0;
+      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
+
+   }
+
+   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
+
+   for (int d=0; d<12; d++)
+   {
+      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
+      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
+      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
+      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
+      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
+      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
+
+      cross(tmp1, m_dxrow2, dtaodxm_tmp);
+      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
+      dtaodxm_tmp += dtaodxm_tmp2;
+
+      dtaodxm.SetCol(d, dtaodxm_tmp);
+   }
+
+   DenseMatrix dndxm(3,12); dndxm = 0.0;
+   dndxm += dtaodxm;
+   dndxm *= 1.0/nnorm;
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
+
+   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
+   dgvdxm -= Ne;
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
+      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
+
+   }
+
+   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
+   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
+   MultAtB(dgvdxs, dndxm, dg2dxsxm);
+
+   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
+   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
+
+   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
+   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
+
+   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
+
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
+      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
+      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
+   }
+
+   dg2dxsxm += dgvdxsxmn;
+
+   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
+   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
+   MultAtB(dgvdxm, dndxs, dg2dxmxs);
+   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
+   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
+
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
+
+   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
+   dgvdxmxsn_tmp *= -1.0;
+   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+      Be_tmp.Transpose(); // Be is now 12*3
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
+      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
+      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
+
+   }
+
+   dg2dxmxs += dgvdxmxsn;
+
+   dg2dx.CopyMN(dg2dxs, 0, 0);
+   dg2dx.CopyMN(dg2dxm, 3, 3);
+   dg2dx.CopyMN(dg2dxsxm, 0, 3);
+   dg2dx.CopyMN(dg2dxmxs, 3, 0);
+
+};
+
+
+
+void NodeSegConPairs(const Vector x1, const Vector xi2,
+                     const DenseMatrix coords2,
+                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
+{
+   double gap = 0.0;
+   Vector normal(3); normal = 0.0;
+   Vector dgdxm(12); dgdxm = 0.0;
+   Vector dgdxs(3); dgdxs = 0.0;
+
+   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
+   node_g = gap;
+
+   node_dg.SetSize(12+3);
+   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
+   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
+
+   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   ComputeGapHessian(x1, xi2, coords2, dg2dx);
+
+   node_dg2.SetSize(15,15);
+   node_dg2 = dg2dx;
+
+   /*
+   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
+
+   v1 = 1:3;
+   v2 = 1:12;
+   %v1 = ones(1,3)
+   %v2 = ones(1,12)
+   v2 = reshape(v2,4,3);
+   x1n1 = x1 + 0.01*v1;
+   coords2n1 = coords2 + 0.001*v2;
+      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
+      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
+   x1n2 = x1 - 0.01*v1;
+   coords2n2 = coords2 - 0.001*v2;
+      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
+      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
+   fprintf('fd\n');
+      %gapv1-gapv2
+   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
+
+   %dgdxsn1-dgdxsn2
+   fprintf('code\n');
+   v2n = v2';
+   %dg2dx(1:3,1:3)*0.04*ones(3,1)
+   temp = zeros(12,3);
+   for i = 1:4
+   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
+   temp((i-1)*3+1:i*3,:) = temp1';
+   end
+   temp2 = zeros(3,12);
+   for i = 1:4
+   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
+   temp2(:,(i-1)*3+1:i*3) = temp3';
+   end
+   %dg2dx
+   %dg2dx(4:end,1:3) = temp;
+   %dg2dx(1:3,4:end) = temp2;
+   %dgvdxm * 0.002*v2n(:)
+   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
+   %dg2dx(4:end,1:3)
+   end*/
+
+};
+
+
+// coordsm : (npoints*4, 3) use what class?
+// m_conn: (npoints*4)
+void Assemble_Contact(const int m, const int npoints, const int ndofs,
+                      const Vector x_s,
+                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
+                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
+                      std::vector<SparseMatrix>& dM)
+{
+   int ndim = 3;
+
+   g.SetSize(m);
+   g = 0.0;
+
+   //SparseMatrix M(m, n);  // M needs to be the correct size
+
+   //dM.resize(m); // needs to clear?
+
+   double g_tmp = 0.;
+   Vector dg(4*ndim+ndim);
+   dg = 0.;
+   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
+   dg2 = 0.;
+
+   for (int i=0; i<npoints; i++)
+   {
+      Vector x1(ndim);
+      x1[0] = x_s[i*ndim];
+      x1[1] = x_s[i*ndim+1];
+      x1[2] = x_s[i*ndim+2];
+
+      Vector xi2(ndim-1);
+      xi2[0] = xi[i*(ndim-1)];
+      xi2[1] = xi[i*(ndim-1)+1];
+
+      DenseMatrix coords2(4,3);
+      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
+
+      //how to get coords2?
+      dg = 0.0;
+      dg2 = 0.;
+      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
+      //x1.Print();
+      //xi2.Print();
+      //coords2.Print();
+      g[s_conn[i]] = g_tmp; // should be unique
+      Array<int> m_conn_i(4);
+      m_conn.GetSubArray(4*i, 4, m_conn_i);
+
+      Array<int> node_conn(5);
+      node_conn[0] = s_conn[i];
+      for (int j=0; j<4; j++)
+      {
+         node_conn[j+1] = m_conn_i[j];
+      }
+
+      Array<int>  M_i_tmp(1);
+      M_i_tmp[0] = s_conn[i];
+
+      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
+      Array<int> j_idx(5*ndim); j_idx = 0;
+      for (int j=0; j< 5; j++)
+      {
+         for (int k=0; k<ndim; k++)
+         {
+            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
+         }
+      }
+      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
+      M_v_tmp.SetRow(0, dg);
+
+      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
+
+      Array<int> dM_i(ndim*(4+1));
+      Array<int> dM_j(ndim*(4+1));
+
+      for (int j=0; j< ndim*(4+1); j++)
+      {
+         dM_i[j] = j_idx[j];
+         dM_j[j] = j_idx[j];
+      }
+      //dg2.Print();
+      //dM[s_conn[i]].Print();
+      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
+   }
+};
+

examples/contact/obstacleProblem.cpp

@@ -0,0 +1,119 @@
+#include "mfem.hpp"
+#include "Problems.hpp"
+#include "IPsolver.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+
+double dmanufacturedFun(const Vector &);
+double fRhs(const Vector &);
+double obstacle(const Vector &);
+
+int main(int argc, char *argv[])
+{
+  int FEorder = 1; // order of the finite elements
+  int linSolver = 0;
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  
+  OptionsParser args(argc, argv);
+  args.AddOption(&FEorder, "-o", "--order",\
+		  "Order of the finite elements.");
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+
+  const char *meshFile = "../../data/inline-quad.mesh";
+  Mesh *mesh = new Mesh(meshFile, 1, 1);
+  int dim = mesh->Dimension(); // geometric dimension of the domain
+  {
+     int ref_levels = 3;
+     for (int l = 0; l < ref_levels; l++)
+     {
+        mesh->UniformRefinement();
+     }
+  }
+
+  FiniteElementCollection *fec = new H1_FECollection(FEorder, dim);
+  FiniteElementSpace      *Vh  = new FiniteElementSpace(mesh, fec);
+  ObstacleProblem problem(Vh, &fRhs, &obstacle);
+  
+  int dimD = problem.GetDimD();
+  Vector x0(dimD); x0 = 0.0;
+  Vector xf(dimD); xf = 0.0;
+
+  InteriorPointSolver optimizer(&problem); 
+  optimizer.SetTol(1.e-7);
+  optimizer.SetLinearSolver(linSolver);
+  optimizer.SetMaxIter(maxIPMiters);
+  optimizer.Mult(x0, xf);
+
+  GridFunction d_gf(Vh);
+
+  d_gf = xf;
+
+  FunctionCoefficient dm_fc(dmanufacturedFun); // pseudo-manufactured solution
+  GridFunction dm_gf(Vh);
+  dm_gf.ProjectCoefficient(dm_fc);
+  
+  ParaViewDataCollection paraview_dc("BarrierProblemSolution", mesh);
+  paraview_dc.SetPrefixPath("ParaView");
+  paraview_dc.SetLevelsOfDetail(FEorder);
+  paraview_dc.SetDataFormat(VTKFormat::BINARY);
+  paraview_dc.SetHighOrderOutput(true);
+  paraview_dc.SetCycle(0);
+  paraview_dc.SetTime(0.0);
+  paraview_dc.RegisterField("d(x) (numerical)", &d_gf);
+  paraview_dc.RegisterField("d(x) (pseudo-manufactured)", &dm_gf);
+  paraview_dc.Save();
+  
+  
+  delete Vh;
+  delete fec;
+  delete mesh;
+  return 0;
+}
+
+
+double dmanufacturedFun(const Vector &x)
+{
+  return cos(2*M_PI*x(0)) + 0.2 - 2.0*(pow(x(0),3) - 1.5*pow(x(0),2));
+}
+
+
+// f(x) forcing term... which enters the objective energy functional
+// E(d) = 0.5 d^T K d - f^T d, where f is a discrete vector representation
+// of f(x). f(x) is such that in the absence of bound-constraints then
+// the solution of the optimization problem satisfies the PDE
+// -div(grad(d)) + d = f + homogeneous Neumann conditions on the unit interval,
+// for d(x) = cos(2 \pi x) + a0 + a3 (x^3 - 1.5 x^2), a2 = 0.2, a3 = -2
+double fRhs(const Vector &x)
+{
+  double fx = 0.;
+  fx = 0.2 - 2.0 * (pow(x(0),3)- 1.5*pow(x(0),2.) - 6 * x(0) + 3.) + (1. + pow(2.*M_PI,2))*cos(2.*M_PI*x(0));
+  return fx;
+}
+
+double obstacle(const Vector &x)
+{
+  return 0.0;
+}

examples/contact/rotatedblock2.mesh

@@ -0,0 +1,70 @@
+
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+# 1 nothing
+elements
+4
+1   5     0     1     3     2     6     7    9     8
+1   5     2     3     5     4     8     9    11    10
+1   5     6     7     9     8    12    13    15    14
+1   5     8     9    11    10    14    15    17    16
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+16
+1   3     1     0     2     3
+1   3     3     2     4     5
+1   3    12    13    15    14
+1   3    14    15    17    16
+3   3     2     0     6     8
+3   3     4     2     8    10
+3   3     8     6    12    14
+3   3    10     8    14    16
+2   3     1     3     9     7
+2   3     3     5    11    9
+2   3     7    9    15    13
+2   3     9    11    17    15
+1   3     6     0     1     7
+1   3    12     6     7    13
+1   3     4    10    11     5
+1   3    10    16    17    11
+
+vertices
+18
+3
+
+0.000000000000 0.145770950245 0.443895630208
+0.507100000000 0.145770950245 0.443895630208
+0.000000000000 0.350937660019 0.294833290227
+0.507100000000 0.350937660019 0.294833290227
+0.000000000000 0.556104369792 0.145770950245
+0.507100000000 0.556104369792 0.145770950245
+0.000000000000 0.294833290227 0.649062339981
+0.507100000000 0.294833290227 0.649062339981
+0.000000000000 0.500000000000 0.500000000000
+0.507100000000 0.500000000000 0.500000000000
+0.000000000000 0.705166709773 0.350937660019
+0.507100000000 0.705166709773 0.350937660019
+0.000000000000 0.443895630208 0.854229049755
+0.507100000000 0.443895630208 0.854229049755
+0.000000000000 0.649062339981 0.705166709773
+0.507100000000 0.649062339981 0.705166709773
+0.000000000000 0.854229049755 0.556104369792
+0.507100000000 0.854229049755 0.556104369792

examples/ex20p.cpp

@@ -246,17 +246,23 @@ int main(int argc, char *argv[])
    e_var /= (nsteps + 1);
    double e_sd = sqrt(e_var);
 
+   double e_loc_stats[2];
+   double *e_stats = (myid == 0) ? new double[2 * num_procs] : (double*)NULL;
+
+   e_loc_stats[0] = e_mean;
+   e_loc_stats[1] = e_sd;
+   MPI_Gather(e_loc_stats, 2, MPI_DOUBLE, e_stats, 2, MPI_DOUBLE, 0, comm);
+
    if (myid == 0)
    {
-      cout << endl << "Mean and standard deviation of the energy" << endl;
-   }
-   for (int i = 0; i < num_procs; i++)
-   {
-      if (myid == i)
+      cout << endl << "Mean and standard deviation of the energy "
+           << "for different initial conditions" << endl;
+      for (int i = 0; i < num_procs; i++)
       {
-         cout << myid << ": " << e_mean << "\t" << e_sd << endl;
+         cout << i << ": " << e_stats[2 * i + 0]
+              << "\t" << e_stats[2 * i + 1] << endl;
       }
-      MPI_Barrier(comm);
+      delete [] e_stats;
    }
 
    // 9. Finalize the GnuPlot output

examples/makefile

@@ -36,7 +36,7 @@ ifeq ($(MFEM_USE_MPI),NO)
 else
    EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
 endif
-SUBDIRS =
+SUBDIRS = contact 
 ifeq ($(MFEM_USE_AMGX),YES)
    SUBDIRS += amgx
 endif

fem/CMakeLists.txt

@@ -44,6 +44,7 @@ set(SRCS
   eltrans.cpp
   estimators.cpp
   fe.cpp
+  fe/face_map_utils.cpp
   fe/fe_base.cpp
   fe/fe_fixed_order.cpp
   fe/fe_h1.cpp
@@ -74,8 +75,10 @@ set(SRCS
   linearform_ext.cpp
   lininteg.cpp
   lininteg_boundary.cpp
+  lininteg_boundary_flux.cpp
   lininteg_domain.cpp
   lininteg_domain_grad.cpp
+  lininteg_vectorfe_domain.cpp
   lor/lor.cpp
   lor/lor_ads.cpp
   lor/lor_ams.cpp
@@ -151,6 +154,7 @@ set(HDRS
   eltrans.hpp
   estimators.hpp
   fe.hpp
+  fe/face_map_utils.hpp
   fe/fe_base.hpp
   fe/fe_fixed_order.hpp
   fe/fe_h1.hpp

fem/fe/face_map_utils.cpp

@@ -0,0 +1,115 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Finite Element Base classes
+
+#include "face_map_utils.hpp"
+#include <cmath> // std::pow
+
+namespace mfem
+{
+
+namespace internal
+{
+
+std::pair<int,int> GetFaceNormal3D(const int face_id)
+{
+   switch (face_id)
+   {
+      case 0: return std::make_pair(2, 0); // z = 0
+      case 1: return std::make_pair(1, 0); // y = 0
+      case 2: return std::make_pair(0, 1); // x = 1
+      case 3: return std::make_pair(1, 1); // y = 1
+      case 4: return std::make_pair(0, 0); // x = 0
+      case 5: return std::make_pair(2, 1); // z = 1
+      default: MFEM_ABORT("Invalid face ID.")
+   }
+   return std::make_pair(-1, -1); // invalid
+}
+
+void FillFaceMap(const int n_face_dofs_per_component,
+                 const std::vector<int> &offsets,
+                 const std::vector<int> &strides,
+                 const std::vector<int> &n_dofs_per_dim,
+                 Array<int> &face_map)
+{
+   const int n_components = offsets.size();
+   const int face_dim = strides.size() / n_components;
+   for (int comp = 0; comp < n_components; ++comp)
+   {
+      const int offset = offsets[comp];
+      for (int i = 0; i < n_face_dofs_per_component; ++i)
+      {
+         int idx = offset;
+         int j = i;
+         for (int d = 0; d < face_dim; ++d)
+         {
+            const int dof1d = n_dofs_per_dim[comp*(face_dim) + d];
+            idx += strides[comp*(face_dim) + d]*(j % dof1d);
+            j /= dof1d;
+         }
+         face_map[comp*n_face_dofs_per_component + i] = idx;
+      }
+   }
+}
+
+void GetTensorFaceMap(const int dim, const int order, const int face_id,
+                      Array<int> &face_map)
+{
+   const int dof1d = order + 1;
+   int n_face_dofs = int(std::pow(dof1d, dim - 1));
+   std::vector<int> offsets, strides;
+   switch (dim)
+   {
+      case 1:
+         offsets = {(face_id == 0) ? 0 : dof1d - 1};
+         break;
+      case 2:
+         strides = {(face_id == 0 || face_id == 2) ? 1 : dof1d};
+         switch (face_id)
+         {
+            case 0: offsets = {0}; break; // y = 0
+            case 1: offsets = {dof1d - 1}; break; // x = 1
+            case 2: offsets = {(dof1d-1)*dof1d}; break; // y = 1
+            case 3: offsets = {0}; break; // x = 0
+         }
+         break;
+      case 3:
+      {
+         const auto f = GetFaceNormal3D(face_id);
+         const int face_normal = f.first, level = f.second;
+         if (face_normal == 0) // x-normal
+         {
+            offsets = {level ? dof1d-1 : 0};
+            strides = {dof1d, dof1d*dof1d};
+         }
+         else if (face_normal == 1) // y-normal
+         {
+            offsets = {level ? (dof1d-1)*dof1d : 0};
+            strides = {1, dof1d*dof1d};
+         }
+         else if (face_normal == 2) // z-normal
+         {
+            offsets = {level ? (dof1d-1)*dof1d*dof1d : 0};
+            strides = {1, dof1d};
+         }
+         break;
+      }
+   }
+
+   // same number of DOFs in each dimension, repeat dof1d (dim - 1) times
+   std::vector<int> n_dofs(dim - 1, dof1d);
+   FillFaceMap(n_face_dofs, offsets, strides, n_dofs, face_map);
+}
+
+} // namespace internal
+
+} // namespace mfem

fem/fe/face_map_utils.hpp

@@ -0,0 +1,58 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FACE_MAP_UTILS_HPP
+#define MFEM_FACE_MAP_UTILS_HPP
+
+#include "../../general/array.hpp"
+#include <utility> // std::pair
+#include <vector>
+
+namespace mfem
+{
+
+namespace internal
+{
+
+/// Each face of a hexahedron is given by a level set x_i = l, where x_i is one
+/// of x, y, or z (corresponding to i=0, i=1, i=2), and l is either 0 or 1.
+/// Returns i and level.
+std::pair<int,int> GetFaceNormal3D(const int face_id);
+
+/// @brief Fills in the entries of the lexicographic face_map.
+///
+/// For use in FiniteElement::GetFaceMap.
+///
+/// n_face_dofs_per_component is the number of DOFs for each vector component
+/// on the face (there is only one vector component in all cases except for 3D
+/// Nedelec elements, where the face DOFs have two components to span the
+/// tangent space).
+///
+/// The DOFs for the i-th vector component begin at offsets[i] (i.e. the number
+/// of vector components is given by offsets.size()).
+///
+/// The DOFs for each vector component are arranged in a Cartesian grid defined
+/// by strides and n_dofs_per_dim.
+void FillFaceMap(const int n_face_dofs_per_component,
+                 const std::vector<int> &offsets,
+                 const std::vector<int> &strides,
+                 const std::vector<int> &n_dofs_per_dim,
+                 Array<int> &face_map);
+
+/// Return the face map for nodal tensor elements (H1, L2, and Bernstein basis).
+void GetTensorFaceMap(const int dim, const int order, const int face_id,
+                      Array<int> &face_map);
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif

fem/fe/fe_base.cpp

@@ -12,6 +12,7 @@
 // Finite Element Base classes
 
 #include "fe_base.hpp"
+#include "face_map_utils.hpp"
 #include "../coefficient.hpp"
 
 namespace mfem
@@ -485,6 +486,12 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
    return *d2q;
 }
 
+void FiniteElement::GetFaceMap(const int face_id,
+                               Array<int> &face_map) const
+{
+   MFEM_ABORT("method is not implemented for this element");
+}
+
 FiniteElement::~FiniteElement()
 {
    for (int i = 0; i < dof2quad_array.Size(); i++)
@@ -2509,6 +2516,12 @@ void NodalTensorFiniteElement::SetMapType(const int map_type)
    }
 }
 
+void NodalTensorFiniteElement::GetFaceMap(const int face_id,
+                                          Array<int> &face_map) const
+{
+   internal::GetTensorFaceMap(dim, order, face_id, face_map);
+}
+
 VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                                                      const int d,
                                                      const int p,

fem/fe/fe_base.hpp

@@ -576,6 +576,20 @@ public:
    virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                          DofToQuad::Mode mode) const;
 
+   /** @brief Return the mapping from lexicographic face DOFs to lexicographic
+       element DOFs for the given local face @a face_id. */
+   /** Given the @a ith DOF (lexicographically ordered) on the face referenced
+       by @a face_id, face_map[i] gives the corresponding index of the DOF in
+       the element (also lexicographically ordered).
+
+       @note For L2 spaces, this is only well-defined for "closed" bases such as
+       the Gauss-Lobatto or Bernstein (positive) bases.
+
+       @warning GetFaceMap() is currently only implemented for tensor-product
+       (quadrilateral and hexahedral) elements. Its functionality may change
+       when simplex elements are supported in the future. */
+   virtual void GetFaceMap(const int face_id, Array<int> &face_map) const;
+
    /// Deconstruct the FiniteElement
    virtual ~FiniteElement();
 
@@ -1248,6 +1262,8 @@ public:
          NodalFiniteElement::GetTransferMatrix(fe, Trans, I);
       }
    }
+
+   void GetFaceMap(const int face_id, Array<int> &face_map) const override;
 };
 
 class VectorTensorFiniteElement : public VectorFiniteElement,

fem/fe/fe_nd.cpp

@@ -12,6 +12,7 @@
 // Nedelec Finite Element classes
 
 #include "fe_nd.hpp"
+#include "face_map_utils.hpp"
 #include "../coefficient.hpp"
 
 namespace mfem
@@ -481,6 +482,51 @@ void ND_HexahedronElement::CalcCurlShape(const IntegrationPoint &ip,
          }
 }
 
+void ND_HexahedronElement::GetFaceMap(const int face_id,
+                                      Array<int> &face_map) const
+{
+   const int p = order;
+   const int pp1 = p + 1;
+   const int n_face_dofs_per_component = p*pp1;
+   const int n_dof_per_dim = p*pp1*pp1;
+
+   std::vector<int> n_dofs = {p, pp1, pp1, p};
+   std::vector<int> offsets, strides;
+
+   const auto f = internal::GetFaceNormal3D(face_id);
+   const int face_normal = f.first, level = f.second;
+   if (face_normal == 0) // x-normal
+   {
+      offsets =
+      {
+         n_dof_per_dim + (level ? pp1 - 1 : 0),
+         2*n_dof_per_dim + (level ? pp1 - 1 : 0)
+      };
+      strides = {pp1, p*pp1, pp1, pp1*pp1};
+   }
+   else if (face_normal == 1) // y-normal
+   {
+      offsets =
+      {
+         level ? p*(pp1 - 1) : 0,
+         2*n_dof_per_dim + (level ? pp1*(pp1 - 1) : 0)
+      };
+      strides = {1, p*pp1, 1, pp1*pp1};
+   }
+   else if (face_normal == 2) // z-normal
+   {
+      offsets =
+      {
+         level ? p*pp1*(pp1 - 1) : 0,
+         n_dof_per_dim + (level ? p*pp1*(pp1 - 1) : 0)
+      };
+      strides = {1, p, 1, pp1};
+   }
+
+   internal::FillFaceMap(n_face_dofs_per_component, offsets, strides, n_dofs,
+                         face_map);
+}
+
 const double ND_QuadrilateralElement::tk[8] =
 { 1.,0.,  0.,1., -1.,0., 0.,-1. };
 
@@ -771,6 +817,26 @@ void ND_QuadrilateralElement::CalcCurlShape(const IntegrationPoint &ip,
       }
 }
 
+void ND_QuadrilateralElement::GetFaceMap(const int face_id,
+                                         Array<int> &face_map) const
+{
+   const int p = order;
+   const int pp1 = order + 1;
+   const int n_face_dofs_per_component = p;
+   std::vector<int> strides = {(face_id == 0 || face_id == 2) ? 1 : pp1};
+   std::vector<int> n_dofs = {p};
+   std::vector<int> offsets;
+   switch (face_id)
+   {
+      case 0: offsets = {0}; break; // y = 0
+      case 1: offsets = {p*pp1 + pp1 - 1}; break; // x = 1
+      case 2: offsets = {p*(pp1 - 1)}; break; // y = 1
+      case 3: offsets = {p*pp1}; break; // x = 0
+   }
+   internal::FillFaceMap(n_face_dofs_per_component, offsets, strides, n_dofs,
+                         face_map);
+}
+
 
 const double ND_TetrahedronElement::tk[18] =
 { 1.,0.,0.,  0.,1.,0.,  0.,0.,1.,  -1.,1.,0.,  -1.,0.,1.,  0.,-1.,1. };

fem/fe/fe_nd.hpp

@@ -91,6 +91,8 @@ public:
                             DenseMatrix &curl) const
    { ProjectCurl_ND(tk, dof2tk, fe, Trans, curl); }
 
+   virtual void GetFaceMap(const int face_id, Array<int> &face_map) const;
+
 protected:
    void ProjectIntegrated(VectorCoefficient &vc,
                           ElementTransformation &Trans,
@@ -155,6 +157,8 @@ public:
                             DenseMatrix &grad) const
    { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
 
+   virtual void GetFaceMap(const int face_id, Array<int> &face_map) const;
+
 protected:
    void ProjectIntegrated(VectorCoefficient &vc,
                           ElementTransformation &Trans,

fem/fe/fe_pos.cpp

@@ -12,6 +12,7 @@
 // H1 Finite Element classes utilizing the Bernstein basis
 
 #include "fe_pos.hpp"
+#include "face_map_utils.hpp"
 #include "../bilininteg.hpp"
 #include "../lininteg.hpp"
 #include "../coefficient.hpp"
@@ -84,6 +85,12 @@ PositiveTensorFiniteElement::PositiveTensorFiniteElement(
                            dims > 1 ? FunctionSpace::Qk : FunctionSpace::Pk),
      TensorBasisElement(dims, p, BasisType::Positive, dmtype) { }
 
+void PositiveTensorFiniteElement::GetFaceMap(const int face_id,
+                                             Array<int> &face_map) const
+{
+   internal::GetTensorFaceMap(dim, order, face_id, face_map);
+}
+
 
 BiQuadPos2DFiniteElement::BiQuadPos2DFiniteElement()
    : PositiveFiniteElement(2, Geometry::SQUARE, 9, 2, FunctionSpace::Qk)

fem/fe/fe_pos.hpp

@@ -70,12 +70,14 @@ public:
                                const DofMapType dmtype);
 
    const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const
+                                 DofToQuad::Mode mode) const override
    {
       return (mode == DofToQuad::FULL) ?
              FiniteElement::GetDofToQuad(ir, mode) :
              GetTensorDofToQuad(*this, ir, mode, basis1d, true, dof2quad_array);
    }
+
+   void GetFaceMap(const int face_id, Array<int> &face_map) const override;
 };
 
 

fem/fe/fe_rt.cpp

@@ -12,6 +12,7 @@
 // Raviart-Thomas Finite Element classes
 
 #include "fe_rt.hpp"
+#include "face_map_utils.hpp"
 #include "../coefficient.hpp"
 
 namespace mfem
@@ -297,6 +298,27 @@ void RT_QuadrilateralElement::ProjectIntegrated(VectorCoefficient &vc,
    }
 }
 
+void RT_QuadrilateralElement::GetFaceMap(const int face_id,
+                                         Array<int> &face_map) const
+{
+   const int p = order;
+   const int pp1 = p + 1;
+   const int n_face_dofs = p;
+
+   std::vector<int> offsets;
+   std::vector<int> strides = {(face_id == 0 || face_id == 2) ? 1 : pp1};
+   switch (face_id)
+   {
+      case 0: offsets = {p*pp1}; break; // y = 0
+      case 1: offsets = {pp1 - 1}; break; // x = 1
+      case 2: offsets = {p*pp1 + p*(pp1 - 1)}; break; // y = 1
+      case 3: offsets = {0}; break; // x = 0
+   }
+
+   std::vector<int> n_dofs(dim - 1, p);
+   internal::FillFaceMap(n_face_dofs, offsets, strides, n_dofs, face_map);
+}
+
 
 const double RT_HexahedronElement::nk[18] =
 { 0.,0.,-1.,  0.,-1.,0.,  1.,0.,0.,  0.,1.,0.,  -1.,0.,0.,  0.,0.,1. };
@@ -686,6 +708,35 @@ void RT_HexahedronElement::ProjectIntegrated(VectorCoefficient &vc,
    }
 }
 
+void RT_HexahedronElement::GetFaceMap(const int face_id,
+                                      Array<int> &face_map) const
+{
+   const int p = order;
+   const int pp1 = p + 1;
+   int n_face_dofs = p*p;
+   std::vector<int> strides, offsets;
+   const int n_dof_per_dim = p*p*pp1;
+   const auto f = internal::GetFaceNormal3D(face_id);
+   const int face_normal = f.first, level = f.second;
+   if (face_normal == 0) // x-normal
+   {
+      offsets = {level ? pp1 - 1 : 0};
+      strides = {pp1, p*pp1};
+   }
+   else if (face_normal == 1) // y-normal
+   {
+      offsets = {n_dof_per_dim + (level ? p*(pp1 - 1) : 0)};
+      strides = {1, p*pp1};
+   }
+   else if (face_normal == 2) // z-normal
+   {
+      offsets = {2*n_dof_per_dim + (level ? p*p*(pp1 - 1) : 0)};
+      strides = {1, p};
+   }
+   std::vector<int> n_dofs = {p, p};
+   internal::FillFaceMap(n_face_dofs, offsets, strides, n_dofs, face_map);
+}
+
 
 const double RT_TriangleElement::nk[6] =
 { 0., -1., 1., 1., -1., 0. };

fem/fe/fe_rt.hpp

@@ -82,6 +82,8 @@ public:
                             DenseMatrix &curl) const
    { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
 
+   virtual void GetFaceMap(const int face_id, Array<int> &face_map) const;
+
 protected:
    void ProjectIntegrated(VectorCoefficient &vc, ElementTransformation &Trans,
                           Vector &dofs) const;
@@ -145,6 +147,11 @@ public:
                             DenseMatrix &curl) const
    { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 
+   /// @brief Return the mapping from lexicographically ordered face DOFs to
+   /// lexicographically ordered element DOFs corresponding to local face
+   /// @a face_id.
+   virtual void GetFaceMap(const int face_id, Array<int> &face_map) const;
+
 protected:
    void ProjectIntegrated(VectorCoefficient &vc,
                           ElementTransformation &Trans,

fem/fe_coll.cpp

@@ -3446,6 +3446,7 @@ NURBSFECollection::NURBSFECollection(int Order)
    : FiniteElementCollection((Order == VariableOrder) ? 1 : Order)
 {
    const int order = (Order == VariableOrder) ? 1 : Order;
+   PointFE          = new PointFiniteElement();
    SegmentFE        = new NURBS1DFiniteElement(order);
    QuadrilateralFE  = new NURBS2DFiniteElement(order);
    ParallelepipedFE = new NURBS3DFiniteElement(order);
@@ -3468,9 +3469,10 @@ void NURBSFECollection::SetOrder(int Order) const
 
 NURBSFECollection::~NURBSFECollection()
 {
-   delete ParallelepipedFE;
-   delete QuadrilateralFE;
+   delete PointFE;
    delete SegmentFE;
+   delete QuadrilateralFE;
+   delete ParallelepipedFE;
 }
 
 const FiniteElement *
@@ -3478,6 +3480,7 @@ NURBSFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
 {
    switch (GeomType)
    {
+      case Geometry::POINT:       return PointFE;
       case Geometry::SEGMENT:     return SegmentFE;
       case Geometry::SQUARE:      return QuadrilateralFE;
       case Geometry::CUBE:        return ParallelepipedFE;

fem/fe_coll.hpp

@@ -638,6 +638,7 @@ public:
 class NURBSFECollection : public FiniteElementCollection
 {
 private:
+   PointFiniteElement   *PointFE;
    NURBS1DFiniteElement *SegmentFE;
    NURBS2DFiniteElement *QuadrilateralFE;
    NURBS3DFiniteElement *ParallelepipedFE;

fem/fespace.cpp

@@ -1290,12 +1290,12 @@ const ElementRestrictionOperator *FiniteElementSpace::GetElementRestriction(
 }
 
 const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
-   ElementDofOrdering e_ordering, FaceType type, L2FaceValues mul) const
+   ElementDofOrdering f_ordering, FaceType type, L2FaceValues mul) const
 {
    const bool is_dg_space = IsDGSpace();
    const L2FaceValues m = (is_dg_space && mul==L2FaceValues::DoubleValued) ?
                           L2FaceValues::DoubleValued : L2FaceValues::SingleValued;
-   key_face key = std::make_tuple(is_dg_space, e_ordering, type, m);
+   key_face key = std::make_tuple(is_dg_space, f_ordering, type, m);
    auto itr = L2F.find(key);
    if (itr != L2F.end())
    {
@@ -1308,16 +1308,16 @@ const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
       {
          if (Conforming())
          {
-            res = new L2FaceRestriction(*this, e_ordering, type, m);
+            res = new L2FaceRestriction(*this, f_ordering, type, m);
          }
          else
          {
-            res = new NCL2FaceRestriction(*this, e_ordering, type, m);
+            res = new NCL2FaceRestriction(*this, f_ordering, type, m);
          }
       }
       else
       {
-         res = new H1FaceRestriction(*this, e_ordering, type);
+         res = new ConformingFaceRestriction(*this, f_ordering, type);
       }
       L2F[key] = res;
       return res;

fem/fespace.hpp

@@ -92,7 +92,130 @@ class FaceQuadratureInterpolator;
 
 
 /** @brief Class FiniteElementSpace - responsible for providing FEM view of the
-    mesh, mainly managing the set of degrees of freedom. */
+    mesh, mainly managing the set of degrees of freedom.
+
+    @details The term "degree of freedom", or "dof" for short, can mean
+    different things in different contexts. In MFEM we use "dof" to refer to
+    four closely related types of data; @ref edof "edofs", @ref ldof "ldofs",
+    @ref tdof "tdofs", and @ref vdof "vdofs".
+
+    @anchor edof @par Element DoF:
+    %Element dofs, sometimes referred to as @b edofs, are the expansion
+    coefficients used to build the linear combination of basis functions which
+    approximate a field within one element of the computational mesh. The
+    arrangement of the element dofs is determined by the basis function and
+    element types.
+    @par
+    %Element dofs are usually accessed one element at a time but they can be
+    concatenated together into a global vector when minimizing access time is
+    crucial. The global number of element dofs is not directly available from
+    the FiniteElementSpace. It can be determined by repeatedly calling
+    FiniteElementSpace::GetElementDofs and summing the lengths of the resulting
+    @a dofs arrays.
+
+    @anchor ldof @par Local DoF:
+    Most basis function types share many of their element dofs with neighboring
+    elements. Consequently, the global @ref edof "edof" vector suggested above
+    would contain many redundant entries. One of the primary roles of the
+    FiniteElementSpace is to collapse out these redundancies and
+    define a unique ordering of the remaining degrees of freedom. The
+    collapsed set of dofs are called @b "local dofs" or @b ldofs in
+    the MFEM parlance.
+    @par
+    The term @b local in this context refers to the local rank in a parallel
+    processing environment. MFEM can, of course, be used in sequential
+    computing environments but it is designed with parallel processing in mind
+    and this terminology reflects that design focus.
+    @par
+    When running in parallel the set of local dofs contains all of the degrees
+    of freedom associated with locally owned elements. When running in serial
+    all elements are locally owned so all element dofs are represented in the
+    set of local dofs.
+    @par
+    There are two important caveats regarding local dofs. First, some basis
+    function types, Nedelec and Raviart-Thomas are the prime examples, have an
+    orientation associated with each basis function. The relative orientations
+    of such basis functions in neighboring elements can lead to shared degrees
+    of freedom with opposite signs from the point of view of these neighboring
+    elements. MFEM typically chooses the orientation of the first such shared
+    degree of freedom that it encounters as the default orientation for the
+    corresponding local dof. When this local dof is referenced by a neighboring
+    element which happens to require the opposite orientation the local dof
+    index will be returned (by calls to functions such as
+    FiniteElementSpace::GetElementDofs) as a negative integer. In such cases
+    the actual offset into the vector of local dofs is @b -index-1 and the
+    value expected by this element should have the opposite sign to the value
+    stored in the local dof vector.
+    @par
+    The second important caveat only pertains to high order Nedelec basis
+    functions when shared triangular faces are present in the mesh. In this
+    very particular case the relative orientation of the face with respect to
+    its two neighboring elements can lead to different definitions of the
+    degrees of freedom associated with the interior of the face which cannot
+    be handled by simply flipping the signs of the corresponding values. The
+    DofTransformation class is designed to manage the necessary @b edof to
+    @b ldof transformations in this case. In the majority of cases the
+    DofTransformation is unnecessary and a NULL pointer will be returned in
+    place of a pointer to this object. See DofTransformation for more
+    information.
+
+    @anchor tdof @par True DoF:
+    As the name suggests "true dofs" or @b tdofs form the minimal set of data
+    values needed (along with mesh and basis function definitions) to uniquely
+    define a finite element discretization of a field. The number of true dofs
+    determines the size of the linear systems which typically need to be solved
+    in FEM simulations.
+    @par
+    Often the true dofs and the local dofs are identical, however, there are
+    important cases where they differ significantly. The first such case is
+    related to non-conforming meshes. On non-conforming meshes it is common
+    for degrees of freedom associated with "hanging" nodes, edges, or faces to
+    be constrained by degrees of freedom associated with another mesh entity.
+    In such cases the "hanging" degrees of freedom should not be considered
+    "true" degrees of freedom since their values cannot be independently
+    assigned. For this reason the FiniteElementSpace must process these
+    constraints and define a reduced set of "true" degrees of freedom which are
+    distinct from the local degrees of freedom.
+    @par
+    The second important distinction arises in parallel processing. When
+    distributing a linear system in parallel each degree of freedom must be
+    assigned to a particular processor, its owner. From the finite element
+    point of view it is convenient to distribute a computational mesh and
+    define an owning processor for each element. Since degrees of freedom may
+    be shared between neighboring elements they may also be shared between
+    neighboring processors. Another role of the FiniteElementSpace is to
+    identify the ownership of degrees of freedom which must be shared between
+    processors. Therefore the set of "true" degrees of freedom must also remove
+    redundant degrees of freedom which are owned by other processors.
+    @par
+    To summarize the set of true degrees of freedom are those degrees of
+    freedom needed to solve a linear system representing the partial
+    differential equation being modeled. True dofs differ from "local" dofs by
+    eliminating redundancies across processor boundaries and applying
+    the constraints needed to properly define fields on non-conforming meshes.
+
+    @anchor vdof @par Vector DoF:
+    %Vector dofs or @b vdofs are related to fields which are constructed using
+    multiple copies of the same set of basis functions. A typical example would
+    be the use of three instances of the scalar H1 basis functions to
+    approximate the x, y, and z components of a displacement vector field in
+    three dimensional space as often seen in elasticity simulations.
+    @par
+    %Vector dofs do not represent a specific index space the way the three
+    previous types of dofs do. Rather they are related to modifications of
+    these other index spaces to accomodate multiple copies of the underlying
+    function spaces.
+    @par
+    When using @b vdofs, i.e. when @b vdim != 1, the FiniteElementSpace only
+    manages a single set of degrees of freedom and then uses simple rules to
+    determine the appropriate offsets into the full index spaces. Two ordering
+    rules are supported; @b byNODES and @b byVDIM. See Ordering::Type for
+    details.
+    @par
+    Clearly the notion of a @b vdof is relevant in each of the three contexts
+    mentioned above so extra care must be taken whenever @b vdim != 1 to ensure
+    that the @b edof, @b ldof, or @b tdof is being interpretted correctly.
+ */
 class FiniteElementSpace
 {
    friend class InterpolationGridTransfer;
@@ -509,7 +632,7 @@ public:
        is the number of the mesh elements.
 
        The parameter @a e_ordering describes how the local DOFs in each element
-       should be ordered, see ElementDofOrdering.
+       should be ordered in the E-vector, see ElementDofOrdering.
 
        For discontinuous spaces, the element restriction corresponds to a
        permutation of the degrees of freedom, implemented by the
@@ -521,7 +644,7 @@ public:
 
    /// Return an Operator that converts L-vectors to E-vectors on each face.
    virtual const FaceRestriction *GetFaceRestriction(
-      ElementDofOrdering e_ordering, FaceType,
+      ElementDofOrdering f_ordering, FaceType,
       L2FaceValues mul = L2FaceValues::DoubleValued) const;
 
    /** @brief Return a QuadratureInterpolator that interpolates E-vectors to
@@ -580,13 +703,14 @@ public:
    /// Returns vector dimension.
    inline int GetVDim() const { return vdim; }
 
-   /// Returns number of degrees of freedom.
+   /// @brief Returns number of degrees of freedom.
+   /// This is the number of @ref ldof "Local Degrees of Freedom"
    inline int GetNDofs() const { return ndofs; }
 
-   /// Return the number of vector dofs, i.e. GetNDofs() x GetVDim().
+   /// @brief Return the number of vector dofs, i.e. GetNDofs() x GetVDim().
    inline int GetVSize() const { return vdim * ndofs; }
 
-   /// Return the number of vector true (conforming) dofs.
+   /// @brief Return the number of vector true (conforming) dofs.
    virtual int GetTrueVSize() const { return GetConformingVSize(); }
 
    /// Returns the number of conforming ("true") degrees of freedom
@@ -660,76 +784,279 @@ public:
 
    int GetBdrAttribute(int i) const { return mesh->GetBdrAttribute(i); }
 
-   /// Returns indices of degrees of freedom of element 'elem'.
+   /// @anchor getdof @name Local DoF Access Members
+   /// These member functions produce arrays of local degree of freedom
+   /// indices, see @ref ldof. If @b vdim == 1 these indices can be used to
+   /// access entries in GridFunction, LinearForm, and BilinearForm objects.
+   /// If @b vdim != 1 the corresponding @ref getvdof "Get*VDofs" methods
+   /// should be used instead or one of the @ref dof2vdof "DofToVDof" methods
+   /// could be used to produce the appropriate offsets from these local dofs.
+   ///@{
+
+   /// @brief Returns indices of degrees of freedom of element 'elem'.
+   /// The returned indices are offsets into an @ref ldof vector. See also
+   /// GetElementVDofs().
+   ///
+   /// @note In many cases the returned DofTransformation object will be NULL.
+   /// In other cases see the documentation of the DofTransformation class for
+   /// guidance on its role in performing @ref edof to @ref ldof transformations
+   /// on local vectors and matrices. At present the DofTransformation is only
+   /// needed for Nedelec basis functions of order 2 and above on 3D elements
+   /// with triangular faces.
+   ///
+   /// @note The returned object should NOT be deleted by the caller.
    virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
 
-   /// Returns indices of degrees of freedom for boundary element 'bel'.
+   /// @brief Returns indices of degrees of freedom for boundary element 'bel'.
+   /// The returned indices are offsets into an @ref ldof vector. See also
+   /// GetBdrElementVDofs().
+   ///
+   /// @note In many cases the returned DofTransformation object will be NULL.
+   /// In other cases see the documentation of the DofTransformation class for
+   /// guidance on its role in performing @ref edof to @ref ldof transformations
+   /// on local vectors and matrices. At present the DofTransformation is only
+   /// needed for Nedelec basis functions of order 2 and above on 3D elements
+   /// with triangular faces.
+   ///
+   /// @note The returned object should NOT be deleted by the caller.
    virtual DofTransformation *GetBdrElementDofs(int bel,
                                                 Array<int> &dofs) const;
 
-   /** @brief Returns the indices of the degrees of freedom for the specified
-       face, including the DOFs for the edges and the vertices of the face. */
-   /** In variable order spaces, multiple variants of DOFs can be returned.
-       See @a GetEdgeDofs for more details.
-       @return Order of the selected variant, or -1 if there are no more
-       variants.*/
+   /// @brief Returns the indices of the degrees of freedom for the specified
+   /// face, including the DOFs for the edges and the vertices of the face.
+   ///
+   /// In variable order spaces, multiple variants of DOFs can be returned.
+   /// See GetEdgeDofs() for more details.
+   /// @return Order of the selected variant, or -1 if there are no more
+   /// variants.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector. See also
+   /// GetFaceVDofs().
    virtual int GetFaceDofs(int face, Array<int> &dofs, int variant = 0) const;
 
-   /** @brief Returns the indices of the degrees of freedom for the specified
-       edge, including the DOFs for the vertices of the edge. */
-   /** In variable order spaces, multiple sets of DOFs may exist on an edge,
-       corresponding to the different polynomial orders of incident elements.
-       The 'variant' parameter is the zero-based index of the desired DOF set.
-       The variants are ordered from lowest polynomial degree to the highest.
-       @return Order of the selected variant, or -1 if there are no more
-       variants. */
+   /// @brief Returns the indices of the degrees of freedom for the specified
+   /// edge, including the DOFs for the vertices of the edge.
+   ///
+   /// In variable order spaces, multiple sets of DOFs may exist on an edge,
+   /// corresponding to the different polynomial orders of incident elements.
+   /// The 'variant' parameter is the zero-based index of the desired DOF set.
+   /// The variants are ordered from lowest polynomial degree to the highest.
+   /// @return Order of the selected variant, or -1 if there are no more
+   /// variants.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector. See also
+   /// GetEdgeVDofs().
    int GetEdgeDofs(int edge, Array<int> &dofs, int variant = 0) const;
 
+   /// @brief Returns the indices of the degrees of freedom for the specified
+   /// vertices.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector. See also
+   /// GetVertexVDofs().
    void GetVertexDofs(int i, Array<int> &dofs) const;
 
+   /// @brief Returns the indices of the degrees of freedom for the interior
+   /// of the specified element.
+   ///
+   /// Specifically this refers to degrees of freedom which are not associated
+   /// with the vertices, edges, or faces of the mesh. This method may be
+   /// useful in conjunction with schemes which process shared and non-shared
+   /// degrees of freedom differently such as static condensation.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector. See also
+   /// GetElementInteriorVDofs().
    void GetElementInteriorDofs(int i, Array<int> &dofs) const;
 
+   /// @brief Returns the indices of the degrees of freedom for the interior
+   /// of the specified face.
+   ///
+   /// Specifically this refers to degrees of freedom which are not associated
+   /// with the vertices, edges, or cell interiors of the mesh. This method may
+   /// be useful in conjunction with schemes which process shared and non-shared
+   /// degrees of freedom differently such as static condensation.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector. See also
+   /// GetFaceInteriorVDofs().
    void GetFaceInteriorDofs(int i, Array<int> &dofs) const;
 
+   /// @brief Returns the number of degrees of freedom associated with the
+   /// interior of the specified element.
+   ///
+   /// See GetElementInteriorDofs() for more information or to obtain the
+   /// relevant indices.
    int GetNumElementInteriorDofs(int i) const;
 
+   /// @brief Returns the indices of the degrees of freedom for the interior
+   /// of the specified edge.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector. See also
+   /// GetEdgeInteriorVDofs().
    void GetEdgeInteriorDofs(int i, Array<int> &dofs) const;
+   ///@}
 
-   /** @brief Returns the indices of all of the VDofs for the specified
-       dimension 'vd'. */
-   /** The 'ndofs' parameter defines the number of Dofs in the
-       FiniteElementSpace. If 'ndofs' is -1 (the default value), then the
-       number of Dofs is determined by the FiniteElementSpace. */
+   /// @anchor dof2vdof @name DoF To VDoF Conversion methods
+   /// These methods convert between local dof and local vector dof using the
+   /// appropriate relationship based on the Ordering::Type defined in this
+   /// FiniteElementSpace object.
+   ///
+   /// These methods assume the index set has a range [0, GetNDofs()) which
+   /// will be mapped to the range [0, GetVSize()). This assumption can be
+   /// changed in the forward mappings by passing a value for @a ndofs which
+   /// differs from that returned by GetNDofs().
+   ///
+   /// @note Thse methods, with the exception of VDofToDof(), are designed to
+   /// produce the correctly encoded values when dof entries are negative,
+   /// see @ref ldof for more on negative dof indices.
+   ///
+   /// @warning When MFEM_DEBUG is enabled at build time the forward mappings
+   /// will verify that each @a dof lies in the proper range. If MFEM_DEBUG is
+   /// disabled no range checking is performed.
+   ///@{
+
+   /// @brief Returns the indices of all of the VDofs for the specified
+   /// dimension 'vd'.
+   ///
+   /// The @a ndofs parameter can be used to indicate the total number of Dofs
+   /// associated with each component of @b vdim. If @a ndofs is -1 (the
+   /// default value), then the number of Dofs is determined by the
+   /// FiniteElementSpace::GetNDofs().
+   ///
+   /// @note This method does not resize the @a dofs array. It takes the range
+   /// of dofs [0, dofs.Size()) and converts these to @ref vdof "vdofs" and
+   /// stores the results in the @a dofs array.
    void GetVDofs(int vd, Array<int> &dofs, int ndofs = -1) const;
 
+   /// @brief Compute the full set of @ref vdof "vdofs" corresponding to each
+   /// entry in @a dofs.
+   ///
+   /// @details Produces a set of @ref vdof "vdofs" of
+   /// length @b vdim * dofs.Size() corresponding to the entries contained in
+   /// the @a dofs array.
+   ///
+   /// The @a ndofs parameter can be used to indicate the total number of Dofs
+   /// associated with each component of @b vdim. If @a ndofs is -1 (the
+   /// default value), then the number of Dofs is <determined by the
+   /// FiniteElementSpace::GetNDofs().
+   ///
+   /// @note The @a dofs array is overwritten and resized to accomodate the
+   /// new values.
    void DofsToVDofs(Array<int> &dofs, int ndofs = -1) const;
 
+   /// @brief Compute the set of @ref vdof "vdofs" corresponding to each entry
+   /// in @a dofs for the given vector index @a vd.
+   ///
+   /// The @a ndofs parameter can be used to indicate the total number of Dofs
+   /// associated with each component of @b vdim. If @a ndofs is -1 (the
+   /// default value), then the number of Dofs is <determined by the
+   /// FiniteElementSpace::GetNDofs().
+   ///
+   /// @note The @a dofs array is overwritten with the new values but its size
+   /// will not be altered.
    void DofsToVDofs(int vd, Array<int> &dofs, int ndofs = -1) const;
 
+   /// @brief Compute a single @ref vdof corresponding to the index @a dof and
+   /// the vector index @a vd.
+   ///
+   /// The @a ndofs parameter can be used to indicate the total number of Dofs
+   /// associated with each component of @b vdim. If @a ndofs is -1 (the
+   /// default value), then the number of Dofs is <determined by the
+   /// FiniteElementSpace::GetNDofs().
    int DofToVDof(int dof, int vd, int ndofs = -1) const;
 
+   /// @brief Compute the inverse of the Dof to VDof mapping for a single
+   /// index @a vdof.
+   ///
+   /// @warning This method is only intended for use with positive indices.
+   /// Passing a negative value for @a vdof will produce an invalid result.
    int VDofToDof(int vdof) const
    { return (ordering == Ordering::byNODES) ? (vdof%ndofs) : (vdof/vdim); }
 
+   ///@}
+
+   /// @brief Remove the orientation information encoded into an array of dofs
+   /// Some basis function types have a relative orientation associated with
+   /// degrees of freedom shared between neighboring elements, see @ref ldof
+   /// for more information. An orientation mismatch is indicated in the dof
+   /// indices by a negative index value. This method replaces such negative
+   /// indices with the corresponding positive offsets.
+   ///
+   /// @note The name of this method reflects the fact that it is most often
+   /// applied to sets of @ref vdof "Vector Dofs" but it would work equally
+   /// well on sets of @ref ldof "Local Dofs".
    static void AdjustVDofs(Array<int> &vdofs);
 
-   /// Returns indexes of degrees of freedom in array dofs for i'th element.
+   /// @anchor getvdof @name Local Vector DoF Access Members
+   /// These member functions produce arrays of local vector degree of freedom
+   /// indices, see @ref ldof and @ref vdof. These indices can be used to
+   /// access entries in GridFunction, LinearForm, and BilinearForm objects
+   /// regardless of the value of @b vdim.
+   /// @{
+
+   /// @brief Returns indices of degrees of freedom for the @a i'th element.
+   /// The returned indices are offsets into an @ref ldof vector with @b vdim
+   /// not necessarily equal to 1. See also GetElementDofs().
+   ///
+   /// @note In many cases the returned DofTransformation object will be NULL.
+   /// In other cases see the documentation of the DofTransformation class for
+   /// guidance on its role in performing @ref edof to @ref ldof transformations
+   /// on local vectors and matrices. At present the DofTransformation is only
+   /// needed for Nedelec basis functions of order 2 and above on 3D elements
+   /// with triangular faces.
+   ///
+   /// @note The returned object should NOT be deleted by the caller.
    DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;
 
-   /// Returns indexes of degrees of freedom for i'th boundary element.
+   /// @brief Returns indices of degrees of freedom for @a i'th boundary
+   /// element.
+   /// The returned indices are offsets into an @ref ldof vector with @b vdim
+   /// not necessarily equal to 1. See also GetBdrElementDofs().
+   ///
+   /// @note In many cases the returned DofTransformation object will be NULL.
+   /// In other cases see the documentation of the DofTransformation class for
+   /// guidance on its role in performing @ref edof to @ref ldof transformations
+   /// on local vectors and matrices. At present the DofTransformation is only
+   /// needed for Nedelec basis functions of order 2 and above on 3D elements
+   /// with triangular faces.
+   ///
+   /// @note The returned object should NOT be deleted by the caller.
    DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;
 
-   /// Returns indexes of degrees of freedom for i'th face element (2D and 3D).
+   /// @brief Returns the indices of the degrees of freedom for the specified
+   /// face, including the DOFs for the edges and the vertices of the face.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector with @b vdim
+   /// not necessarily equal to 1. See GetFaceDofs() for more information.
    void GetFaceVDofs(int i, Array<int> &vdofs) const;
 
-   /// Returns indexes of degrees of freedom for i'th edge.
+   /// @brief Returns the indices of the degrees of freedom for the specified
+   /// edge, including the DOFs for the vertices of the edge.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector with @b vdim
+   /// not necessarily equal to 1. See GetEdgeDofs() for more information.
    void GetEdgeVDofs(int i, Array<int> &vdofs) const;
 
+   /// @brief Returns the indices of the degrees of freedom for the specified
+   /// vertices.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector with @b vdim
+   /// not necessarily equal to 1. See also GetVertexDofs().
    void GetVertexVDofs(int i, Array<int> &vdofs) const;
 
+   /// @brief Returns the indices of the degrees of freedom for the interior
+   /// of the specified element.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector with @b vdim
+   /// not necessarily equal to 1. See GetElementInteriorDofs() for more
+   /// information.
    void GetElementInteriorVDofs(int i, Array<int> &vdofs) const;
 
+   /// @brief Returns the indices of the degrees of freedom for the interior
+   /// of the specified edge.
+   ///
+   /// The returned indices are offsets into an @ref ldof vector with @b vdim
+   /// not necessarily equal to 1. See also GetEdgeInteriorDofs().
    void GetEdgeInteriorVDofs(int i, Array<int> &vdofs) const;
+   /// @}
 
    /// (@deprecated) Use the Update() method if the space or mesh changed.
    MFEM_DEPRECATED void RebuildElementToDofTable();

fem/gridfunc.cpp

@@ -2015,6 +2015,24 @@ void GridFunction::GetNodalValues(Vector &nval, int vdim) const
    }
 }
 
+
+void GridFunction::CountElementsPerVDof(Array<int> &elem_per_vdof) const
+{
+   elem_per_vdof.SetSize(fes->GetVSize());
+   elem_per_vdof = 0;
+   Array<int> vdofs;
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      fes->GetElementVDofs(i, vdofs);
+      // Accumulate values in all dofs, count the zones.
+      for (int j = 0; j < vdofs.Size(); j++)
+      {
+         elem_per_vdof[vdofs[j]]++;
+      }
+   }
+}
+
 void GridFunction::AccumulateAndCountZones(Coefficient &coeff,
                                            AvgType type,
                                            Array<int> &zones_per_vdof)

fem/gridfunc.hpp

@@ -458,6 +458,10 @@ protected:
    void ComputeMeans(AvgType type, Array<int> &zones_per_vdof);
 
 public:
+   /** @brief For each vdof, counts how many elements contain the vdof,
+       as containment is determined by FiniteElementSpace::GetElementVDofs(). */
+   virtual void CountElementsPerVDof(Array<int> &elem_per_vdof) const;
+
    /** @brief Project a Coefficient on the GridFunction, modifying only DOFs on
        the boundary associated with the boundary attributes marked in the
        @a attr array. */

fem/linearform.cpp

@@ -101,7 +101,7 @@ void LinearForm::AddInteriorFaceIntegrator(LinearFormIntegrator *lfi)
    interior_face_integs.Append(lfi);
 }
 
-bool LinearForm::SupportsDevice()
+bool LinearForm::SupportsDevice() const
 {
    // return false for NURBS meshes, so we don’t convert it to non-NURBS
    // through Assemble, AssembleDevice, GetGeometricFactors and EnsureNodes

fem/linearform.hpp

@@ -203,7 +203,7 @@ public:
    void Assemble();
 
    /// Return true if assembly on device is supported, false otherwise.
-   virtual bool SupportsDevice();
+   virtual bool SupportsDevice() const;
 
    /// Assembles delta functions of the linear form
    void AssembleDelta();

fem/lininteg.hpp

@@ -31,7 +31,7 @@ protected:
 public:
 
    /// Method probing for assembly on device
-   virtual bool SupportsDevice() { return false; }
+   virtual bool SupportsDevice() const { return false; }
 
    /// Method defining assembly on device
    virtual void AssembleDevice(const FiniteElementSpace &fes,
@@ -121,7 +121,7 @@ public:
    DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir)
       : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1) { }
 
-   virtual bool SupportsDevice() { return true; }
+   virtual bool SupportsDevice() const { return true; }
 
    /// Method defining assembly on device
    virtual void AssembleDevice(const FiniteElementSpace &fes,
@@ -154,7 +154,7 @@ public:
    DomainLFGradIntegrator(VectorCoefficient &QF)
       : DeltaLFIntegrator(QF), Q(QF) { }
 
-   virtual bool SupportsDevice() { return true; }
+   virtual bool SupportsDevice() const { return true; }
 
    /// Method defining assembly on device
    virtual void AssembleDevice(const FiniteElementSpace &fes,
@@ -187,7 +187,7 @@ public:
    BoundaryLFIntegrator(Coefficient &QG, int a = 1, int b = 1)
       : Q(QG), oa(a), ob(b) { }
 
-   virtual bool SupportsDevice() { return true; }
+   virtual bool SupportsDevice() const { return true; }
 
    /// Method defining assembly on device
    virtual void AssembleDevice(const FiniteElementSpace &fes,
@@ -217,7 +217,7 @@ public:
    BoundaryNormalLFIntegrator(VectorCoefficient &QG, int a = 1, int b = 1)
       : Q(QG), oa(a), ob(b) { }
 
-   virtual bool SupportsDevice() { return true; }
+   virtual bool SupportsDevice() const { return true; }
 
    /// Method defining assembly on device
    virtual void AssembleDevice(const FiniteElementSpace &fes,
@@ -262,7 +262,7 @@ public:
    VectorDomainLFIntegrator(VectorCoefficient &QF)
       : DeltaLFIntegrator(QF), Q(QF) { }
 
-   virtual bool SupportsDevice() { return true; }
+   virtual bool SupportsDevice() const { return true; }
 
    /// Method defining assembly on device
    virtual void AssembleDevice(const FiniteElementSpace &fes,
@@ -296,7 +296,7 @@ public:
    VectorDomainLFGradIntegrator(VectorCoefficient &QF)
       : DeltaLFIntegrator(QF), Q(QF) { }
 
-   virtual bool SupportsDevice() override { return true; }
+   virtual bool SupportsDevice() const override { return true; }
 
    /// Method defining assembly on device
    virtual void AssembleDevice(const FiniteElementSpace &fes,
@@ -362,6 +362,12 @@ public:
                                          ElementTransformation &Trans,
                                          Vector &elvect);
 
+   virtual bool SupportsDevice() const { return true; }
+
+   virtual void AssembleDevice(const FiniteElementSpace &fes,
+                               const Array<int> &markers,
+                               Vector &b);
+
    using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
@@ -456,6 +462,12 @@ public:
                                        Vector &elvect);
 
    using LinearFormIntegrator::AssembleRHSElementVect;
+
+   virtual bool SupportsDevice() const { return true; }
+
+   virtual void AssembleDevice(const FiniteElementSpace &fes,
+                               const Array<int> &markers,
+                               Vector &b);
 };
 
 /// Class for boundary integration \f$ L(v) = (n \times f, v) \f$

fem/lininteg_boundary_flux.cpp

@@ -0,0 +1,180 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "../fem/kernels.hpp"
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+
+template<int T_D1D = 0, int T_Q1D = 0> static
+void BFLFEvalAssemble2D(const int nbe, const int d, const int q,
+                        const int *markers, const double *b,
+                        const double *weights, const Vector &coeff, double *y)
+{
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, nbe);
+   const auto B = Reshape(b, q, d);
+   const auto W = Reshape(weights, q);
+   const bool const_coeff = coeff.Size() == 1;
+   const auto C = const_coeff ? Reshape(F,1,1) : Reshape(F,q,nbe);
+   auto Y = Reshape(y, d, nbe);
+
+   MFEM_FORALL(e, nbe,
+   {
+      if (M(e) == 0) { return; } // ignore (in a lambda return acts as continue)
+
+      constexpr int Q = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QQ[Q];
+
+      for (int qx = 0; qx < q; ++qx)
+      {
+         const double coeff_val = const_coeff ? C(0,0) : C(qx,e);
+         QQ[qx] = W(qx) * coeff_val;
+      }
+      for (int dx = 0; dx < d; ++dx)
+      {
+         double u = 0;
+         for (int qx = 0; qx < q; ++qx) { u += QQ[qx] * B(qx,dx); }
+         Y(dx,e) += u;
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0> static
+void BFLFEvalAssemble3D(const int nbe, const int d, const int q,
+                        const int *markers, const double *b,
+                        const double *weights, const Vector &coeff, double *y)
+{
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, nbe);
+   const auto B = Reshape(b, q, d);
+   const auto W = Reshape(weights, q, q);
+   const bool const_coeff = coeff.Size() == 1;
+   const auto C = const_coeff ? Reshape(F,1,1,1) : Reshape(F,q,q,nbe);
+   auto Y = Reshape(y, d, d, nbe);
+
+   MFEM_FORALL_2D(e, nbe, q, q, 1,
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : MAX_D1D;
+
+      MFEM_SHARED double sBt[Q*D];
+      MFEM_SHARED double sQQ[Q*Q];
+      MFEM_SHARED double sQD[Q*D];
+
+      const DeviceMatrix Bt(sBt, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, B, sBt);
+
+      const DeviceMatrix QQ(sQQ, q, q);
+      const DeviceMatrix QD(sQD, q, d);
+
+      MFEM_FOREACH_THREAD(x,x,q)
+      {
+         MFEM_FOREACH_THREAD(y,y,q)
+         {
+            const double coeff_val = const_coeff ? C(0,0,0) : C(x,y,e);
+            QQ(y,x) = W(x,y) * coeff_val;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,q)
+      {
+         MFEM_FOREACH_THREAD(dx,x,d)
+         {
+            double u = 0.0;
+            for (int qx = 0; qx < q; ++qx) { u += QQ(qy,qx) * Bt(dx,qx); }
+            QD(qy,dx) = u;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,d)
+      {
+         MFEM_FOREACH_THREAD(dx,x,d)
+         {
+            double u = 0.0;
+            for (int qy = 0; qy < q; ++qy) { u += QD(qy,dx) * Bt(dy,qy); }
+            Y(dx,dy,e) += u;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+static void BFLFEvalAssemble(const FiniteElementSpace &fes,
+                             const IntegrationRule &ir,
+                             const Array<int> &markers,
+                             const Vector &coeff,
+                             Vector &y)
+{
+   Mesh &mesh = *fes.GetMesh();
+   const int dim = mesh.Dimension();
+   const FiniteElement &el = *fes.GetBE(0);
+   const DofToQuad &maps = el.GetDofToQuad(ir, DofToQuad::TENSOR);
+   const int d = maps.ndof, q = maps.nqpt;
+   auto ker = (dim == 2) ? BFLFEvalAssemble2D<> : BFLFEvalAssemble3D<>;
+
+   if (dim==2)
+   {
+      if (d==1 && q==1) { ker=BFLFEvalAssemble2D<1,1>; }
+      if (d==2 && q==2) { ker=BFLFEvalAssemble2D<2,2>; }
+      if (d==3 && q==3) { ker=BFLFEvalAssemble2D<3,3>; }
+      if (d==4 && q==4) { ker=BFLFEvalAssemble2D<4,4>; }
+      if (d==5 && q==5) { ker=BFLFEvalAssemble2D<5,5>; }
+      if (d==2 && q==3) { ker=BFLFEvalAssemble2D<2,3>; }
+      if (d==3 && q==4) { ker=BFLFEvalAssemble2D<3,4>; }
+      if (d==4 && q==5) { ker=BFLFEvalAssemble2D<4,5>; }
+      if (d==5 && q==6) { ker=BFLFEvalAssemble2D<5,6>; }
+   }
+
+   if (dim==3)
+   {
+      if (d==1 && q==1) { ker=BFLFEvalAssemble3D<1,1>; }
+      if (d==2 && q==2) { ker=BFLFEvalAssemble3D<2,2>; }
+      if (d==3 && q==3) { ker=BFLFEvalAssemble3D<3,3>; }
+      if (d==4 && q==4) { ker=BFLFEvalAssemble3D<4,4>; }
+      if (d==5 && q==5) { ker=BFLFEvalAssemble3D<5,5>; }
+      if (d==2 && q==3) { ker=BFLFEvalAssemble3D<2,3>; }
+      if (d==3 && q==4) { ker=BFLFEvalAssemble3D<3,4>; }
+      if (d==4 && q==5) { ker=BFLFEvalAssemble3D<4,5>; }
+      if (d==5 && q==6) { ker=BFLFEvalAssemble3D<5,6>; }
+   }
+
+   MFEM_VERIFY(ker, "No kernel ndof " << d << " nqpt " << q);
+
+   const int nbe = fes.GetMesh()->GetNFbyType(FaceType::Boundary);
+   const int *M = markers.Read();
+   const double *B = maps.B.Read();
+   const double *W = ir.GetWeights().Read();
+   double *Y = y.ReadWrite();
+   ker(nbe, d, q, M, B, W, coeff, Y);
+}
+
+void VectorFEBoundaryFluxLFIntegrator::AssembleDevice(
+   const FiniteElementSpace &fes,
+   const Array<int> &markers,
+   Vector &b)
+{
+   const FiniteElement &fe = *fes.GetBE(0);
+   const int qorder = oa * fe.GetOrder() + ob;
+   const Geometry::Type gtype = fe.GetGeomType();
+   const IntegrationRule &ir = IntRule ? *IntRule : IntRules.Get(gtype, qorder);
+   Mesh &mesh = *fes.GetMesh();
+
+   FaceQuadratureSpace qs(mesh, ir, FaceType::Boundary);
+   CoefficientVector coeff(F, qs, CoefficientStorage::COMPRESSED);
+   BFLFEvalAssemble(fes, ir, markers, coeff, b);
+}
+
+} // namespace mfem

fem/lininteg_vectorfe_domain.cpp

@@ -0,0 +1,347 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "../fem/kernels.hpp"
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void HdivDLFAssemble2D(
+   const int ne, const int d, const int q, const int *markers, const double *bo,
+   const double *bc, const double *j, const double *weights,
+   const Vector &coeff, double *y)
+{
+   MFEM_VERIFY(T_D1D || d <= HDIV_MAX_D1D, "Problem size too large.");
+   MFEM_VERIFY(T_Q1D || q <= HDIV_MAX_Q1D, "Problem size too large.");
+
+   static constexpr int vdim = 2;
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, ne);
+   const auto BO = Reshape(bo, q, d-1);
+   const auto BC = Reshape(bc, q, d);
+   const auto J = Reshape(j, q, q, vdim, vdim, ne);
+   const auto W = Reshape(weights, q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne);
+   auto Y = Reshape(y, 2*(d-1)*d, ne);
+
+   MFEM_FORALL_3D(e, ne, q, q, vdim,
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : HDIV_MAX_D1D;
+
+      MFEM_SHARED double sBot[Q*D];
+      MFEM_SHARED double sBct[Q*D];
+      MFEM_SHARED double sQQ[vdim*Q*Q];
+      MFEM_SHARED double sQD[vdim*Q*D];
+
+      // Bo and Bc into shared memory
+      const DeviceMatrix Bot(sBot, d-1, q);
+      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
+      const DeviceMatrix Bct(sBct, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
+
+      const DeviceCube QQ(sQQ, q, q, vdim);
+      const DeviceCube QD(sQD, q, d, vdim);
+
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const double cst_val_0 = C(0,0,0,0);
+         const double cst_val_1 = C(1,0,0,0);
+         MFEM_FOREACH_THREAD(y,y,q)
+         {
+            MFEM_FOREACH_THREAD(x,x,q)
+            {
+               const double J0 = J(x,y,0,vd,e);
+               const double J1 = J(x,y,1,vd,e);
+               const double C0 = cst ? cst_val_0 : C(0,x,y,e);
+               const double C1 = cst ? cst_val_1 : C(1,x,y,e);
+               QQ(x,y,vd) = W(x,y)*(J0*C0 + J1*C1);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double qd = 0.0;
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
+               }
+               QD(dx,qy,vd) = qd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
+         DeviceTensor<4> Yxy(Y, nx, ny, vdim, ne);
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double dd = 0.0;
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  dd += QD(dx,qy,vd) * Bty(dy,qy);
+               }
+               Yxy(dx,dy,vd,e) += dd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void HdivDLFAssemble3D(
+   const int ne, const int d, const int q, const int *markers, const double *bo,
+   const double *bc, const double *j, const double *weights,
+   const Vector &coeff, double *y)
+{
+   MFEM_VERIFY(T_D1D || d <= HDIV_MAX_D1D, "Problem size too large.");
+   MFEM_VERIFY(T_Q1D || q <= HDIV_MAX_Q1D, "Problem size too large.");
+
+   static constexpr int vdim = 3;
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, ne);
+   const auto BO = Reshape(bo, q, d-1);
+   const auto BC = Reshape(bc, q, d);
+   const auto J = Reshape(j, q, q, q, vdim, vdim, ne);
+   const auto W = Reshape(weights, q, q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1,1) : Reshape(F,vdim,q,q,q,ne);
+   auto Y = Reshape(y, 2*(d-1)*(d-1)*d, ne);
+
+   MFEM_FORALL_3D(e, ne, q, q, vdim,
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : HDIV_MAX_D1D;
+
+      MFEM_SHARED double sBot[Q*D];
+      MFEM_SHARED double sBct[Q*D];
+
+      // Bo and Bc into shared memory
+      const DeviceMatrix Bot(sBot, d-1, q);
+      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
+      const DeviceMatrix Bct(sBct, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
+
+      MFEM_SHARED double sm0[vdim*Q*Q*Q];
+      MFEM_SHARED double sm1[vdim*Q*Q*Q];
+      DeviceTensor<4> QQQ(sm1, q, q, q, vdim);
+      DeviceTensor<4> DQQ(sm0, d, q, q, vdim);
+      DeviceTensor<4> DDQ(sm1, d, d, q, vdim);
+
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const double cst_val_0 = C(0,0,0,0,0);
+         const double cst_val_1 = C(1,0,0,0,0);
+         const double cst_val_2 = C(2,0,0,0,0);
+         MFEM_FOREACH_THREAD(y,y,q)
+         {
+            MFEM_FOREACH_THREAD(x,x,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  const double J0 = J(x,y,z,0,vd,e);
+                  const double J1 = J(x,y,z,1,vd,e);
+                  const double J2 = J(x,y,z,2,vd,e);
+                  const double C0 = cst ? cst_val_0 : C(0,x,y,z,e);
+                  const double C1 = cst ? cst_val_1 : C(1,x,y,z,e);
+                  const double C2 = cst ? cst_val_2 : C(2,x,y,z,e);
+                  QQQ(x,y,z,vd) = W(x,y,z)*(J0*C0 + J1*C1 + J2*C2);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         const int nz = (vd == 2) ? d : d-1;
+         DeviceTensor<5> Yxyz(Y, nx, ny, nz, vdim, ne);
+         DeviceMatrix Btz = (vd == 2) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[D];
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  MFEM_UNROLL(D)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+static void HdivDLFAssemble(const FiniteElementSpace &fes,
+                            const IntegrationRule *ir,
+                            const Array<int> &markers,
+                            const Vector &coeff,
+                            Vector &y)
+{
+   Mesh &mesh = *fes.GetMesh();
+   const int dim = mesh.Dimension();
+   const FiniteElement *el = fes.GetFE(0);
+   const auto *vel = dynamic_cast<const VectorTensorFiniteElement *>(el);
+   MFEM_VERIFY(vel != nullptr, "Must be VectorTensorFiniteElement");
+   const MemoryType mt = Device::GetDeviceMemoryType();
+   const DofToQuad &maps_o = vel->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   const DofToQuad &maps_c = vel->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int d = maps_c.ndof, q = maps_c.nqpt;
+   constexpr int flags = GeometricFactors::JACOBIANS;
+   const GeometricFactors *geom = mesh.GetGeometricFactors(*ir, flags, mt);
+   decltype(&HdivDLFAssemble2D<>) ker =
+      dim == 2 ? HdivDLFAssemble2D<> : HdivDLFAssemble3D<>;
+
+   if (dim==2)
+   {
+      if (d==1 && q==1) { ker=HdivDLFAssemble2D<1,1>; }
+      if (d==2 && q==2) { ker=HdivDLFAssemble2D<2,2>; }
+      if (d==3 && q==3) { ker=HdivDLFAssemble2D<3,3>; }
+      if (d==4 && q==4) { ker=HdivDLFAssemble2D<4,4>; }
+      if (d==5 && q==5) { ker=HdivDLFAssemble2D<5,5>; }
+      if (d==6 && q==6) { ker=HdivDLFAssemble2D<6,6>; }
+      if (d==7 && q==7) { ker=HdivDLFAssemble2D<7,7>; }
+      if (d==8 && q==8) { ker=HdivDLFAssemble2D<8,8>; }
+   }
+
+   if (dim==3)
+   {
+      if (d==2 && q==2) { ker=HdivDLFAssemble3D<2,2>; }
+      if (d==3 && q==3) { ker=HdivDLFAssemble3D<3,3>; }
+      if (d==4 && q==4) { ker=HdivDLFAssemble3D<4,4>; }
+      if (d==5 && q==5) { ker=HdivDLFAssemble3D<5,5>; }
+      if (d==6 && q==6) { ker=HdivDLFAssemble3D<6,6>; }
+      if (d==7 && q==7) { ker=HdivDLFAssemble3D<7,7>; }
+      if (d==8 && q==8) { ker=HdivDLFAssemble3D<8,8>; }
+   }
+
+   MFEM_VERIFY(ker, "No kernel ndof " << d << " nqpt " << q);
+
+   const int ne = mesh.GetNE();
+   const int *M = markers.Read();
+   const double *Bo = maps_o.B.Read();
+   const double *Bc = maps_c.B.Read();
+   const double *J = geom->J.Read();
+   const double *W = ir->GetWeights().Read();
+   double *Y = y.ReadWrite();
+   ker(ne, d, q, M, Bo, Bc, J, W, coeff, Y);
+}
+
+void VectorFEDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
+                                                const Array<int> &markers,
+                                                Vector &b)
+{
+   const FiniteElement &fe = *fes.GetFE(0);
+   const int qorder = 2 * fe.GetOrder();
+   const Geometry::Type gtype = fe.GetGeomType();
+   const IntegrationRule *ir = IntRule ? IntRule : &IntRules.Get(gtype, qorder);
+
+   QuadratureSpace qs(*fes.GetMesh(), *ir);
+   CoefficientVector coeff(QF, qs, CoefficientStorage::COMPRESSED);
+
+   const int fe_type = fe.GetDerivType();
+   if (fe_type == FiniteElement::DIV)
+   {
+      HdivDLFAssemble(fes, ir, markers, coeff, b);
+   }
+   else
+   {
+      MFEM_ABORT("Not implemented.");
+   }
+}
+
+} // namespace mfem

fem/pfespace.cpp

@@ -540,12 +540,12 @@ const FiniteElement *ParFiniteElementSpace::GetFE(int i) const
 }
 
 const FaceRestriction *ParFiniteElementSpace::GetFaceRestriction(
-   ElementDofOrdering e_ordering, FaceType type, L2FaceValues mul) const
+   ElementDofOrdering f_ordering, FaceType type, L2FaceValues mul) const
 {
    const bool is_dg_space = IsDGSpace();
    const L2FaceValues m = (is_dg_space && mul==L2FaceValues::DoubleValued) ?
                           L2FaceValues::DoubleValued : L2FaceValues::SingleValued;
-   auto key = std::make_tuple(is_dg_space, e_ordering, type, m);
+   auto key = std::make_tuple(is_dg_space, f_ordering, type, m);
    auto itr = L2F.find(key);
    if (itr != L2F.end())
    {
@@ -558,22 +558,22 @@ const FaceRestriction *ParFiniteElementSpace::GetFaceRestriction(
       {
          if (Conforming())
          {
-            res = new ParL2FaceRestriction(*this, e_ordering, type, m);
+            res = new ParL2FaceRestriction(*this, f_ordering, type, m);
          }
          else
          {
-            res = new ParNCL2FaceRestriction(*this, e_ordering, type, m);
+            res = new ParNCL2FaceRestriction(*this, f_ordering, type, m);
          }
       }
       else
       {
          if (Conforming())
          {
-            res = new H1FaceRestriction(*this, e_ordering, type);
+            res = new ConformingFaceRestriction(*this, f_ordering, type);
          }
          else
          {
-            res = new ParNCH1FaceRestriction(*this, e_ordering, type);
+            res = new ParNCH1FaceRestriction(*this, f_ordering, type);
          }
       }
       L2F[key] = res;

fem/pfespace.hpp

@@ -310,7 +310,7 @@ public:
        the returned operator handles the communication needed to get the
        shared face values from other MPI ranks */
    virtual const FaceRestriction *GetFaceRestriction(
-      ElementDofOrdering e_ordering, FaceType type,
+      ElementDofOrdering f_ordering, FaceType type,
       L2FaceValues mul = L2FaceValues::DoubleValued) const;
 
    void GetSharedEdgeDofs(int group, int ei, Array<int> &dofs) const;

fem/pgridfunc.cpp

@@ -482,6 +482,15 @@ void ParGridFunction::GetVectorValue(ElementTransformation &T,
    }
 }
 
+void ParGridFunction::CountElementsPerVDof(Array<int> &elem_per_vdof) const
+{
+   GridFunction::CountElementsPerVDof(elem_per_vdof);
+   // Count the zones globally.
+   GroupCommunicator &gcomm = this->ParFESpace()->GroupComm();
+   gcomm.Reduce<int>(elem_per_vdof, GroupCommunicator::Sum);
+   gcomm.Bcast(elem_per_vdof);
+}
+
 void ParGridFunction::GetDerivative(int comp, int der_comp,
                                     ParGridFunction &der)
 {

fem/pgridfunc.hpp

@@ -226,6 +226,10 @@ public:
                                const IntegrationPoint &ip,
                                Vector &val, Vector *tr = NULL) const;
 
+   /** @brief For each vdof, counts how many elements contain the vdof,
+       as containment is determined by FiniteElementSpace::GetElementVDofs(). */
+   virtual void CountElementsPerVDof(Array<int> &elem_per_vdof) const;
+
    /// Parallel version of GridFunction::GetDerivative(); see its documentation.
    void GetDerivative(int comp, int der_comp, ParGridFunction &der);
 

fem/plinearform.cpp

@@ -54,7 +54,7 @@ void ParLinearForm::Assemble()
    }
 }
 
-bool ParLinearForm::SupportsDevice()
+bool ParLinearForm::SupportsDevice() const
 {
    bool parallel;
    bool local = LinearForm::SupportsDevice();

fem/plinearform.hpp

@@ -120,7 +120,7 @@ public:
    void Assemble();
 
    /// Return true if assembly on device is supported, false otherwise.
-   virtual bool SupportsDevice();
+   virtual bool SupportsDevice() const;
 
    void AssembleSharedFaces();
 

fem/prestriction.cpp

@@ -24,20 +24,24 @@ namespace mfem
 {
 
 ParNCH1FaceRestriction::ParNCH1FaceRestriction(const ParFiniteElementSpace &fes,
-                                               ElementDofOrdering ordering,
+                                               ElementDofOrdering f_ordering,
                                                FaceType type)
-   : H1FaceRestriction(fes, ordering, type, false),
+   : H1FaceRestriction(fes, f_ordering, type, false),
      type(type),
-     interpolations(fes, ordering, type)
+     interpolations(fes, f_ordering, type)
 {
    if (nf==0) { return; }
    x_interp.UseDevice(true);
 
-   CheckFESpace(ordering);
+   // Check that the space is H1 (not currently implemented for ND or RT spaces)
+   const bool is_h1 = dynamic_cast<const H1_FECollection*>(fes.FEColl());
+   MFEM_VERIFY(is_h1, "ParNCH1FaceRestriction is only implemented for H1 spaces.")
 
-   ComputeScatterIndicesAndOffsets(ordering, type);
+   CheckFESpace(f_ordering);
 
-   ComputeGatherIndices(ordering, type);
+   ComputeScatterIndicesAndOffsets(f_ordering, type);
+
+   ComputeGatherIndices(f_ordering, type);
 }
 
 void ParNCH1FaceRestriction::Mult(const Vector &x, Vector &y) const
@@ -171,7 +175,7 @@ void ParNCH1FaceRestriction::NonconformingTransposeInterpolationInPlace(
 }
 
 void ParNCH1FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType face_type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -198,12 +202,12 @@ void ParNCH1FaceRestriction::ComputeScatterIndicesAndOffsets(
          if ( face.IsConforming() )
          {
             interpolations.RegisterFaceConformingInterpolation(face,f_ind);
-            SetFaceDofsScatterIndices(face, f_ind, ordering);
+            SetFaceDofsScatterIndices(face, f_ind, f_ordering);
             f_ind++;
          }
          else // Non-conforming face
          {
-            SetFaceDofsScatterIndices(face, f_ind, ordering);
+            SetFaceDofsScatterIndices(face, f_ind, f_ordering);
             if ( face.element[0].conformity==Mesh::ElementConformity::Superset )
             {
                // In this case the local face is the master (coarse) face, thus
@@ -221,7 +225,7 @@ void ParNCH1FaceRestriction::ComputeScatterIndicesAndOffsets(
       }
       else if (face_type==FaceType::Boundary && face.IsBoundary())
       {
-         SetFaceDofsScatterIndices(face, f_ind, ordering);
+         SetFaceDofsScatterIndices(face, f_ind, f_ordering);
          f_ind++;
       }
    }
@@ -239,7 +243,7 @@ void ParNCH1FaceRestriction::ComputeScatterIndicesAndOffsets(
 }
 
 void ParNCH1FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType face_type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -257,7 +261,7 @@ void ParNCH1FaceRestriction::ComputeGatherIndices(
       }
       else if (face.IsOfFaceType(face_type))
       {
-         SetFaceDofsGatherIndices(face, f_ind, ordering);
+         SetFaceDofsGatherIndices(face, f_ind, f_ordering);
          f_ind++;
       }
    }
@@ -272,27 +276,27 @@ void ParNCH1FaceRestriction::ComputeGatherIndices(
 }
 
 ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
-                                           ElementDofOrdering ordering,
+                                           ElementDofOrdering f_ordering,
                                            FaceType type,
                                            L2FaceValues m,
                                            bool build)
-   : L2FaceRestriction(fes, ordering, type, m, false)
+   : L2FaceRestriction(fes, f_ordering, type, m, false)
 {
    if (!build) { return; }
    if (nf==0) { return; }
 
-   CheckFESpace(ordering);
+   CheckFESpace(f_ordering);
 
-   ComputeScatterIndicesAndOffsets(ordering, type);
+   ComputeScatterIndicesAndOffsets(f_ordering, type);
 
-   ComputeGatherIndices(ordering, type);
+   ComputeGatherIndices(f_ordering, type);
 }
 
 ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
-                                           ElementDofOrdering ordering,
+                                           ElementDofOrdering f_ordering,
                                            FaceType type,
                                            L2FaceValues m)
-   : ParL2FaceRestriction(fes, ordering, type, m, true)
+   : ParL2FaceRestriction(fes, f_ordering, type, m, true)
 { }
 
 void ParL2FaceRestriction::DoubleValuedConformingMult(
@@ -557,7 +561,7 @@ void ParL2FaceRestriction::FillJAndData(const Vector &ea_data,
 }
 
 void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -612,7 +616,7 @@ void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
 
 
 void ParL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -645,21 +649,21 @@ void ParL2FaceRestriction::ComputeGatherIndices(
 }
 
 ParNCL2FaceRestriction::ParNCL2FaceRestriction(const ParFiniteElementSpace &fes,
-                                               ElementDofOrdering ordering,
+                                               ElementDofOrdering f_ordering,
                                                FaceType type,
                                                L2FaceValues m)
-   : L2FaceRestriction(fes, ordering, type, m, false),
-     NCL2FaceRestriction(fes, ordering, type, m, false),
-     ParL2FaceRestriction(fes, ordering, type, m, false)
+   : L2FaceRestriction(fes, f_ordering, type, m, false),
+     NCL2FaceRestriction(fes, f_ordering, type, m, false),
+     ParL2FaceRestriction(fes, f_ordering, type, m, false)
 {
    if (nf==0) { return; }
    x_interp.UseDevice(true);
 
-   CheckFESpace(ordering);
+   CheckFESpace(f_ordering);
 
-   ComputeScatterIndicesAndOffsets(ordering, type);
+   ComputeScatterIndicesAndOffsets(f_ordering, type);
 
-   ComputeGatherIndices(ordering, type);
+   ComputeGatherIndices(f_ordering, type);
 }
 
 void ParNCL2FaceRestriction::SingleValuedNonconformingMult(
@@ -863,7 +867,7 @@ void ParNCL2FaceRestriction::FillJAndData(const Vector &ea_data,
 }
 
 void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -947,7 +951,7 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
 }
 
 void ParNCL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();

fem/prestriction.hpp

@@ -37,11 +37,11 @@ protected:
 public:
    /** @brief Constructs an ParNCH1FaceRestriction.
 
-       @param[in] fes      The ParFiniteElementSpace on which this operates
-       @param[in] ordering Request a specific ordering
-       @param[in] type     Request internal or boundary faces dofs */
+       @param[in] fes        The ParFiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs */
    ParNCH1FaceRestriction(const ParFiniteElementSpace &fes,
-                          ElementDofOrdering ordering,
+                          ElementDofOrdering f_ordering,
                           FaceType type);
 
    /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -90,20 +90,20 @@ private:
        for the gathering: E-vector to L-vector, and the interpolators from
        coarse to fine face for master non-comforming faces.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
                                         const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices(const ElementDofOrdering ordering,
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
                              const FaceType type);
 
 public: // For nvcc
@@ -141,16 +141,16 @@ class ParL2FaceRestriction : virtual public L2FaceRestriction
 protected:
    /** @brief Constructs an ParL2FaceRestriction.
 
-       @param[in] fes      The ParFiniteElementSpace on which this operates
-       @param[in] ordering Request a specific ordering
-       @param[in] type     Request internal or boundary faces dofs
-       @param[in] m        Request the face dofs for elem1, or both elem1 and
-                           elem2
-       @param[in] build    Request the ParL2FaceRestriction to compute the
-                           scatter/gather indices. False should only be used
-                           when inheriting from ParL2FaceRestriction. */
+       @param[in] fes        The ParFiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs
+       @param[in] m          Request the face dofs for elem1, or both elem1 and
+                             elem2
+       @param[in] build      Request the ParL2FaceRestriction to compute the
+                             scatter/gather indices. False should only be used
+                             when inheriting from ParL2FaceRestriction. */
    ParL2FaceRestriction(const ParFiniteElementSpace& fes,
-                        ElementDofOrdering ordering,
+                        ElementDofOrdering f_ordering,
                         FaceType type,
                         L2FaceValues m,
                         bool build);
@@ -158,13 +158,13 @@ protected:
 public:
    /** @brief Constructs an ParL2FaceRestriction.
 
-       @param[in] fes      The ParFiniteElementSpace on which this operates
-       @param[in] ordering Request a specific ordering
-       @param[in] type     Request internal or boundary faces dofs
-       @param[in] m        Request the face dofs for elem1, or both elem1 and
-                           elem2 */
+       @param[in] fes        The ParFiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs
+       @param[in] m          Request the face dofs for elem1, or both elem1 and
+                             elem2 */
    ParL2FaceRestriction(const ParFiniteElementSpace& fes,
-                        ElementDofOrdering ordering,
+                        ElementDofOrdering f_ordering,
                         FaceType type,
                         L2FaceValues m = L2FaceValues::DoubleValued);
 
@@ -230,20 +230,20 @@ private:
    /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
        for the gathering: E-vector to L-vector.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
                                         const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices(const ElementDofOrdering ordering,
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
                              const FaceType type);
 
 public:
@@ -271,13 +271,13 @@ class ParNCL2FaceRestriction
 public:
    /** @brief Constructs an ParNCL2FaceRestriction.
 
-       @param[in] fes      The ParFiniteElementSpace on which this operates
-       @param[in] ordering Request a specific ordering
-       @param[in] type     Request internal or boundary faces dofs
-       @param[in] m        Request the face dofs for elem1, or both elem1 and
-                           elem2 */
+       @param[in] fes        The ParFiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific ordering
+       @param[in] type       Request internal or boundary faces dofs
+       @param[in] m          Request the face dofs for elem1, or both elem1 and
+                             elem2 */
    ParNCL2FaceRestriction(const ParFiniteElementSpace& fes,
-                          ElementDofOrdering ordering,
+                          ElementDofOrdering f_ordering,
                           FaceType type,
                           L2FaceValues m = L2FaceValues::DoubleValued);
 
@@ -382,20 +382,20 @@ private:
        for the gathering: E-vector to L-vector, and the interpolators from
        coarse to fine face for master non-comforming faces.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
                                         const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices(const ElementDofOrdering ordering,
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
                              const FaceType type);
 
 public:

fem/restriction.cpp

@@ -598,119 +598,11 @@ void L2ElementRestriction::FillJAndData(const Vector &ea_data,
    });
 }
 
-/** Return the face degrees of freedom returned in Lexicographic order.
-    Note: Only for quad and hex */
-void GetFaceDofs(const int dim, const int face_id,
-                 const int dof1d, Array<int> &face_map)
-{
-   switch (dim)
-   {
-      case 1:
-         switch (face_id)
-         {
-            case 0: // WEST
-               face_map[0] = 0;
-               break;
-            case 1: // EAST
-               face_map[0] = dof1d-1;
-               break;
-         }
-         break;
-      case 2:
-         switch (face_id)
-         {
-            case 0: // SOUTH
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  face_map[i] = i;
-               }
-               break;
-            case 1: // EAST
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  face_map[i] = dof1d-1 + i*dof1d;
-               }
-               break;
-            case 2: // NORTH
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  face_map[i] = (dof1d-1)*dof1d + i;
-               }
-               break;
-            case 3: // WEST
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  face_map[i] = i*dof1d;
-               }
-               break;
-         }
-         break;
-      case 3:
-         switch (face_id)
-         {
-            case 0: // BOTTOM
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  for (int j = 0; j < dof1d; ++j)
-                  {
-                     face_map[i+j*dof1d] = i + j*dof1d;
-                  }
-               }
-               break;
-            case 1: // SOUTH
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  for (int j = 0; j < dof1d; ++j)
-                  {
-                     face_map[i+j*dof1d] = i + j*dof1d*dof1d;
-                  }
-               }
-               break;
-            case 2: // EAST
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  for (int j = 0; j < dof1d; ++j)
-                  {
-                     face_map[i+j*dof1d] = dof1d-1 + i*dof1d + j*dof1d*dof1d;
-                  }
-               }
-               break;
-            case 3: // NORTH
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  for (int j = 0; j < dof1d; ++j)
-                  {
-                     face_map[i+j*dof1d] = (dof1d-1)*dof1d + i + j*dof1d*dof1d;
-                  }
-               }
-               break;
-            case 4: // WEST
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  for (int j = 0; j < dof1d; ++j)
-                  {
-                     face_map[i+j*dof1d] = i*dof1d + j*dof1d*dof1d;
-                  }
-               }
-               break;
-            case 5: // TOP
-               for (int i = 0; i < dof1d; ++i)
-               {
-                  for (int j = 0; j < dof1d; ++j)
-                  {
-                     face_map[i+j*dof1d] = (dof1d-1)*dof1d*dof1d + i + j*dof1d;
-                  }
-               }
-               break;
-         }
-         break;
-   }
-}
-
-H1FaceRestriction::H1FaceRestriction(const FiniteElementSpace &fes,
-                                     const ElementDofOrdering e_ordering,
-                                     const FaceType type,
-                                     bool build)
+ConformingFaceRestriction::ConformingFaceRestriction(
+   const FiniteElementSpace &fes,
+   const ElementDofOrdering f_ordering,
+   const FaceType type,
+   bool build)
    : fes(fes),
      nf(fes.GetNFbyType(type)),
      vdim(fes.GetVDim()),
@@ -726,23 +618,40 @@ H1FaceRestriction::H1FaceRestriction(const FiniteElementSpace &fes,
 {
    height = vdim*nf*face_dofs;
    width = fes.GetVSize();
-   if (!build) { return; }
    if (nf==0) { return; }
 
-   CheckFESpace(e_ordering);
+   CheckFESpace(f_ordering);
 
-   ComputeScatterIndicesAndOffsets(e_ordering, type);
+   // Get the mapping from lexicographic DOF ordering to native ordering.
+   const TensorBasisElement* el =
+      dynamic_cast<const TensorBasisElement*>(fes.GetFE(0));
+   const Array<int> &dof_map_ = el->GetDofMap();
+   if (dof_map_.Size() > 0)
+   {
+      vol_dof_map.MakeRef(dof_map_);
+   }
+   else
+   {
+      // For certain types of elements dof_map_ is empty. In this case, that
+      // means the element is already ordered lexicographically, so the
+      // permutation is the identity.
+      vol_dof_map.SetSize(elem_dofs);
+      for (int i = 0; i < elem_dofs; ++i) { vol_dof_map[i] = i; }
+   }
 
-   ComputeGatherIndices(e_ordering,type);
+   if (!build) { return; }
+   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeGatherIndices(f_ordering,type);
 }
 
-H1FaceRestriction::H1FaceRestriction(const FiniteElementSpace &fes,
-                                     const ElementDofOrdering e_ordering,
-                                     const FaceType type)
-   : H1FaceRestriction(fes, e_ordering, type, true)
+ConformingFaceRestriction::ConformingFaceRestriction(
+   const FiniteElementSpace &fes,
+   const ElementDofOrdering f_ordering,
+   const FaceType type)
+   : ConformingFaceRestriction(fes, f_ordering, type, true)
 { }
 
-void H1FaceRestriction::Mult(const Vector& x, Vector& y) const
+void ConformingFaceRestriction::Mult(const Vector& x, Vector& y) const
 {
    if (nf==0) { return; }
    // Assumes all elements have the same number of dofs
@@ -754,18 +663,20 @@ void H1FaceRestriction::Mult(const Vector& x, Vector& y) const
    auto d_y = Reshape(y.Write(), nface_dofs, vd, nf);
    MFEM_FORALL(i, nfdofs,
    {
-      const int idx = d_indices[i];
+      const int s_idx = d_indices[i];
+      const int sgn = (s_idx >= 0) ? 1 : -1;
+      const int idx = (s_idx >= 0) ? s_idx : -1 - s_idx;
       const int dof = i % nface_dofs;
       const int face = i / nface_dofs;
       for (int c = 0; c < vd; ++c)
       {
-         d_y(dof, c, face) = d_x(t?c:idx, t?idx:c);
+         d_y(dof, c, face) = sgn*d_x(t?c:idx, t?idx:c);
       }
    });
 }
 
-void H1FaceRestriction::AddMultTranspose(const Vector& x, Vector& y,
-                                         const double a) const
+void ConformingFaceRestriction::AddMultTranspose(
+   const Vector& x, Vector& y, const double a) const
 {
    MFEM_VERIFY(a == 1.0, "General coefficient case is not yet supported!");
    if (nf==0) { return; }
@@ -786,15 +697,18 @@ void H1FaceRestriction::AddMultTranspose(const Vector& x, Vector& y,
          double dof_value = 0;
          for (int j = offset; j < next_offset; ++j)
          {
-            const int idx_j = d_indices[j];
-            dof_value +=  d_x(idx_j % nface_dofs, c, idx_j / nface_dofs);
+            const int s_idx_j = d_indices[j];
+            const int sgn = (s_idx_j >= 0) ? 1 : -1;
+            const int idx_j = (s_idx_j >= 0) ? s_idx_j : -1 - s_idx_j;
+            dof_value += sgn*d_x(idx_j % nface_dofs, c, idx_j / nface_dofs);
          }
          d_y(t?c:i,t?i:c) += dof_value;
       }
    });
 }
 
-void H1FaceRestriction::CheckFESpace(const ElementDofOrdering e_ordering)
+void ConformingFaceRestriction::CheckFESpace(const ElementDofOrdering
+                                             f_ordering)
 {
 #ifdef MFEM_USE_MPI
 
@@ -809,17 +723,16 @@ void H1FaceRestriction::CheckFESpace(const ElementDofOrdering e_ordering)
 #endif
 
 #ifdef MFEM_DEBUG
-   // If fespace == H1
    const FiniteElement *fe0 = fes.GetFE(0);
    const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe0);
    MFEM_VERIFY(tfe != NULL &&
                (tfe->GetBasisType()==BasisType::GaussLobatto ||
                 tfe->GetBasisType()==BasisType::Positive),
                "Only Gauss-Lobatto and Bernstein basis are supported in "
-               "H1FaceRestriction.");
+               "ConformingFaceRestriction.");
 
    // Assuming all finite elements are using Gauss-Lobatto.
-   const bool dof_reorder = (e_ordering == ElementDofOrdering::LEXICOGRAPHIC);
+   const bool dof_reorder = (f_ordering == ElementDofOrdering::LEXICOGRAPHIC);
    if (dof_reorder && nf > 0)
    {
       for (int f = 0; f < fes.GetNF(); ++f)
@@ -830,17 +743,12 @@ void H1FaceRestriction::CheckFESpace(const ElementDofOrdering e_ordering)
          if (el) { continue; }
          MFEM_ABORT("Finite element not suitable for lexicographic ordering");
       }
-      const FiniteElement *fe = fes.GetFaceElement(0);
-      const TensorBasisElement* el =
-         dynamic_cast<const TensorBasisElement*>(fe);
-      const Array<int> &fe_dof_map = el->GetDofMap();
-      MFEM_VERIFY(fe_dof_map.Size() > 0, "invalid dof map");
    }
 #endif
 }
 
-void H1FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering ordering,
+void ConformingFaceRestriction::ComputeScatterIndicesAndOffsets(
+   const ElementDofOrdering f_ordering,
    const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -864,7 +772,7 @@ void H1FaceRestriction::ComputeScatterIndicesAndOffsets(
       }
       else if ( face.IsOfFaceType(type) )
       {
-         SetFaceDofsScatterIndices(face, f_ind, ordering);
+         SetFaceDofsScatterIndices(face, f_ind, f_ordering);
          f_ind++;
       }
    }
@@ -877,8 +785,8 @@ void H1FaceRestriction::ComputeScatterIndicesAndOffsets(
    }
 }
 
-void H1FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering ordering,
+void ConformingFaceRestriction::ComputeGatherIndices(
+   const ElementDofOrdering f_ordering,
    const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -896,7 +804,7 @@ void H1FaceRestriction::ComputeGatherIndices(
       }
       else if ( face.IsOfFaceType(type) )
       {
-         SetFaceDofsGatherIndices(face, f_ind, ordering);
+         SetFaceDofsGatherIndices(face, f_ind, f_ordering);
          f_ind++;
       }
    }
@@ -910,68 +818,65 @@ void H1FaceRestriction::ComputeGatherIndices(
    gather_offsets[0] = 0;
 }
 
-void H1FaceRestriction::SetFaceDofsScatterIndices(
+static inline int absdof(int i) { return i < 0 ? -1-i : i; }
+
+void ConformingFaceRestriction::SetFaceDofsScatterIndices(
    const Mesh::FaceInformation &face,
    const int face_index,
-   const ElementDofOrdering ordering)
+   const ElementDofOrdering f_ordering)
 {
    MFEM_ASSERT(!(face.IsNonconformingCoarse()),
                "This method should not be used on nonconforming coarse faces.");
    MFEM_ASSERT(face.element[0].orientation==0,
                "FaceRestriction used on degenerated mesh.");
+   MFEM_CONTRACT_VAR(f_ordering); // not supported yet
+
+   fes.GetFE(0)->GetFaceMap(face.element[0].local_face_id, face_map);
 
-   const TensorBasisElement* el =
-      dynamic_cast<const TensorBasisElement*>(fes.GetFE(0));
-   const int *dof_map = el->GetDofMap().GetData();
    const Table& e2dTable = fes.GetElementToDofTable();
    const int* elem_map = e2dTable.GetJ();
-   const int face_id = face.element[0].local_face_id;
-   const int dim = fes.GetMesh()->Dimension();
-   const int dof1d = fes.GetFE(0)->GetOrder()+1;
    const int elem_index = face.element[0].index;
-   const bool dof_reorder = (ordering == ElementDofOrdering::LEXICOGRAPHIC);
-   GetFaceDofs(dim, face_id, dof1d, face_map); // Only for quad and hex
 
    for (int face_dof = 0; face_dof < face_dofs; ++face_dof)
    {
-      const int nat_volume_dof = face_map[face_dof];
-      const int volume_dof = (!dof_reorder)?
-                             nat_volume_dof:
-                             dof_map[nat_volume_dof];
-      const int global_dof = elem_map[elem_index*elem_dofs + volume_dof];
+      const int lex_volume_dof = face_map[face_dof];
+      const int s_volume_dof = vol_dof_map[lex_volume_dof]; // signed
+      const int volume_dof = absdof(s_volume_dof);
+      const int s_global_dof = elem_map[elem_index*elem_dofs + volume_dof];
+      const int global_dof = absdof(s_global_dof);
       const int restriction_dof = face_dofs*face_index + face_dof;
-      scatter_indices[restriction_dof] = global_dof;
+      scatter_indices[restriction_dof] = s_global_dof;
       ++gather_offsets[global_dof + 1];
    }
 }
 
-void H1FaceRestriction::SetFaceDofsGatherIndices(
+void ConformingFaceRestriction::SetFaceDofsGatherIndices(
    const Mesh::FaceInformation &face,
    const int face_index,
-   const ElementDofOrdering ordering)
+   const ElementDofOrdering f_ordering)
 {
    MFEM_ASSERT(!(face.IsNonconformingCoarse()),
                "This method should not be used on nonconforming coarse faces.");
+   MFEM_CONTRACT_VAR(f_ordering); // not supported yet
+
+   fes.GetFE(0)->GetFaceMap(face.element[0].local_face_id, face_map);
 
-   const TensorBasisElement* el =
-      dynamic_cast<const TensorBasisElement*>(fes.GetFE(0));
-   const int *dof_map = el->GetDofMap().GetData();
    const Table& e2dTable = fes.GetElementToDofTable();
    const int* elem_map = e2dTable.GetJ();
-   const int face_id = face.element[0].local_face_id;
-   const int dim = fes.GetMesh()->Dimension();
-   const int dof1d = fes.GetFE(0)->GetOrder()+1;
    const int elem_index = face.element[0].index;
-   const bool dof_reorder = (ordering == ElementDofOrdering::LEXICOGRAPHIC);
-   GetFaceDofs(dim, face_id, dof1d, face_map); // Only for quad and hex
 
    for (int face_dof = 0; face_dof < face_dofs; ++face_dof)
    {
-      const int nat_volume_dof = face_map[face_dof];
-      const int volume_dof = (!dof_reorder)?nat_volume_dof:dof_map[nat_volume_dof];
-      const int global_dof = elem_map[elem_index*elem_dofs + volume_dof];
+      const int lex_volume_dof = face_map[face_dof];
+      const int s_volume_dof = vol_dof_map[lex_volume_dof];
+      const int volume_dof = absdof(s_volume_dof);
+      const int s_global_dof = elem_map[elem_index*elem_dofs + volume_dof];
+      const int sgn = (s_global_dof >= 0) ? 1 : -1;
+      const int global_dof = absdof(s_global_dof);
       const int restriction_dof = face_dofs*face_index + face_dof;
-      gather_indices[gather_offsets[global_dof]++] = restriction_dof;
+      const int s_restriction_dof = (sgn >= 0) ? restriction_dof : -1 -
+                                    restriction_dof;
+      gather_indices[gather_offsets[global_dof]++] = s_restriction_dof;
    }
 }
 
@@ -1101,7 +1006,7 @@ int PermuteFaceL2(const int dim, const int face_id1,
 }
 
 L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
-                                     const ElementDofOrdering e_ordering,
+                                     const ElementDofOrdering f_ordering,
                                      const FaceType type,
                                      const L2FaceValues m,
                                      bool build)
@@ -1128,18 +1033,18 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
    width = fes.GetVSize();
    if (!build) { return; }
 
-   CheckFESpace(e_ordering);
+   CheckFESpace(f_ordering);
 
-   ComputeScatterIndicesAndOffsets(e_ordering,type);
+   ComputeScatterIndicesAndOffsets(f_ordering,type);
 
-   ComputeGatherIndices(e_ordering, type);
+   ComputeGatherIndices(f_ordering, type);
 }
 
 L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
-                                     const ElementDofOrdering e_ordering,
+                                     const ElementDofOrdering f_ordering,
                                      const FaceType type,
                                      const L2FaceValues m)
-   : L2FaceRestriction(fes, e_ordering, type, m, true)
+   : L2FaceRestriction(fes, f_ordering, type, m, true)
 { }
 
 void L2FaceRestriction::SingleValuedConformingMult(const Vector& x,
@@ -1394,7 +1299,7 @@ void L2FaceRestriction::AddFaceMatricesToElementMatrices(const Vector &fea_data,
    }
 }
 
-void L2FaceRestriction::CheckFESpace(const ElementDofOrdering e_ordering)
+void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
 {
 #ifdef MFEM_USE_MPI
 
@@ -1418,7 +1323,7 @@ void L2FaceRestriction::CheckFESpace(const ElementDofOrdering e_ordering)
                "Only Gauss-Lobatto and Bernstein basis are supported in "
                "L2FaceRestriction.");
    if (nf==0) { return; }
-   const bool dof_reorder = (e_ordering == ElementDofOrdering::LEXICOGRAPHIC);
+   const bool dof_reorder = (f_ordering == ElementDofOrdering::LEXICOGRAPHIC);
    if (!dof_reorder)
    {
       MFEM_ABORT("Non-Tensor L2FaceRestriction not yet implemented.");
@@ -1439,7 +1344,7 @@ void L2FaceRestriction::CheckFESpace(const ElementDofOrdering e_ordering)
 }
 
 void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType face_type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -1483,7 +1388,7 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
 }
 
 void L2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType face_type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -1525,10 +1430,8 @@ void L2FaceRestriction::SetFaceDofsScatterIndices1(
    const Table& e2dTable = fes.GetElementToDofTable();
    const int* elem_map = e2dTable.GetJ();
    const int face_id1 = face.element[0].local_face_id;
-   const int dim = fes.GetMesh()->Dimension();
-   const int dof1d = fes.GetFE(0)->GetOrder()+1;
    const int elem_index = face.element[0].index;
-   GetFaceDofs(dim, face_id1, dof1d, face_map); // Only for quad and hex
+   fes.GetFE(0)->GetFaceMap(face_id1, face_map);
 
    for (int face_dof_elem1 = 0; face_dof_elem1 < face_dofs; ++face_dof_elem1)
    {
@@ -1554,7 +1457,7 @@ void L2FaceRestriction::PermuteAndSetFaceDofsScatterIndices2(
    const int orientation = face.element[1].orientation;
    const int dim = fes.GetMesh()->Dimension();
    const int dof1d = fes.GetFE(0)->GetOrder()+1;
-   GetFaceDofs(dim, face_id2, dof1d, face_map); // Only for quad and hex
+   fes.GetFE(0)->GetFaceMap(face_id2, face_map);
 
    for (int face_dof_elem1 = 0; face_dof_elem1 < face_dofs; ++face_dof_elem1)
    {
@@ -1582,7 +1485,7 @@ void L2FaceRestriction::PermuteAndSetSharedFaceDofsScatterIndices2(
    const int orientation = face.element[1].orientation;
    const int dim = fes.GetMesh()->Dimension();
    const int dof1d = fes.GetFE(0)->GetOrder()+1;
-   GetFaceDofs(dim, face_id2, dof1d, face_map); // Only for quad and hex
+   fes.GetFE(0)->GetFaceMap(face_id2, face_map);
    Array<int> face_nbr_dofs;
    const ParFiniteElementSpace &pfes =
       static_cast<const ParFiniteElementSpace&>(this->fes);
@@ -1624,10 +1527,8 @@ void L2FaceRestriction::SetFaceDofsGatherIndices1(
    const Table& e2dTable = fes.GetElementToDofTable();
    const int* elem_map = e2dTable.GetJ();
    const int face_id1 = face.element[0].local_face_id;
-   const int dim = fes.GetMesh()->Dimension();
-   const int dof1d = fes.GetFE(0)->GetOrder()+1;
    const int elem_index = face.element[0].index;
-   GetFaceDofs(dim, face_id1, dof1d, face_map); // Only for quad and hex
+   fes.GetFE(0)->GetFaceMap(face_id1, face_map);
 
    for (int face_dof_elem1 = 0; face_dof_elem1 < face_dofs; ++face_dof_elem1)
    {
@@ -1653,7 +1554,7 @@ void L2FaceRestriction::PermuteAndSetFaceDofsGatherIndices2(
    const int orientation = face.element[1].orientation;
    const int dim = fes.GetMesh()->Dimension();
    const int dof1d = fes.GetFE(0)->GetOrder()+1;
-   GetFaceDofs(dim, face_id2, dof1d, face_map); // Only for quad and hex
+   fes.GetFE(0)->GetFaceMap(face_id2, face_map);
 
    for (int face_dof_elem1 = 0; face_dof_elem1 < face_dofs; ++face_dof_elem1)
    {
@@ -1836,28 +1737,28 @@ void InterpolationManager::InitializeNCInterpConfig()
 }
 
 NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
-                                         const ElementDofOrdering ordering,
+                                         const ElementDofOrdering f_ordering,
                                          const FaceType type,
                                          const L2FaceValues m,
                                          bool build)
-   : L2FaceRestriction(fes, ordering, type, m, false),
-     interpolations(fes, ordering, type)
+   : L2FaceRestriction(fes, f_ordering, type, m, false),
+     interpolations(fes, f_ordering, type)
 {
    if (!build) { return; }
    x_interp.UseDevice(true);
 
-   CheckFESpace(ordering);
+   CheckFESpace(f_ordering);
 
-   ComputeScatterIndicesAndOffsets(ordering, type);
+   ComputeScatterIndicesAndOffsets(f_ordering, type);
 
-   ComputeGatherIndices(ordering, type);
+   ComputeGatherIndices(f_ordering, type);
 }
 
 NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
-                                         const ElementDofOrdering ordering,
+                                         const ElementDofOrdering f_ordering,
                                          const FaceType type,
                                          const L2FaceValues m)
-   : NCL2FaceRestriction(fes, ordering, type, m, true)
+   : NCL2FaceRestriction(fes, f_ordering, type, m, true)
 { }
 
 void NCL2FaceRestriction::DoubleValuedNonconformingMult(
@@ -2157,7 +2058,7 @@ int ToLexOrdering(const int dim, const int face_id, const int size1d,
 }
 
 void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
@@ -2222,7 +2123,7 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
 }
 
 void NCL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering ordering,
+   const ElementDofOrdering f_ordering,
    const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();

fem/restriction.hpp

@@ -218,10 +218,12 @@ public:
    }
 };
 
-/// Operator that extracts Face degrees of freedom for H1 FiniteElementSpaces.
-/** Objects of this type are typically created and owned by FiniteElementSpace
-    objects, see FiniteElementSpace::GetFaceRestriction(). */
-class H1FaceRestriction : public FaceRestriction
+/// @brief Operator that extracts face degrees of freedom for H1, ND, or RT
+/// FiniteElementSpaces.
+///
+/// Objects of this type are typically created and owned by FiniteElementSpace
+/// objects, see FiniteElementSpace::GetFaceRestriction().
+class ConformingFaceRestriction : public FaceRestriction
 {
 protected:
    const FiniteElementSpace &fes;
@@ -235,29 +237,30 @@ protected:
    Array<int> scatter_indices; // Scattering indices for element 1 on each face
    Array<int> gather_offsets; // offsets for the gathering indices of each dof
    Array<int> gather_indices; // gathering indices for each dof
+   Array<int> vol_dof_map; // mapping from lexicographic to native ordering
 
-   /** @brief Construct an H1FaceRestriction.
+   /** @brief Construct a ConformingFaceRestriction.
 
-       @param[in] fes      The FiniteElementSpace on which this operates
-       @param[in] ordering Request a specific element ordering
-       @param[in] type     Request internal or boundary faces dofs
-       @param[in] build    Request the NCL2FaceRestriction to compute the
-                           scatter/gather indices. False should only be used
-                           when inheriting from H1FaceRestriction.
+       @param[in] fes        The FiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs
+       @param[in] build      Request the NCL2FaceRestriction to compute the
+                             scatter/gather indices. False should only be used
+                             when inheriting from ConformingFaceRestriction.
    */
-   H1FaceRestriction(const FiniteElementSpace& fes,
-                     const ElementDofOrdering ordering,
-                     const FaceType type,
-                     bool build);
+   ConformingFaceRestriction(const FiniteElementSpace& fes,
+                             const ElementDofOrdering f_ordering,
+                             const FaceType type,
+                             bool build);
 public:
-   /** @brief Construct an H1FaceRestriction.
+   /** @brief Construct a ConformingFaceRestriction.
 
-       @param[in] fes      The FiniteElementSpace on which this operates
-       @param[in] ordering Request a specific element ordering
-       @param[in] type     Request internal or boundary faces dofs */
-   H1FaceRestriction(const FiniteElementSpace& fes,
-                     const ElementDofOrdering ordering,
-                     const FaceType type);
+       @param[in] fes        The FiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs */
+   ConformingFaceRestriction(const FiniteElementSpace& fes,
+                             const ElementDofOrdering f_ordering,
+                             const FaceType type);
 
    /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
        face E-Vector.
@@ -291,54 +294,59 @@ private:
    /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
        for the gathering: E-vector to L-vector.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
                                         const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices(const ElementDofOrdering ordering,
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
                              const FaceType type);
 
 protected:
    mutable Array<int> face_map; // Used in the computation of GetFaceDofs
 
-   /** @brief Verify that H1FaceRestriction is build from an H1 FESpace.
+   /** @brief Verify that ConformingFaceRestriction is built from a supported
+       finite element space.
 
-       @param[in] ordering The FESpace element ordering.
+       @param[in] f_ordering The requested face dof ordering.
    */
-   void CheckFESpace(const ElementDofOrdering ordering);
+   void CheckFESpace(const ElementDofOrdering f_ordering);
 
    /** @brief Set the scattering indices of elem1, and increment the offsets for
        the face described by the @a face.
 
-       @param[in] face The face information of the current face.
-       @param[in] face_index The interior/boundary face index.
-       @param[in] ordering Request a specific element ordering.
+       @param[in] face        The face information of the current face.
+       @param[in] face_index  The interior/boundary face index.
+       @param[in] f_ordering  Request a specific face dof ordering.
     */
    void SetFaceDofsScatterIndices(const Mesh::FaceInformation &face,
                                   const int face_index,
-                                  const ElementDofOrdering ordering);
+                                  const ElementDofOrdering f_ordering);
 
    /** @brief Set the gathering indices of elem1 for the interior face described
        by the @a face.
 
-       @param[in] face The face information of the current face.
-       @param[in] face_index The interior/boundary face index.
-       @param[in] ordering Request a specific element ordering.
+       @param[in] face        The face information of the current face.
+       @param[in] face_index  The interior/boundary face index.
+       @param[in] f_ordering  Request a specific face dof ordering.
     */
    void SetFaceDofsGatherIndices(const Mesh::FaceInformation &face,
                                  const int face_index,
-                                 const ElementDofOrdering ordering);
+                                 const ElementDofOrdering f_ordering);
 };
 
+/// @brief Alias for ConformingFaceRestriction, for backwards compatibility and
+/// as base class for ParNCH1FaceRestriction.
+using H1FaceRestriction = ConformingFaceRestriction;
+
 /// Operator that extracts Face degrees of freedom for L2 spaces.
 /** Objects of this type are typically created and owned by FiniteElementSpace
     objects, see FiniteElementSpace::GetFaceRestriction(). */
@@ -363,17 +371,17 @@ protected:
 
    /** @brief Constructs an L2FaceRestriction.
 
-       @param[in] fes      The FiniteElementSpace on which this operates
-       @param[in] ordering Request a specific ordering
-       @param[in] type     Request internal or boundary faces dofs
-       @param[in] m        Request the face dofs for elem1, or both elem1 and
-                           elem2
-       @param[in] build    Request the NCL2FaceRestriction to compute the
-                           scatter/gather indices. False should only be used
-                           when inheriting from L2FaceRestriction.
+       @param[in] fes        The FiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs
+       @param[in] m          Request the face dofs for elem1, or both elem1 and
+                             elem2
+       @param[in] build      Request the NCL2FaceRestriction to compute the
+                             scatter/gather indices. False should only be used
+                             when inheriting from L2FaceRestriction.
    */
    L2FaceRestriction(const FiniteElementSpace& fes,
-                     const ElementDofOrdering ordering,
+                     const ElementDofOrdering f_ordering,
                      const FaceType type,
                      const L2FaceValues m,
                      bool build);
@@ -381,13 +389,13 @@ protected:
 public:
    /** @brief Constructs an L2FaceRestriction.
 
-       @param[in] fes      The FiniteElementSpace on which this operates
-       @param[in] ordering Request a specific ordering
-       @param[in] type     Request internal or boundary faces dofs
-       @param[in] m        Request the face dofs for elem1, or both elem1 and
-                           elem2 */
+       @param[in] fes        The FiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs
+       @param[in] m          Request the face dofs for elem1, or both elem1 and
+                             elem2 */
    L2FaceRestriction(const FiniteElementSpace& fes,
-                     const ElementDofOrdering ordering,
+                     const ElementDofOrdering f_ordering,
                      const FaceType type,
                      const L2FaceValues m = L2FaceValues::DoubleValued);
 
@@ -472,30 +480,30 @@ private:
    /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
        for the gathering: E-vector to L-vector.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
                                         const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices(const ElementDofOrdering ordering,
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
                              const FaceType type);
 
 protected:
    mutable Array<int> face_map; // Used in the computation of GetFaceDofs
 
-   /** @brief Verify that L2FaceRestriction is build from an L2 FESpace.
+   /** @brief Verify that L2FaceRestriction is built from an L2 FESpace.
 
-       @param[in] ordering The FESpace element ordering.
+       @param[in] f_ordering The requested face dof ordering.
    */
-   void CheckFESpace(const ElementDofOrdering ordering);
+   void CheckFESpace(const ElementDofOrdering f_ordering);
 
    /** @brief Set the scattering indices of elem1, and increment the offsets for
        the face described by the @a face. The ordering of the face dofs of elem1
@@ -788,17 +796,17 @@ protected:
    /** @brief Constructs an NCL2FaceRestriction, this is a specialization of a
        L2FaceRestriction for nonconforming meshes.
 
-       @param[in] fes      The FiniteElementSpace on which this operates
-       @param[in] ordering Request a specific ordering
-       @param[in] type     Request internal or boundary faces dofs
-       @param[in] m        Request the face dofs for elem1, or both elem1 and
-                           elem2
-       @param[in] build    Request the NCL2FaceRestriction to compute the
-                           scatter/gather indices. False should only be used
-                           when inheriting from NCL2FaceRestriction.
+       @param[in] fes        The FiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs
+       @param[in] m          Request the face dofs for elem1, or both elem1 and
+                             elem2
+       @param[in] build      Request the NCL2FaceRestriction to compute the
+                             scatter/gather indices. False should only be used
+                             when inheriting from NCL2FaceRestriction.
    */
    NCL2FaceRestriction(const FiniteElementSpace& fes,
-                       const ElementDofOrdering ordering,
+                       const ElementDofOrdering f_ordering,
                        const FaceType type,
                        const L2FaceValues m,
                        bool build);
@@ -806,14 +814,14 @@ public:
    /** @brief Constructs an NCL2FaceRestriction, this is a specialization of a
        L2FaceRestriction for nonconforming meshes.
 
-       @param[in] fes      The FiniteElementSpace on which this operates
-       @param[in] ordering Request a specific ordering
-       @param[in] type     Request internal or boundary faces dofs
-       @param[in] m        Request the face dofs for elem1, or both elem1 and
-                           elem2
+       @param[in] fes        The FiniteElementSpace on which this operates
+       @param[in] f_ordering Request a specific face dof ordering
+       @param[in] type       Request internal or boundary faces dofs
+       @param[in] m          Request the face dofs for elem1, or both elem1 and
+                             elem2
    */
    NCL2FaceRestriction(const FiniteElementSpace& fes,
-                       const ElementDofOrdering ordering,
+                       const ElementDofOrdering f_ordering,
                        const FaceType type,
                        const L2FaceValues m = L2FaceValues::DoubleValued);
 
@@ -920,20 +928,20 @@ private:
        for the gathering: E-vector to L-vector, and the interpolators from
        coarse to fine face for master non-comforming faces.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering ordering,
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
                                         const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
 
-       @param[in] ordering Request a specific element ordering.
-       @param[in] type     Request internal or boundary faces dofs.
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices(const ElementDofOrdering ordering,
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
                              const FaceType type);
 
 public:
@@ -996,16 +1004,6 @@ public:
    void DoubleValuedNonconformingTransposeInterpolationInPlace(Vector& x) const;
 };
 
-/** @brief Return the face map that extracts the degrees of freedom for the
-    requested local face of a quad or hex, returned in Lexicographic order.
-
-    @param[in] dim The dimension of the space
-    @param[in] face_id The local face identifier
-    @param[in] dof1d The 1D number of degrees of freedom for each dimension
-    @param[out] face_map The map that maps each face dof to an element dof
-*/
-void GetFaceDofs(const int dim, const int face_id,
-                 const int dof1d, Array<int> &face_map);
 
 /** @brief Convert a dof face index from Native ordering to lexicographic
     ordering for quads and hexes.

fem/tmop.cpp

@@ -2776,6 +2776,8 @@ TMOP_Integrator::~TMOP_Integrator()
    delete lim_func;
    delete adapt_lim_gf;
    delete surf_fit_gf;
+   delete surf_fit_grad;
+   delete surf_fit_hess;
    for (int i = 0; i < ElemDer.Size(); i++)
    {
       delete ElemDer[i];
@@ -2850,6 +2852,7 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &s0,
 {
    delete surf_fit_gf;
    surf_fit_gf = new GridFunction(s0);
+   surf_fit_gf->CountElementsPerVDof(surf_fit_dof_count);
    surf_fit_marker = &smarker;
    surf_fit_coeff = &coeff;
    surf_fit_eval = &ae;
@@ -2868,6 +2871,7 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
 {
    delete surf_fit_gf;
    surf_fit_gf = new GridFunction(s0);
+   s0.CountElementsPerVDof(surf_fit_dof_count);
    surf_fit_marker = &smarker;
    surf_fit_coeff = &coeff;
    surf_fit_eval = &ae;
@@ -2876,6 +2880,75 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
                                  *s0.ParFESpace());
    surf_fit_eval->SetInitialField
    (*surf_fit_gf->FESpace()->GetMesh()->GetNodes(), *surf_fit_gf);
+   surf_fit_gf_bg = false;
+}
+
+void TMOP_Integrator::EnableSurfaceFittingFromSource(
+   const ParGridFunction &s_bg, ParGridFunction &s0,
+   const Array<bool> &smarker, Coefficient &coeff, AdaptivityEvaluator &ae,
+   const ParGridFunction &s_bg_grad,
+   ParGridFunction &s0_grad, AdaptivityEvaluator &age,
+   const ParGridFunction &s_bg_hess,
+   ParGridFunction &s0_hess, AdaptivityEvaluator &ahe)
+{
+#ifndef MFEM_USE_GSLIB
+   MFEM_ABORT("Surface fitting from source requires GSLIB!");
+#endif
+
+   // Setup for level set function
+   delete surf_fit_gf;
+   surf_fit_gf = new GridFunction(s0);
+   *surf_fit_gf = 0.0;
+   surf_fit_marker = &smarker;
+   surf_fit_coeff = &coeff;
+   surf_fit_eval = &ae;
+
+   surf_fit_gf_bg = true;
+   surf_fit_eval->SetParMetaInfo(*s_bg.ParFESpace()->GetParMesh(),
+                                 *s_bg.ParFESpace());
+   surf_fit_eval->SetInitialField
+   (*s_bg.FESpace()->GetMesh()->GetNodes(), s_bg);
+
+   // Setup for gradient on background mesh
+   MFEM_VERIFY(s_bg_grad.ParFESpace()->GetOrdering() ==
+               s0_grad.ParFESpace()->GetOrdering(),
+               "Nodal ordering for gridfunction on source mesh and current mesh"
+               "should be the same.");
+   delete surf_fit_grad;
+   surf_fit_grad = new GridFunction(s0_grad);
+   *surf_fit_grad = 0.0;
+   surf_fit_eval_bg_grad = &age;
+   surf_fit_eval_bg_hess = &ahe;
+   surf_fit_eval_bg_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
+                                         *s_bg_grad.ParFESpace());
+   surf_fit_eval_bg_grad->SetInitialField
+   (*s_bg_grad.FESpace()->GetMesh()->GetNodes(), s_bg_grad);
+
+   // Setup for Hessian on background mesh
+   MFEM_VERIFY(s_bg_hess.ParFESpace()->GetOrdering() ==
+               s0_hess.ParFESpace()->GetOrdering(),
+               "Nodal ordering for gridfunction on source mesh and current mesh"
+               "should be the same.");
+   delete surf_fit_hess;
+   surf_fit_hess = new GridFunction(s0_hess);
+   *surf_fit_hess = 0.0;
+   surf_fit_eval_bg_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
+                                         *s_bg_hess.ParFESpace());
+   surf_fit_eval_bg_hess->SetInitialField
+   (*s_bg_hess.FESpace()->GetMesh()->GetNodes(), s_bg_hess);
+
+   // Count number of zones that share each of the DOFs
+   s0.CountElementsPerVDof(surf_fit_dof_count);
+   // Store DOF indices that are marked for fitting. Used to reduce work for
+   // transferring information between source/background and current mesh.
+   surf_fit_marker_dof_index.SetSize(0);
+   for (int i = 0; i < surf_fit_marker->Size(); i++)
+   {
+      if ((*surf_fit_marker)[i] == true)
+      {
+         surf_fit_marker_dof_index.Append(i);
+      }
+   }
 }
 #endif
 
@@ -3601,51 +3674,52 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,
 {
    const int el_id = Tpr.ElementNo;
    // Check if the element has any DOFs marked for surface fitting.
-   Array<int> dofs;
-   surf_fit_gf->FESpace()->GetElementDofs(el_id, dofs);
+   Array<int> sdofs, dofs;
+   surf_fit_gf->FESpace()->GetElementDofs(el_id, sdofs);
    int count = 0;
-   for (int s = 0; s < dofs.Size(); s++)
+   for (int s = 0; s < sdofs.Size(); s++)
    {
-      count += ((*surf_fit_marker)[dofs[s]]) ? 1 : 0;
+      count += ((*surf_fit_marker)[sdofs[s]]) ? 1 : 0;
    }
    if (count == 0) { return; }
 
    const FiniteElement &el_s = *surf_fit_gf->FESpace()->GetFE(el_id);
 
-   const int dof_x = el_x.GetDof(), dim = el_x.GetDim(),
-             dof_s = el_s.GetDof();
+   const int dof_s = el_s.GetDof(), dim = el_x.GetDim();
 
    Vector sigma_e;
-   surf_fit_gf->GetSubVector(dofs, sigma_e);
+   surf_fit_gf->GetSubVector(sdofs, sigma_e);
 
    // Project the gradient of sigma in the same space.
    // The FE coefficients of the gradient go in surf_fit_grad_e.
    DenseMatrix surf_fit_grad_e(dof_s, dim);
    Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
    DenseMatrix grad_phys; // This will be (dof x dim, dof).
-   el_s.ProjectGrad(el_s, Tpr, grad_phys);
-   grad_phys.Mult(sigma_e, grad_ptr);
+   if (surf_fit_gf_bg)
+   {
+      surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
+      surf_fit_grad->GetSubVector(dofs, grad_ptr);
+   }
+   else
+   {
+      el_s.ProjectGrad(el_s, Tpr, grad_phys);
+      grad_phys.Mult(sigma_e, grad_ptr);
+   }
 
-   Vector shape_x(dof_x), shape_s(dof_s);
    const IntegrationRule &ir = el_s.GetNodes();
-   Vector surf_fit_grad_s(dim);
-   surf_fit_grad_s = 0.0;
-
    for (int s = 0; s < dof_s; s++)
    {
-      if ((*surf_fit_marker)[dofs[s]] == false) { continue; }
+      if ((*surf_fit_marker)[sdofs[s]] == false) { continue; }
 
       const IntegrationPoint &ip = ir.IntPoint(s);
       Tpr.SetIntPoint(&ip);
-      el_x.CalcShape(ip, shape_x);
-      el_s.CalcShape(ip, shape_s);
-
-      // Note that this gradient is already in physical space.
-      surf_fit_grad_e.MultTranspose(shape_s, surf_fit_grad_s);
-      surf_fit_grad_s *= 2.0 * surf_fit_normal *
-                         surf_fit_coeff->Eval(Tpr, ip) * sigma_e(s);
-
-      AddMultVWt(shape_x, surf_fit_grad_s, mat);
+      const double w = 2.0 * surf_fit_normal *
+                       surf_fit_coeff->Eval(Tpr, ip) * sigma_e(s) *
+                       1.0/surf_fit_dof_count[sdofs[s]];
+      for (int d = 0; d < dim; d++)
+      {
+         mat(s, d) += w * surf_fit_grad_e(s, d);
+      }
    }
 }
 
@@ -3655,71 +3729,80 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
 {
    const int el_id = Tpr.ElementNo;
    // Check if the element has any DOFs marked for surface fitting.
-   Array<int> dofs;
-   surf_fit_gf->FESpace()->GetElementDofs(el_id, dofs);
+   Array<int> dofs, sdofs;
+   surf_fit_gf->FESpace()->GetElementDofs(el_id, sdofs);
+   int ndofs = sdofs.Size();
    int count = 0;
-   for (int s = 0; s < dofs.Size(); s++)
+   for (int s = 0; s < ndofs; s++)
    {
-      count += ((*surf_fit_marker)[dofs[s]]) ? 1 : 0;
+      count += ((*surf_fit_marker)[sdofs[s]]) ? 1 : 0;
    }
    if (count == 0) { return; }
 
    const FiniteElement &el_s = *surf_fit_gf->FESpace()->GetFE(el_id);
 
-   const int dof_x = el_x.GetDof(), dim = el_x.GetDim(),
-             dof_s = el_s.GetDof();
+   const int dof_s = el_s.GetDof(), dim = el_x.GetDim();
 
    Vector sigma_e;
-   surf_fit_gf->GetSubVector(dofs, sigma_e);
+   surf_fit_gf->GetSubVector(sdofs, sigma_e);
 
    DenseMatrix surf_fit_grad_e(dof_s, dim);
    Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
    DenseMatrix grad_phys;
-   el_s.ProjectGrad(el_s, Tpr, grad_phys);
-   grad_phys.Mult(sigma_e, grad_ptr);
+   if (surf_fit_gf_bg)
+   {
+      surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
+      surf_fit_grad->GetSubVector(dofs, grad_ptr);
+   }
+   else
+   {
+      el_s.ProjectGrad(el_s, Tpr, grad_phys);
+      grad_phys.Mult(sigma_e, grad_ptr);
+   }
 
    DenseMatrix surf_fit_hess_e(dof_s, dim*dim);
    Vector hess_ptr(surf_fit_hess_e.GetData(), dof_s*dim*dim);
-   surf_fit_hess_e.SetSize(dof_s*dim, dim);
-   Mult(grad_phys, surf_fit_grad_e, surf_fit_hess_e);
-   surf_fit_hess_e.SetSize(dof_s, dim * dim);
+   if (surf_fit_gf_bg)
+   {
+      surf_fit_hess->FESpace()->GetElementVDofs(el_id, dofs);
+      surf_fit_hess->GetSubVector(dofs, hess_ptr);
+   }
+   else
+   {
+      surf_fit_hess_e.SetSize(dof_s*dim, dim);
+      Mult(grad_phys, surf_fit_grad_e, surf_fit_hess_e);
+      surf_fit_hess_e.SetSize(dof_s, dim * dim);
+   }
 
    const IntegrationRule &ir = el_s.GetNodes();
-   Vector shape_x(dof_x), shape_s(dof_s);
 
    Vector surf_fit_grad_s(dim);
    DenseMatrix surf_fit_hess_s(dim, dim);
 
-
    for (int s = 0; s < dof_s; s++)
    {
-      if ((*surf_fit_marker)[dofs[s]] == false) { continue; }
+      if ((*surf_fit_marker)[sdofs[s]] == false) { continue; }
 
       const IntegrationPoint &ip = ir.IntPoint(s);
       Tpr.SetIntPoint(&ip);
-      el_x.CalcShape(ip, shape_x);
-      el_s.CalcShape(ip, shape_s);
 
-      // These are the sums over k at the dof s (looking at the notes).
-      surf_fit_grad_e.MultTranspose(shape_s, surf_fit_grad_s);
       Vector gg_ptr(surf_fit_hess_s.GetData(), dim * dim);
-      surf_fit_hess_e.MultTranspose(shape_s, gg_ptr);
+      surf_fit_hess_e.GetRow(s, gg_ptr);
 
       // Loops over the local matrix.
       const double w = surf_fit_normal * surf_fit_coeff->Eval(Tpr, ip);
-      for (int i = 0; i < dof_x * dim; i++)
+      for (int idim = 0; idim < dim; idim++)
       {
-         const int idof = i % dof_x, idim = i / dof_x;
-         for (int j = 0; j <= i; j++)
+         for (int jdim = 0; jdim <= idim; jdim++)
          {
-            const int jdof = j % dof_x, jdim = j / dof_x;
-            const double entry =
-               w * ( 2.0 * surf_fit_grad_s(idim) * shape_x(idof) *
-                     /* */ surf_fit_grad_s(jdim) * shape_x(jdof) +
-                     2.0 * sigma_e(s) * surf_fit_hess_s(idim, jdim) *
-                     /* */ shape_x(idof) * shape_x(jdof));
-            mat(i, j) += entry;
-            if (i != j) { mat(j, i) += entry; }
+            double entry = w * ( 2.0 * surf_fit_grad_e(s, idim) *
+                                 /* */ surf_fit_grad_e(s, jdim) +
+                                 2.0 * sigma_e(s) * surf_fit_hess_s(idim, jdim));
+            entry *= 1.0/surf_fit_dof_count[sdofs[s]];
+            int idx = s + idim*ndofs;
+            int jdx = s + jdim*ndofs;
+            mat(idx, jdx) += entry;
+            if (idx != jdx) { mat(jdx, idx) += entry; }
          }
       }
    }
@@ -4062,7 +4145,7 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
 }
 
 void TMOP_Integrator::UpdateAfterMeshPositionChange(const Vector &new_x,
-                                                    int new_x_ordering)
+                                                    int x_ordering)
 {
    if (discr_tc)
    {
@@ -4071,13 +4154,115 @@ void TMOP_Integrator::UpdateAfterMeshPositionChange(const Vector &new_x,
    // Update adapt_lim_gf if adaptive limiting is enabled.
    if (adapt_lim_gf)
    {
-      adapt_lim_eval->ComputeAtNewPosition(new_x, *adapt_lim_gf, new_x_ordering);
+      adapt_lim_eval->ComputeAtNewPosition(new_x, *adapt_lim_gf, x_ordering);
    }
 
    // Update surf_fit_gf if surface fitting is enabled.
    if (surf_fit_gf)
    {
-      surf_fit_eval->ComputeAtNewPosition(new_x, *surf_fit_gf, new_x_ordering);
+      if (surf_fit_gf_bg)
+      {
+         // Interpolate information for only DOFs marked for fitting.
+         const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
+         const int cnt = surf_fit_marker_dof_index.Size();
+         const int total_cnt = new_x.Size()/dim;
+         Vector new_x_sorted(cnt*dim);
+         if (x_ordering == 0)
+         {
+            for (int d = 0; d < dim; d++)
+            {
+               for (int i = 0; i < cnt; i++)
+               {
+                  int dof_index = surf_fit_marker_dof_index[i];
+                  new_x_sorted(i + d*cnt) = new_x(dof_index + d*total_cnt);
+               }
+            }
+         }
+         else
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               for (int d = 0; d < dim; d++)
+               {
+                  new_x_sorted(d + i*dim) = new_x(d + dof_index*dim);
+               }
+            }
+         }
+
+         Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
+         surf_fit_eval->ComputeAtNewPosition(
+            new_x_sorted, surf_fit_gf_int, x_ordering);
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
+         }
+
+         surf_fit_eval_bg_grad->ComputeAtNewPosition(
+            new_x_sorted, surf_fit_grad_int, x_ordering);
+         // Assumes surf_fit_grad and surf_fit_gf share the same space
+         const int grad_dim = surf_fit_grad->VectorDim();
+         const int grad_cnt = surf_fit_grad->Size()/grad_dim;
+         if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
+         {
+            for (int d = 0; d < grad_dim; d++)
+            {
+               for (int i = 0; i < cnt; i++)
+               {
+                  int dof_index = surf_fit_marker_dof_index[i];
+                  (*surf_fit_grad)[dof_index + d*grad_cnt] =
+                     surf_fit_grad_int(i + d*cnt);
+               }
+            }
+         }
+         else
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               for (int d = 0; d < grad_dim; d++)
+               {
+                  (*surf_fit_grad)[dof_index*dim + d] =
+                     surf_fit_grad_int(i*dim + d);
+               }
+            }
+         }
+
+         surf_fit_eval_bg_hess->ComputeAtNewPosition(
+            new_x_sorted, surf_fit_hess_int, x_ordering);
+         // Assumes surf_fit_hess and surf_fit_gf share the same space
+         const int hess_dim = surf_fit_hess->VectorDim();
+         const int hess_cnt = surf_fit_hess->Size()/hess_dim;
+         if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
+         {
+            for (int d = 0; d < hess_dim; d++)
+            {
+               for (int i = 0; i < cnt; i++)
+               {
+                  int dof_index = surf_fit_marker_dof_index[i];
+                  (*surf_fit_hess)[dof_index + d*hess_cnt] =
+                     surf_fit_hess_int(i + d*cnt);
+               }
+            }
+         }
+         else
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               for (int d = 0; d < hess_dim; d++)
+               {
+                  (*surf_fit_hess)[dof_index*dim + d] =
+                     surf_fit_hess_int(i*dim + d);
+               }
+            }
+         }
+      }
+      else
+      {
+         surf_fit_eval->ComputeAtNewPosition(new_x, *surf_fit_gf, x_ordering);
+      }
    }
 }
 

fem/tmop.hpp

@@ -1681,6 +1681,11 @@ protected:
    Coefficient *surf_fit_coeff;         // Not owned.
    AdaptivityEvaluator *surf_fit_eval;  // Not owned.
    double surf_fit_normal;
+   bool surf_fit_gf_bg;
+   GridFunction *surf_fit_grad, *surf_fit_hess;
+   AdaptivityEvaluator *surf_fit_eval_bg_grad, *surf_fit_eval_bg_hess;
+   Array<int> surf_fit_dof_count;
+   Array<int> surf_fit_marker_dof_index;
 
    DiscreteAdaptTC *discr_tc;
 
@@ -1799,7 +1804,7 @@ protected:
    void ComputeMinJac(const Vector &x, const FiniteElementSpace &fes);
 
    void UpdateAfterMeshPositionChange(const Vector &new_x,
-                                      int new_x_ordering = Ordering::byNODES);
+                                      int x_ordering = Ordering::byNODES);
 
    void DisableLimiting()
    {
@@ -1878,6 +1883,8 @@ public:
         surf_fit_gf(NULL), surf_fit_marker(NULL),
         surf_fit_coeff(NULL),
         surf_fit_eval(NULL), surf_fit_normal(1.0),
+        surf_fit_gf_bg(false), surf_fit_grad(NULL), surf_fit_hess(NULL),
+        surf_fit_eval_bg_grad(NULL), surf_fit_eval_bg_hess(NULL),
         discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
    { PA.enabled = false; }
@@ -1963,11 +1970,45 @@ public:
    void EnableSurfaceFitting(const GridFunction &s0,
                              const Array<bool> &smarker, Coefficient &coeff,
                              AdaptivityEvaluator &ae);
+
 #ifdef MFEM_USE_MPI
    /// Parallel support for surface fitting.
    void EnableSurfaceFitting(const ParGridFunction &s0,
                              const Array<bool> &smarker, Coefficient &coeff,
                              AdaptivityEvaluator &ae);
+
+   /** @brief Fitting of certain DOFs in the current mesh to the zero level set
+       of a function defined on another (finer) source mesh.
+
+       Having a level set function s_bg(x_bg) on a source/background mesh,
+       a set of marked nodes (or DOFs) in the current mesh, we move the marked
+       nodes to the zero level set of s_bg. This functionality is used for
+       surface fitting and tangential relaxation.
+
+       @param[in] s_bg       The level set function on the background mesh.
+       @param[in] s0         The level set function (automatically) interpolated
+                             on the initial mesh.
+       @param[in] smarker    Marker for aligned DOFs in the current mesh.
+       @param[in] coeff      Coefficient c for the fitting penalty term.
+       @param[in] ae         Interpolates s(x) from s_bg(x_bg).
+       @param[in] s_bg_grad  Gradient of s_bg on the background mesh.
+       @param[in] s0_grad    Gradient of s0 on the initial mesh.
+       @param[in] age        Interpolates s_grad(x) from s_bg_grad(x_bg).
+       @param[in] s_bg_hess  Hessian of s(x) on the background mesh.
+       @param[in] s0_hess    Hessian of s0 on the initial mesh.
+       @param[in] ahe        Interpolates s_hess(x) from s_bg_hess(x_bg).
+       See the pmesh-fitting miniapp for details on usage. */
+   void EnableSurfaceFittingFromSource(const ParGridFunction &s_bg,
+                                       ParGridFunction &s0,
+                                       const Array<bool> &smarker,
+                                       Coefficient &coeff,
+                                       AdaptivityEvaluator &ae,
+                                       const ParGridFunction &s_bg_grad,
+                                       ParGridFunction &s0_grad,
+                                       AdaptivityEvaluator &age,
+                                       const ParGridFunction &s_bg_hess,
+                                       ParGridFunction &s0_hess,
+                                       AdaptivityEvaluator &ahe);
 #endif
    void GetSurfaceFittingErrors(double &err_avg, double &err_max);
    bool IsSurfaceFittingEnabled() { return (surf_fit_gf != NULL); }

fem/tmop_tools.cpp

@@ -155,7 +155,7 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
 
    double t = 0.0;
    bool last_step = false;
-   for (int ti = 1; !last_step; ti++)
+   while (!last_step)
    {
       if (t + dt >= 1.0)
       {
@@ -416,11 +416,11 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 #endif
 
    double scale = 1.0;
+   double avg_surf_fit_err, max_surf_fit_err = 0.0;
    if (surf_fit_max_threshold > 0.0)
    {
-      double avg_err, max_err;
-      GetSurfaceFittingError(avg_err, max_err);
-      if (max_err < surf_fit_max_threshold)
+      GetSurfaceFittingError(avg_surf_fit_err, max_surf_fit_err);
+      if (max_surf_fit_err < surf_fit_max_threshold)
       {
          if (print_options.iterations)
          {
@@ -431,6 +431,17 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
          return scale;
       }
    }
+   if (adapt_inc_count >= max_adapt_inc_count)
+   {
+      if (print_options.iterations)
+      {
+         mfem::out << "TMOPNewtonSolver converged "
+                   "based on max number of times surface fitting weight can"
+                   "be increased. \n";
+      }
+      scale = 0.0;
+      return scale;
+   }
 
    // Check if the starting mesh (given by x) is inverted. Note that x hasn't
    // been modified by the Newton update yet.
@@ -443,6 +454,8 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       // reference to detect deteriorations.
       MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
                   " intermediate mesh is invalid. Contact TMOP Developers.");
+      MFEM_VERIFY(min_detJ_threshold == 0.0,
+                  "This setup is not supported. Contact TMOP Developers.");
       *min_det_ptr = untangle_factor * min_detT_in;
    }
 
@@ -476,9 +489,9 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
       // Check the changes in detJ.
       min_detT_out = ComputeMinDet(x_out_loc, *fes);
-      if (untangling == false && min_detT_out < 0.0)
+      if (untangling == false && min_detT_out <= min_detJ_threshold)
       {
-         // No untangling, and detJ got negative -- no good.
+         // No untangling, and detJ got negative (or small) -- no good.
          if (print_options.iterations)
          {
             mfem::out << "Scale = " << scale << " Neg det(J) found.\n";
@@ -503,6 +516,20 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       // Check the changes in total energy.
       ProcessNewState(x_out);
 
+      double avg_fit_err, max_fit_err = 0.0;
+      if (surf_fit_max_threshold > 0.0)
+      {
+         GetSurfaceFittingError(avg_fit_err, max_fit_err);
+      }
+      if (surf_fit_max_threshold > 0.0 && max_fit_err >= 1.2*max_surf_fit_err)
+      {
+         if (print_options.iterations)
+         {
+            mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+         }
+         scale *= 0.5; continue;
+      }
+
       if (serial)
       {
          energy_out = nlf->GetGridFunctionEnergy(x_out_loc);
@@ -574,7 +601,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
    if (x_out_ok == false) { scale = 0.0; }
 
-   if (adaptive_surf_fit) { update_surf_fit_coeff = true; }
+   if (surf_fit_scale_factor > 0.0) { update_surf_fit_coeff = true; }
    compute_metric_quantile_flag = true;
 
    return scale;
@@ -795,8 +822,6 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
    // decrease between subsequent TMOPNewtonSolver iterations.
    if (update_surf_fit_coeff)
    {
-      double surf_fit_err_max = -10;
-      double surf_fit_err_avg = -10;
       // Get surface fitting errors.
       GetSurfaceFittingError(surf_fit_err_avg, surf_fit_err_max);
       // Get array with surface fitting weights.
@@ -816,9 +841,14 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
       double rel_change_surf_fit_err = change_surf_fit_err/surf_fit_err_avg_prvs;
       // Increase the surface fitting coefficient if the surface fitting error
       // does not decrease sufficiently.
-      if (rel_change_surf_fit_err < 1.e-2)
+      if (rel_change_surf_fit_err < surf_fit_rel_change_threshold)
       {
-         UpdateSurfaceFittingWeight(10);
+         UpdateSurfaceFittingWeight(surf_fit_scale_factor);
+         adapt_inc_count += 1;
+      }
+      else
+      {
+         adapt_inc_count = 0;
       }
       surf_fit_err_avg_prvs = surf_fit_err_avg;
       update_surf_fit_coeff = false;

fem/tmop_tools.hpp

@@ -134,11 +134,18 @@ protected:
    int solver_type;
    bool parallel;
 
+   // Line search step is rejected if min(detJ) <= min_detJ_threshold.
+   double min_detJ_threshold = 0.0;
+
    // Surface fitting variables.
-   bool adaptive_surf_fit = false;
    mutable double surf_fit_err_avg_prvs = 10000.0;
+   mutable double surf_fit_err_avg, surf_fit_err_max;
    mutable bool update_surf_fit_coeff = false;
    double surf_fit_max_threshold = -1.0;
+   double surf_fit_rel_change_threshold = 0.001;
+   double surf_fit_scale_factor = 0.0;
+   mutable int adapt_inc_count = 0;
+   mutable int max_adapt_inc_count = 10;
 
    // Minimum determinant over the whole mesh. Used for mesh untangling.
    double *min_det_ptr = nullptr;
@@ -220,16 +227,38 @@ public:
    virtual void ProcessNewState(const Vector &x) const;
 
    /** @name Methods for adaptive surface fitting weight. (Experimental) */
-   /// Enable adaptive surface fitting weight.
-   /// The weight is modified after each TMOPNewtonSolver iteration.
-   void EnableAdaptiveSurfaceFitting() { adaptive_surf_fit = true; }
-
-   /// Set the termination criterion for mesh optimization based on
-   /// the maximum surface fitting error.
+   /// Enable/Disable adaptive surface fitting weight.
+   /// The weight is modified after each TMOPNewtonSolver iteration as:
+   /// w_{k+1} = w_{k} * @a surf_fit_scale_factor if relative change in
+   /// max surface fitting error < @a surf_fit_rel_change_threshold.
+   /// The solver terminates if the maximum surface fitting error does
+   /// not sufficiently decrease for @a max_adapt_inc_count consecutive
+   /// solver iterations or if the max error falls below @a surf_fit_max_threshold.
+   void EnableAdaptiveSurfaceFitting()
+   {
+      surf_fit_scale_factor = 10.0;
+      surf_fit_rel_change_threshold = 0.001;
+   }
+   void SetAdaptiveSurfaceFittingScalingFactor(double factor)
+   {
+      surf_fit_scale_factor = factor;
+   }
+   void SetAdaptiveSurfaceFittingRelativeChangeThreshold(double threshold)
+   {
+      surf_fit_rel_change_threshold = threshold;
+   }
+   void SetMaxNumberofIncrementsForAdaptiveFitting(int count)
+   {
+      max_adapt_inc_count = count;
+   }
    void SetTerminationWithMaxSurfaceFittingError(double max_error)
    {
       surf_fit_max_threshold = max_error;
    }
+   void SetMinimumDeterminantThreshold(double threshold)
+   {
+      min_detJ_threshold = threshold;
+   }
 
    virtual void Mult(const Vector &b, Vector &x) const
    {

general/array.hpp

@@ -877,9 +877,8 @@ template <class T>
 inline void Array<T>::MakeRef(const Array &master)
 {
    data.Delete();
-   data = master.data; // note: copies the device flag
    size = master.size;
-   data.ClearOwnerFlags();
+   data.MakeAlias(master.GetMemory(), 0, size);
 }
 
 template <class T>

general/mem_manager.cpp

@@ -802,9 +802,7 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
                                MemoryType mt,
                                bool own, bool alias, unsigned &flags)
 {
-   MFEM_CONTRACT_VAR(alias);
    MFEM_ASSERT(exists, "Internal error!");
-   MFEM_VERIFY(!alias, "Cannot register an alias!");
    const bool is_host_mem = IsHostMemory(mt);
    const MemType h_mt = is_host_mem ? mt : GetDualMemoryType(mt);
    const MemType d_mt = is_host_mem ? MemoryType::DEFAULT : mt;
@@ -820,6 +818,8 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
       return nullptr;
    }
 
+   MFEM_VERIFY(!alias, "Cannot register an alias!");
+
    flags |= Mem::Registered | Mem::OWNS_INTERNAL;
    void *h_ptr;
 
@@ -876,8 +876,8 @@ void MemoryManager::Alias_(void *base_h_ptr, size_t offset, size_t bytes,
 {
    mm.InsertAlias(base_h_ptr, (char*)base_h_ptr + offset, bytes,
                   base_flags & Mem::ALIAS);
-   flags = (base_flags | Mem::ALIAS | Mem::OWNS_INTERNAL) &
-           ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE);
+   flags = (base_flags | Mem::ALIAS) & ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE);
+   if (base_h_ptr) { flags |= Mem::OWNS_INTERNAL; }
 }
 
 void MemoryManager::SetDeviceMemoryType_(void *h_ptr, unsigned flags,

linalg/hiop.cpp

@@ -88,6 +88,7 @@ bool HiopOptimizationProblem::eval_f(const size_type &n, const double *x,
 
    Vector x_vec(ntdofs_loc);
    x_vec = x;
+   problem.new_x = new_x;
    obj_value = problem.CalcObjective(x_vec);
 
    return true;
@@ -102,6 +103,7 @@ bool HiopOptimizationProblem::eval_grad_f(const size_type &n, const double *x,
 
    Vector x_vec(ntdofs_loc), gradf_vec(ntdofs_loc);
    x_vec = x;
+   problem.new_x = new_x;
    problem.CalcObjectiveGrad(x_vec, gradf_vec);
    std::memcpy(gradf, gradf_vec.GetData(), ntdofs_loc * sizeof(double));
 
@@ -123,6 +125,7 @@ bool HiopOptimizationProblem::eval_cons(const size_type &n, const size_type &m,
    if (new_x) { constr_info_is_current = false; }
    Vector x_vec(ntdofs_loc);
    x_vec = x;
+   problem.new_x = new_x;
    UpdateConstrValsGrads(x_vec);
 
    for (int c = 0; c < num_cons; c++)
@@ -150,6 +153,7 @@ bool HiopOptimizationProblem::eval_Jac_cons(const size_type &n,
    if (new_x) { constr_info_is_current = false; }
    Vector x_vec(ntdofs_loc);
    x_vec = x;
+   problem.new_x = new_x;
    UpdateConstrValsGrads(x_vec);
 
    for (int c = 0; c < num_cons; c++)

linalg/solvers.cpp

@@ -1028,6 +1028,8 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
       final_norm = beta;
       final_iter = 0;
       converged = true;
+      j = 0;
+      resid = beta;
       goto finish;
    }
 

linalg/solvers.hpp

@@ -793,6 +793,9 @@ int aGMRES(const Operator &A, Vector &x, const Vector &b,
            int m_max, int m_min, int m_step, double cf,
            double &tol, double &atol, int printit);
 
+#ifdef MFEM_USE_HIOP
+class HiopOptimizationProblem;
+#endif
 
 /** Defines operators and constraints for the following optimization problem:
  *
@@ -812,11 +815,25 @@ int aGMRES(const Operator &A, Vector &x, const Vector &b,
  *  the operators are expected to be defined for tdof vectors. */
 class OptimizationProblem
 {
+#ifdef MFEM_USE_HIOP
+   friend class HiopOptimizationProblem;
+#endif
+
+private:
+   /// See NewX().
+   mutable bool new_x = true;
+
 protected:
    /// Not owned, some can remain unused (NULL).
    const Operator *C, *D;
    const Vector *c_e, *d_lo, *d_hi, *x_lo, *x_hi;
 
+   /// Implementations of CalcObjective() and CalcObjectiveGrad() can use this
+   /// method to check if the argument Vector x has been changed after the last
+   /// call to CalcObjective() or CalcObjectiveGrad().
+   /// The result is on by default, and gets set by the OptimizationSolver.
+   bool NewX() const { return new_x; }
+
 public:
    const int input_size;
 

makefile

@@ -119,7 +119,7 @@ $(if $(word 2,$(SRC)),$(error Spaces in SRC = "$(SRC)" are not supported))
 MFEM_GIT_STRING = $(shell [ -d $(MFEM_DIR)/.git ] && git -C $(MFEM_DIR) \
    describe --all --long --abbrev=40 --dirty --always 2> /dev/null)
 
-EXAMPLE_SUBDIRS = amgx caliper ginkgo hiop petsc pumi sundials superlu moonolith
+EXAMPLE_SUBDIRS = amgx caliper ginkgo hiop petsc pumi sundials superlu moonolith contact
 EXAMPLE_DIRS := examples $(addprefix examples/,$(EXAMPLE_SUBDIRS))
 EXAMPLE_TEST_DIRS := examples
 

mesh/mesh.cpp

@@ -1319,6 +1319,7 @@ std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info)
          os << "NA";
          break;
    }
+   os << '\n';
    os << "element[0].location=";
    switch (info.element[0].location)
    {
@@ -1332,7 +1333,7 @@ std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info)
          os << "NA";
          break;
    }
-   os << std::endl;
+   os << '\n';
    os << "element[1].location=";
    switch (info.element[1].location)
    {
@@ -1346,7 +1347,7 @@ std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info)
          os << "NA";
          break;
    }
-   os << std::endl;
+   os << '\n';
    os << "element[0].conformity=";
    switch (info.element[0].conformity)
    {
@@ -1363,7 +1364,7 @@ std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info)
          os << "NA";
          break;
    }
-   os << std::endl;
+   os << '\n';
    os << "element[1].conformity=";
    switch (info.element[1].conformity)
    {
@@ -1380,13 +1381,13 @@ std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info)
          os << "NA";
          break;
    }
-   os << std::endl;
-   os << "element[0].index=" << info.element[0].index << std::endl
-      << "element[1].index=" << info.element[1].index << std::endl
-      << "element[0].local_face_id=" << info.element[0].local_face_id << std::endl
-      << "element[1].local_face_id=" << info.element[1].local_face_id << std::endl
-      << "element[0].orientation=" << info.element[0].orientation << std::endl
-      << "element[1].orientation=" << info.element[1].orientation << std::endl
+   os << '\n';
+   os << "element[0].index=" << info.element[0].index << '\n'
+      << "element[1].index=" << info.element[1].index << '\n'
+      << "element[0].local_face_id=" << info.element[0].local_face_id << '\n'
+      << "element[1].local_face_id=" << info.element[1].local_face_id << '\n'
+      << "element[0].orientation=" << info.element[0].orientation << '\n'
+      << "element[1].orientation=" << info.element[1].orientation << '\n'
       << "ncface=" << info.ncface << std::endl;
    return os;
 }
@@ -5221,17 +5222,13 @@ void Mesh::UpdateNURBS()
    if (el_to_edge)
    {
       NumOfEdges = GetElementToEdgeTable(*el_to_edge, be_to_edge);
-      if (Dim == 2)
-      {
-         GenerateFaces();
-      }
    }
 
    if (el_to_face)
    {
       GetElementToFaceTable();
-      GenerateFaces();
    }
+   GenerateFaces();
 }
 
 void Mesh::LoadPatchTopo(std::istream &input, Array<int> &edge_to_knot)
@@ -5609,6 +5606,8 @@ int Mesh::CheckElementOrientation(bool fix_it)
                 << NumOfElements << " (" << fixed_or_not[(wo == fo) ? 0 : 1]
                 << ")" << endl;
    }
+#else
+   MFEM_CONTRACT_VAR(fo);
 #endif
    return wo;
 }

mesh/mesh_readers.cpp

@@ -2632,6 +2632,8 @@ void Mesh::ReadGmshMesh(std::istream &input, int &curved, int &read_gf)
          // Suppress warnings (MFEM_CONTRACT_VAR does not work here with nvcc):
          ++n_partitions;
          ++elem_domain;
+         MFEM_CONTRACT_VAR(n_partitions);
+         MFEM_CONTRACT_VAR(elem_domain);
 
       } // section '$Elements'
       else if (buff == "$Periodic") // Reading master/slave node pairs

mesh/ncmesh.cpp

@@ -6193,6 +6193,7 @@ void NCMesh::LegacyToNewVertexOrdering(Array<int> &order) const
       }
    }
    MFEM_ASSERT(count == order.Size(), "");
+   MFEM_CONTRACT_VAR(count);
 }
 
 

mesh/nurbs.hpp

@@ -96,18 +96,23 @@ protected:
 
    Array<KnotVector *> kv;
 
-   int sd, nd;
-
-   void swap(NURBSPatch *np);
-
    // Special B-NET access functions
+   //  - SetLoopDirection(int dir) flattens the multi-dimensional B-NET in the
+   //    requested direction. It effectively creates a 1D net.
+   //  - The slice(int, int) operator is the access function in that flattened structure.
+   //    The first int gives the slice and the second int the element in that slice.
+   //  - Both routines are used in 'InsertKnot', 'DegreeElevate' and 'UniformRefinement'.
+   //  - In older implementations slice(int int) was implemented as operator()(int, int)
+   int nd; // Number of knots in flattened structure
+   int ls; // Number of variables per knot in flattened structure
+   int sd; // Stride for data access
    int SetLoopDirection(int dir);
-   inline       double &operator()(int i, int j);
-   inline const double &operator()(int i, int j) const;
-
-   void init(int dim_);
+   inline       double &slice(int i, int j);
+   inline const double &slice(int i, int j) const;
 
    NURBSPatch(NURBSPatch *parent, int dir, int Order, int NCP);
+   void swap(NURBSPatch *np);
+   void init(int dim_);
 
 public:
    NURBSPatch(const NURBSPatch &orig);
@@ -140,6 +145,9 @@ public:
    KnotVector *GetKV(int i) { return kv[i]; }
 
    // Standard B-NET access functions
+   inline       double &operator()(int i, int j);
+   inline const double &operator()(int i, int j) const;
+
    inline       double &operator()(int i, int j, int l);
    inline const double &operator()(int i, int j, int l) const;
 
@@ -246,6 +254,7 @@ protected:
    // periodic BC helper functions
    void InitDofMap();
    void ConnectBoundaries();
+   void ConnectBoundaries1D(int bnd0, int bnd1);
    void ConnectBoundaries2D(int bnd0, int bnd1);
    void ConnectBoundaries3D(int bnd0, int bnd1);
    int DofMap(int dof) const
@@ -261,14 +270,15 @@ protected:
    void CountBdrElements();
 
    // generate the mesh elements
+   void Get1DElementTopo(Array<Element *> &elements) const;
    void Get2DElementTopo(Array<Element *> &elements) const;
    void Get3DElementTopo(Array<Element *> &elements) const;
 
    // generate the boundary mesh elements
+   void Get1DBdrElementTopo(Array<Element *> &boundary) const;
    void Get2DBdrElementTopo(Array<Element *> &boundary) const;
    void Get3DBdrElementTopo(Array<Element *> &boundary) const;
 
-
    // FE space generation functions
 
    // based on activeElem, count NumOfActiveDofs, generate el_dof,
@@ -277,6 +287,7 @@ protected:
 
    // generate elem_to_global-dof table for the active elements
    // define el_to_patch, el_to_IJK, activeDof (as bool)
+   void Generate1DElementDofTable();
    void Generate2DElementDofTable();
    void Generate3DElementDofTable();
 
@@ -285,17 +296,20 @@ protected:
 
    // generate the bdr-elem_to_global-dof table for the active bdr. elements
    // define bel_to_patch, bel_to_IJK
+   void Generate1DBdrElementDofTable();
    void Generate2DBdrElementDofTable();
    void Generate3DBdrElementDofTable();
 
    // FE --> Patch translation functions
    void GetPatchNets  (const Vector &Nodes, int vdim);
+   void Get1DPatchNets(const Vector &Nodes, int vdim);
    void Get2DPatchNets(const Vector &Nodes, int vdim);
    void Get3DPatchNets(const Vector &Nodes, int vdim);
 
    // Patch --> FE translation functions
    // Side effects: delete the patches, update the weights from the patches
    void SetSolutionVector  (Vector &Nodes, int vdim);
+   void Set1DSolutionVector(Vector &Nodes, int vdim);
    void Set2DSolutionVector(Vector &Nodes, int vdim);
    void Set3DSolutionVector(Vector &Nodes, int vdim);
 
@@ -446,6 +460,7 @@ private:
    int *partitioning;
 
    Table *GetGlobalElementDofTable();
+   Table *Get1DGlobalElementDofTable();
    Table *Get2DGlobalElementDofTable();
    Table *Get3DGlobalElementDofTable();
 
@@ -518,16 +533,55 @@ public:
 
 // Inline function implementations
 
-inline double &NURBSPatch::operator()(int i, int j)
+inline double &NURBSPatch::slice(int i, int j)
 {
+#ifdef MFEM_DEBUG
+   if (data == 0 || i < 0 || i >= nd || j < 0 || j > ls)
+   {
+      mfem_error("NURBSPatch::slice()");
+   }
+#endif
    return data[j%sd + sd*(i + (j/sd)*nd)];
 }
 
-inline const double &NURBSPatch::operator()(int i, int j) const
+inline const double &NURBSPatch::slice(int i, int j) const
 {
+#ifdef MFEM_DEBUG
+   if (data == 0 || i < 0 || i >= nd || j < 0 || j > ls)
+   {
+      mfem_error("NURBSPatch::slice()");
+   }
+#endif
    return data[j%sd + sd*(i + (j/sd)*nd)];
 }
 
+
+inline double &NURBSPatch::operator()(int i, int l)
+{
+#ifdef MFEM_DEBUG
+   if (data == 0 || i < 0 || i >= ni || nj > 0 || nk > 0 ||
+       l < 0 || l >= Dim)
+   {
+      mfem_error("NURBSPatch::operator() 1D");
+   }
+#endif
+
+   return data[i*Dim+l];
+}
+
+inline const double &NURBSPatch::operator()(int i, int l) const
+{
+#ifdef MFEM_DEBUG
+   if (data == 0 || i < 0 || i >= ni ||  nj > 0 || nk > 0 ||
+       l < 0 || l >= Dim)
+   {
+      mfem_error("NURBSPatch::operator() const 1D");
+   }
+#endif
+
+   return data[i*Dim+l];
+}
+
 inline double &NURBSPatch::operator()(int i, int j, int l)
 {
 #ifdef MFEM_DEBUG

mesh/pncmesh.cpp

@@ -2012,6 +2012,9 @@ void ParNCMesh::RedistributeElements(Array<int> &new_ranks, int target_elements,
                   "(glob_sent, glob_recv) = ("
                   << glob_sent << ", " << glob_recv << ")");
    }
+#else
+   MFEM_CONTRACT_VAR(nsent);
+   MFEM_CONTRACT_VAR(nrecv);
 #endif
 }
 

miniapps/common/CMakeLists.txt

@@ -21,9 +21,11 @@ list(APPEND HDRS
 
 if (MFEM_USE_MPI)
   list(APPEND SRCS
-    pfem_extras.cpp)
+    pfem_extras.cpp
+    dist_solver.cpp)
   list(APPEND HDRS
-    pfem_extras.hpp)
+    pfem_extras.hpp
+    dist_solver.hpp)
 endif()
 
 if (MFEM_USE_CUDA)

miniapps/common/dist_solver.cpp

@@ -11,9 +11,14 @@
 
 #include "dist_solver.hpp"
 
+#ifdef MFEM_USE_MPI
+
 namespace mfem
 {
 
+namespace common
+{
+
 void DiffuseField(ParGridFunction &field, int smooth_steps)
 {
    // Setup the Laplacian operator.
@@ -811,4 +816,8 @@ void PDEFilter::Filter(Coefficient &func, ParGridFunction &ffield)
    ffield.ProjectCoefficient(gfc);
 }
 
-}
+} // namespace common
+
+} // namespace mfem
+
+#endif

miniapps/common/dist_solver.hpp

@@ -14,9 +14,14 @@
 
 #include "mfem.hpp"
 
+#ifdef MFEM_USE_MPI
+
 namespace mfem
 {
 
+namespace common
+{
+
 double AvgElementSize(ParMesh &pmesh);
 
 class DistanceSolver
@@ -303,6 +308,9 @@ private:
    ScreenedPoisson* sint;
 };
 
+} // namespace common
+
 } // namespace mfem
 
+#endif // MFEM_USE_MPI
 #endif

miniapps/common/makefile

@@ -51,7 +51,7 @@ SEQ_MINIOBJS = mesh_extras.o fem_extras.o
 ifeq ($(MFEM_USE_MPI),NO)
    MINIOBJS = $(SEQ_MINIOBJS)
 else
-   MINIOBJS = $(SEQ_MINIOBJS) pfem_extras.o
+   MINIOBJS = $(SEQ_MINIOBJS) pfem_extras.o dist_solver.o
 endif
 
 .SUFFIXES:

miniapps/common/mfem-common.hpp

@@ -15,5 +15,6 @@
 #include "fem_extras.hpp"
 #include "mesh_extras.hpp"
 #include "pfem_extras.hpp"
+#include "dist_solver.hpp"
 
 #endif

miniapps/meshing/CMakeLists.txt

@@ -98,6 +98,12 @@ if (MFEM_USE_MPI)
     LIBRARIES mfem mfem-common)
   add_dependencies(pmesh-optimizer copy_miniapps_meshing_data)
 
+  add_mfem_miniapp(pmesh-fitting
+    MAIN pmesh-fitting.cpp
+    ${MFEM_MINIAPPS_COMMON_HEADERS}
+    LIBRARIES mfem mfem-common)
+  add_dependencies(pmesh-fitting copy_miniapps_meshing_data)
+
   add_mfem_miniapp(pminimal-surface
     MAIN pminimal-surface.cpp
     LIBRARIES mfem)

miniapps/meshing/makefile

@@ -27,7 +27,7 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_MINIAPPS = mobius-strip klein-bottle toroid trimmer twist mesh-explorer\
 	shaper extruder mesh-optimizer minimal-surface polar-nc reflector
-PAR_MINIAPPS = pmesh-optimizer pminimal-surface
+PAR_MINIAPPS = pmesh-optimizer pminimal-surface pmesh-fitting
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
 else
@@ -65,7 +65,7 @@ MESH_FILES = amr-quad-q2.mesh blade.mesh cube.mesh icf.mesh jagged.mesh\
  square01.mesh stretched2D.mesh
 $(MESH_FILES): %: $(SRC)%
 	ln -sf $(<) .
-mesh-optimizer pmesh-optimizer: | $(MESH_FILES)
+mesh-optimizer pmesh-optimizer pmesh-fitting: | $(MESH_FILES)
 .PHONY: copy-data
 copy-data: | $(MESH_FILES)
 endif
@@ -87,6 +87,8 @@ mesh-optimizer-test-seq: mesh-optimizer
 	@$(call mfem-test,$<,, Meshing miniapp)
 pmesh-optimizer-test-par: pmesh-optimizer
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel meshing miniapp)
+pmesh-fitting-test-par: pmesh-fitting
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel mesh fitting miniapp)
 minimal-surface-test-seq: minimal-surface
 	@$(call mfem-test,$<,, Meshing miniapp)
 pminimal-surface-test-par: pminimal-surface
@@ -111,7 +113,7 @@ clean: clean-build clean-exec
 clean-build:
 	rm -f *.o *~ mobius-strip klein-bottle toroid twist
 	rm -f mesh-explorer shaper extruder trimmer reflector
-	rm -f mesh-optimizer pmesh-optimizer polar-nc
+	rm -f mesh-optimizer pmesh-optimizer pmesh-fitting polar-nc
 	rm -f minimal-surface pminimal-surface
 	rm -rf *.dSYM *.TVD.*breakpoints
 

miniapps/meshing/mesh-fitting.hpp

@@ -0,0 +1,480 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mesh-optimizer.hpp"
+#include "../common/mfem-common.hpp"
+
+using namespace std;
+using namespace mfem;
+using namespace common;
+
+// Used for exact surface alignment
+double circle_level_set(const Vector &x)
+{
+   const int dim = x.Size();
+   if (dim == 2)
+   {
+      const double xc = x(0) - 0.5, yc = x(1) - 0.5;
+      const double r = sqrt(xc*xc + yc*yc);
+      return r-0.25;
+   }
+   else
+   {
+      const double xc = x(0) - 0.5, yc = x(1) - 0.5, zc = x(2) - 0.5;
+      const double r = sqrt(xc*xc + yc*yc + zc*zc);
+      return r-0.3;
+   }
+}
+
+double in_circle(const Vector &x, const Vector &x_center, double radius)
+{
+   Vector x_current = x;
+   x_current -= x_center;
+   double dist = x_current.Norml2();
+   if (dist < radius)
+   {
+      return 1.0;
+   }
+   else if (dist == radius)
+   {
+      return 0.0;
+   }
+
+   return -1.0;
+}
+
+double in_trapezium(const Vector &x, double a, double b, double l)
+{
+   double phi_t = x(1) + (a-b)*x(0)/l - a;
+   return (phi_t <= 0.0) ? 1.0 : -1.0;
+}
+
+double in_parabola(const Vector &x, double h, double k, double t)
+{
+   double phi_p1 = (x(0)-h-t/2) - k*x(1)*x(1);
+   double phi_p2 = (x(0)-h+t/2) - k*x(1)*x(1);
+   return (phi_p1 <= 0.0 && phi_p2 >= 0.0) ? 1.0 : -1.0;
+}
+
+double in_rectangle(const Vector &x, double xc, double yc, double w, double h)
+{
+   double dx = fabs(x(0) - xc);
+   double dy = fabs(x(1) - yc);
+   return (dx <= w/2 && dy <= h/2) ? 1.0 : -1.0;
+}
+
+// Fischer-Tropsch like geometry
+double reactor(const Vector &x)
+{
+   // Circle
+   Vector x_circle1(2);
+   x_circle1(0) = 0.0;
+   x_circle1(1) = 0.0;
+   double in_circle1_val = in_circle(x, x_circle1, 0.2);
+
+   double r1 = 0.2;
+   double r2 = 1.0;
+   double in_trapezium_val = in_trapezium(x, 0.05, 0.1, r2-r1);
+
+   double return_val = max(in_circle1_val, in_trapezium_val);
+
+   double h = 0.4;
+   double k = 2;
+   double t = 0.15;
+   double in_parabola_val = in_parabola(x, h, k, t);
+   return_val = max(return_val, in_parabola_val);
+
+   double in_rectangle_val = in_rectangle(x, 0.99, 0.0, 0.12, 0.35);
+   return_val = max(return_val, in_rectangle_val);
+
+   double in_rectangle_val2 = in_rectangle(x, 0.99, 0.5, 0.12, 0.28);
+   return_val = max(return_val, in_rectangle_val2);
+   return return_val;
+}
+
+double in_cube(const Vector &x, double xc, double yc, double zc, double lx,
+               double ly, double lz)
+{
+   double dx = fabs(x(0) - xc);
+   double dy = fabs(x(1) - yc);
+   double dz = fabs(x(2) - zc);
+   return (dx <= lx/2 && dy <= ly/2 && dz <= lz/2) ? 1.0 : -1.0;
+}
+
+double in_pipe(const Vector &x, int pipedir, Vector x_pipe_center,
+               double radius, double minv, double maxv)
+{
+   Vector x_pipe_copy = x_pipe_center;
+   x_pipe_copy -= x;
+   x_pipe_copy(pipedir-1) = 0.0;
+   double dist = x_pipe_copy.Norml2();
+   double xv = x(pipedir-1);
+   if (dist < radius && xv > minv && xv < maxv)
+   {
+      return 1.0;
+   }
+   else if (dist == radius || (xv == minv && dist < radius) || (xv == maxv &&
+                                                                dist < radius))
+   {
+      return 0.0;
+   }
+
+   return -1.0;
+}
+
+double r_intersect(double r1, double r2)
+{
+   return r1 + r2 - std::pow(r1*r1 + r2*r2, 0.5);
+}
+
+double r_union(double r1, double r2)
+{
+   return r1 + r2 + std::pow(r1*r1 + r2*r2, 0.5);
+}
+
+double r_remove(double r1, double r2)
+{
+   return r_intersect(r1, -r2);
+}
+
+double csg_cubecylsph(const Vector &x)
+{
+   Vector xcc(3);
+   xcc = 0.5;
+   double cube_x = 0.25*2;
+   double cube_y = 0.25*2;
+   double cube_z = 0.25*2;
+
+   double in_cube_val = in_cube(x, xcc(0), xcc(1), xcc(2), cube_x, cube_y, cube_z);
+
+   Vector x_circle_c(3);
+   x_circle_c = 0.5;
+
+   double sphere_radius = 0.30;
+   double in_sphere_val = in_circle(x, x_circle_c, sphere_radius);
+   double in_return_val = std::min(in_cube_val, in_sphere_val);
+
+   int pipedir = 1;
+   Vector x_pipe_center(3);
+   x_pipe_center = 0.5;
+   double xmin = 0.5-sphere_radius;
+   double xmax = 0.5+sphere_radius;
+   double pipe_radius = 0.075;
+   double in_pipe_x = in_pipe(x, pipedir, x_pipe_center, pipe_radius, xmin, xmax);
+
+   in_return_val = std::min(in_return_val, -1*in_pipe_x);
+
+   pipedir = 2;
+   in_pipe_x = in_pipe(x, pipedir, x_pipe_center, pipe_radius, xmin, xmax);
+   in_return_val = std::min(in_return_val, -1*in_pipe_x);
+
+   pipedir = 3;
+   in_pipe_x = in_pipe(x, pipedir, x_pipe_center, pipe_radius, xmin, xmax);
+   in_return_val = std::min(in_return_val, -1*in_pipe_x);
+
+   return in_return_val;
+}
+
+#ifdef MFEM_USE_MPI
+void ModifyBoundaryAttributesForNodeMovement(ParMesh *pmesh, ParGridFunction &x)
+{
+   const int dim = pmesh->Dimension();
+   for (int i = 0; i < pmesh->GetNBE(); i++)
+   {
+      mfem::Array<int> dofs;
+      pmesh->GetNodalFESpace()->GetBdrElementDofs(i, dofs);
+      mfem::Vector bdr_xy_data;
+      mfem::Vector dof_xyz(dim);
+      mfem::Vector dof_xyz_compare;
+      mfem::Array<int> xyz_check(dim);
+      for (int j = 0; j < dofs.Size(); j++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            dof_xyz(d) = x(pmesh->GetNodalFESpace()->DofToVDof(dofs[j], d));
+         }
+         if (j == 0)
+         {
+            dof_xyz_compare = dof_xyz;
+            xyz_check = 1;
+         }
+         else
+         {
+            for (int d = 0; d < dim; d++)
+            {
+               if (std::fabs(dof_xyz(d)-dof_xyz_compare(d)) < 1.e-10)
+               {
+                  xyz_check[d] += 1;
+               }
+            }
+         }
+      }
+      if (dim == 2)
+      {
+         if (xyz_check[0] == dofs.Size())
+         {
+            pmesh->GetNodalFESpace()->GetMesh()->SetBdrAttribute(i, 1);
+         }
+         else if (xyz_check[1] == dofs.Size())
+         {
+            pmesh->GetNodalFESpace()->GetMesh()->SetBdrAttribute(i, 2);
+         }
+         else
+         {
+            pmesh->GetNodalFESpace()->GetMesh()->SetBdrAttribute(i, 4);
+         }
+      }
+      else if (dim == 3)
+      {
+         if (xyz_check[0] == dofs.Size())
+         {
+            pmesh->GetNodalFESpace()->GetMesh()->SetBdrAttribute(i, 1);
+         }
+         else if (xyz_check[1] == dofs.Size())
+         {
+            pmesh->GetNodalFESpace()->GetMesh()->SetBdrAttribute(i, 2);
+         }
+         else if (xyz_check[2] == dofs.Size())
+         {
+            pmesh->GetNodalFESpace()->GetMesh()->SetBdrAttribute(i, 3);
+         }
+         else
+         {
+            pmesh->GetNodalFESpace()->GetMesh()->SetBdrAttribute(i, 4);
+         }
+      }
+   }
+}
+
+void ModifyAttributeForMarkingDOFS(ParMesh *pmesh, ParGridFunction &mat,
+                                   int attr_to_switch)
+{
+   mat.ExchangeFaceNbrData();
+   // Switch attribute if all but 1 of the faces of an element will be marked?
+   Array<int> element_attr(pmesh->GetNE());
+   element_attr = 0;
+   for (int e = 0; e < pmesh->GetNE(); e++)
+   {
+      Array<int> faces, ori;
+      if (pmesh->Dimension() == 2)
+      {
+         pmesh->GetElementEdges(e, faces, ori);
+      }
+      else
+      {
+         pmesh->GetElementFaces(e, faces, ori);
+      }
+      int inf1, inf2;
+      int elem1, elem2;
+      int diff_attr_count = 0;
+      int attr1;
+      int attr2;
+      attr1 = mat(e);
+      bool bdr_element = false;
+      element_attr[e] = attr1;
+      int target_attr = -1;
+      for (int f = 0; f < faces.Size(); f++)
+      {
+         pmesh->GetFaceElements(faces[f], &elem1, &elem2);
+         if (elem2 >= 0)
+         {
+            attr2 = elem1 == e ? (int)(mat(elem2)) : (int)(mat(elem1));
+            if (attr1 != attr2 && attr1 == attr_to_switch)
+            {
+               diff_attr_count += 1;
+               target_attr = attr2;
+            }
+         }
+         else
+         {
+            pmesh->GetFaceInfos(faces[f], &inf1, &inf2);
+            if (inf2 >= 0)
+            {
+               Vector dof_vals;
+               Array<int> dofs;
+               mat.GetElementDofValues(pmesh->GetNE() + (-1-elem2), dof_vals);
+               attr2 = (int)(dof_vals(0));
+               if (attr1 != attr2 && attr1 == attr_to_switch)
+               {
+                  diff_attr_count += 1;
+                  target_attr = attr2;
+               }
+            }
+            else
+            {
+               bdr_element = true;
+            }
+         }
+      }
+
+      if (diff_attr_count == faces.Size()-1 && !bdr_element)
+      {
+         element_attr[e] = target_attr;
+      }
+   }
+   for (int e = 0; e < pmesh->GetNE(); e++)
+   {
+      mat(e) = element_attr[e];
+      pmesh->SetAttribute(e, element_attr[e]+1);
+   }
+   mat.ExchangeFaceNbrData();
+   pmesh->SetAttributes();
+}
+
+void OptimizeMeshWithAMRAroundZeroLevelSet(ParMesh &pmesh,
+                                           FunctionCoefficient &ls_coeff,
+                                           int amr_iter,
+                                           ParGridFunction &distance_s,
+                                           const int quad_order = 5,
+                                           Array<ParGridFunction *> *pgf_to_update = NULL)
+{
+   mfem::H1_FECollection h1fec(distance_s.ParFESpace()->FEColl()->GetOrder(),
+                               pmesh.Dimension());
+   mfem::ParFiniteElementSpace h1fespace(&pmesh, &h1fec);
+   mfem::ParGridFunction x(&h1fespace);
+
+   mfem::L2_FECollection l2fec(0, pmesh.Dimension());
+   mfem::ParFiniteElementSpace l2fespace(&pmesh, &l2fec);
+   mfem::ParGridFunction el_to_refine(&l2fespace);
+
+   mfem::H1_FECollection lhfec(1, pmesh.Dimension());
+   mfem::ParFiniteElementSpace lhfespace(&pmesh, &lhfec);
+   mfem::ParGridFunction lhx(&lhfespace);
+
+   x.ProjectCoefficient(ls_coeff);
+   x.ExchangeFaceNbrData();
+
+   IntegrationRules irRules = IntegrationRules(0, Quadrature1D::GaussLobatto);
+   for (int iter = 0; iter < amr_iter; iter++)
+   {
+      el_to_refine = 0.0;
+      for (int e = 0; e < pmesh.GetNE(); e++)
+      {
+         Array<int> dofs;
+         Vector x_vals;
+         DenseMatrix x_grad;
+         h1fespace.GetElementDofs(e, dofs);
+         const IntegrationRule &ir = irRules.Get(pmesh.GetElementGeometry(e),
+                                                 quad_order);
+         x.GetValues(e, ir, x_vals);
+         double min_val = x_vals.Min();
+         double max_val = x_vals.Max();
+         // If the zero level set cuts the elements, mark it for refinement
+         if (min_val < 0 && max_val >= 0)
+         {
+            el_to_refine(e) = 1.0;
+         }
+      }
+
+      // Refine an element if its neighbor will be refined
+      for (int inner_iter = 0; inner_iter < 2; inner_iter++)
+      {
+         el_to_refine.ExchangeFaceNbrData();
+         GridFunctionCoefficient field_in_dg(&el_to_refine);
+         lhx.ProjectDiscCoefficient(field_in_dg, GridFunction::ARITHMETIC);
+         for (int e = 0; e < pmesh.GetNE(); e++)
+         {
+            Array<int> dofs;
+            Vector x_vals;
+            lhfespace.GetElementDofs(e, dofs);
+            const IntegrationRule &ir =
+               irRules.Get(pmesh.GetElementGeometry(e), quad_order);
+            lhx.GetValues(e, ir, x_vals);
+            double max_val = x_vals.Max();
+            if (max_val > 0)
+            {
+               el_to_refine(e) = 1.0;
+            }
+         }
+      }
+
+      // Make the list of elements to be refined
+      Array<int> el_to_refine_list;
+      for (int e = 0; e < el_to_refine.Size(); e++)
+      {
+         if (el_to_refine(e) > 0.0)
+         {
+            el_to_refine_list.Append(e);
+         }
+      }
+
+      int loc_count = el_to_refine_list.Size();
+      int glob_count = loc_count;
+      MPI_Allreduce(&loc_count, &glob_count, 1, MPI_INT, MPI_SUM,
+                    pmesh.GetComm());
+      MPI_Barrier(pmesh.GetComm());
+      if (glob_count > 0)
+      {
+         pmesh.GeneralRefinement(el_to_refine_list, 1);
+      }
+
+      // Update
+      h1fespace.Update();
+      x.Update();
+      x.ProjectCoefficient(ls_coeff);
+
+      l2fespace.Update();
+      el_to_refine.Update();
+
+      lhfespace.Update();
+      lhx.Update();
+
+      distance_s.ParFESpace()->Update();
+      distance_s.Update();
+
+      if (pgf_to_update != NULL)
+      {
+         for (int i = 0; i < pgf_to_update->Size(); i++)
+         {
+            (*pgf_to_update)[i]->ParFESpace()->Update();
+            (*pgf_to_update)[i]->Update();
+         }
+      }
+   }
+}
+
+void ComputeScalarDistanceFromLevelSet(ParMesh &pmesh,
+                                       FunctionCoefficient &ls_coeff,
+                                       ParGridFunction &distance_s,
+                                       const int nDiffuse = 2,
+                                       const int pLapOrder = 5,
+                                       const int pLapNewton = 50)
+{
+   mfem::H1_FECollection h1fec(distance_s.ParFESpace()->FEColl()->GetOrder(),
+                               pmesh.Dimension());
+   mfem::ParFiniteElementSpace h1fespace(&pmesh, &h1fec);
+   mfem::ParGridFunction x(&h1fespace);
+
+   x.ProjectCoefficient(ls_coeff);
+   x.ExchangeFaceNbrData();
+
+   //Now determine distance
+   const double dx = AvgElementSize(pmesh);
+   PLapDistanceSolver dist_solver(pLapOrder, pLapNewton);
+
+   ParFiniteElementSpace pfes_s(*distance_s.ParFESpace());
+
+   // Smooth-out Gibbs oscillations from the input level set. The smoothing
+   // parameter here is specified to be mesh dependent with length scale dx.
+   ParGridFunction filt_gf(&pfes_s);
+   PDEFilter filter(pmesh, 1.0 * dx);
+   filter.Filter(ls_coeff, filt_gf);
+   GridFunctionCoefficient ls_filt_coeff(&filt_gf);
+
+   dist_solver.ComputeScalarDistance(ls_filt_coeff, distance_s);
+   distance_s.SetTrueVector();
+   distance_s.SetFromTrueVector();
+
+   DiffuseField(distance_s, nDiffuse);
+   distance_s.SetTrueVector();
+   distance_s.SetFromTrueVector();
+}
+#endif

miniapps/meshing/mesh-optimizer.cpp

@@ -69,12 +69,6 @@
 //   Adaptive limiting through FD (requires GSLIB):
 //   * mesh-optimizer -m stretched2D.mesh -o 2 -mid 2 -tid 1 -ni 50 -qo 5 -nor -vl 1 -alc 0.5 -fd -ae 1
 //
-//  Adaptive surface fitting:
-//    mesh-optimizer -m square01.mesh -o 3 -rs 1 -mid 58 -tid 1 -ni 200 -vl 1 -sfc 5e4 -rtol 1e-5
-//    mesh-optimizer -m square01-tri.mesh -o 3 -rs 0 -mid 58 -tid 1 -ni 200 -vl 1 -sfc 1e4 -rtol 1e-5
-//  Surface fitting with weight adaptation and termination based on fitting error
-//    mesh-optimizer -m square01.mesh -o 2 -rs 1 -mid 2 -tid 1 -ni 100 -vl 2 -sfc 10 -rtol 1e-20 -st 0 -sfa -sft 1e-5
-//
 //   Blade shape:
 //     mesh-optimizer -m blade.mesh -o 4 -mid 2 -tid 1 -ni 30 -ls 3 -art 1 -bnd -qt 1 -qo 8
 //     (requires CUDA):
@@ -127,7 +121,6 @@ int main(int argc, char *argv[])
    int target_id         = 1;
    double lim_const      = 0.0;
    double adapt_lim_const   = 0.0;
-   double surface_fit_const = 0.0;
    int quad_type         = 1;
    int quad_order        = 8;
    int solver_type       = 0;
@@ -151,8 +144,6 @@ int main(int argc, char *argv[])
    bool pa               = false;
    int n_hr_iter         = 5;
    int n_h_iter          = 1;
-   bool surface_fit_adapt = false;
-   double surface_fit_threshold = -10;
    int mesh_node_ordering = 0;
    int barrier_type       = 0;
    int worst_case_type    = 0;
@@ -220,8 +211,6 @@ int main(int argc, char *argv[])
    args.AddOption(&lim_const, "-lc", "--limit-const", "Limiting constant.");
    args.AddOption(&adapt_lim_const, "-alc", "--adapt-limit-const",
                   "Adaptive limiting coefficient constant.");
-   args.AddOption(&surface_fit_const, "-sfc", "--surface-fit-const",
-                  "Surface preservation constant.");
    args.AddOption(&quad_type, "-qt", "--quad-type",
                   "Quadrature rule type:\n\t"
                   "1: Gauss-Lobatto\n\t"
@@ -291,12 +280,6 @@ int main(int argc, char *argv[])
    args.AddOption(&n_h_iter, "-nh", "--n_h_iter",
                   "Number of h-adaptivity iterations per r-adaptivity"
                   "iteration.");
-   args.AddOption(&surface_fit_adapt, "-sfa", "--adaptive-surface-fit", "-no-sfa",
-                  "--no-adaptive-surface-fit",
-                  "Enable or disable adaptive surface fitting.");
-   args.AddOption(&surface_fit_threshold, "-sft", "--surf-fit-threshold",
-                  "Set threshold for surface fitting. TMOP solver will"
-                  "terminate when max surface fitting error is below this limit");
    args.AddOption(&mesh_node_ordering, "-mno", "--mesh_node_ordering",
                   "Ordering of mesh nodes."
                   "0 (default): byNodes, 1: byVDIM");
@@ -868,75 +851,6 @@ int main(int argc, char *argv[])
       }
    }
 
-   // Surface fitting.
-   L2_FECollection mat_coll(0, dim);
-   H1_FECollection surf_fit_fec(mesh_poly_deg, dim);
-   FiniteElementSpace surf_fit_fes(mesh, &surf_fit_fec);
-   FiniteElementSpace mat_fes(mesh, &mat_coll);
-   GridFunction mat(&mat_fes);
-   GridFunction surf_fit_mat_gf(&surf_fit_fes);
-   GridFunction surf_fit_gf0(&surf_fit_fes);
-   Array<bool> surf_fit_marker(surf_fit_gf0.Size());
-   ConstantCoefficient surf_fit_coeff(surface_fit_const);
-   AdaptivityEvaluator *adapt_surface = NULL;
-   if (surface_fit_const > 0.0)
-   {
-      MFEM_VERIFY(hradaptivity == false,
-                  "Surface fitting with HR is not implemented yet.");
-      MFEM_VERIFY(pa == false,
-                  "Surface fitting with PA is not implemented yet.");
-
-      FunctionCoefficient ls_coeff(surface_level_set);
-      surf_fit_gf0.ProjectCoefficient(ls_coeff);
-
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         mat(i) = material_id(i, surf_fit_gf0);
-         mesh->SetAttribute(i, static_cast<int>(mat(i) + 1));
-      }
-
-      GridFunctionCoefficient mat_coeff(&mat);
-      surf_fit_mat_gf.ProjectDiscCoefficient(mat_coeff, GridFunction::ARITHMETIC);
-      for (int j = 0; j < surf_fit_marker.Size(); j++)
-      {
-         if (surf_fit_mat_gf(j) > 0.1 && surf_fit_mat_gf(j) < 0.9)
-         {
-            surf_fit_marker[j] = true;
-            surf_fit_mat_gf(j) = 1.0;
-         }
-         else
-         {
-            surf_fit_marker[j] = false;
-            surf_fit_mat_gf(j) = 0.0;
-         }
-      }
-
-      if (adapt_eval == 0) { adapt_surface = new AdvectorCG; }
-      else if (adapt_eval == 1)
-      {
-#ifdef MFEM_USE_GSLIB
-         adapt_surface = new InterpolatorFP;
-#else
-         MFEM_ABORT("MFEM is not built with GSLIB support!");
-#endif
-      }
-      else { MFEM_ABORT("Bad interpolation option."); }
-
-      tmop_integ->EnableSurfaceFitting(surf_fit_gf0, surf_fit_marker,
-                                       surf_fit_coeff, *adapt_surface);
-      if (visualization)
-      {
-         socketstream vis1, vis2, vis3;
-         common::VisualizeField(vis1, "localhost", 19916, surf_fit_gf0, "Level Set 0",
-                                300, 600, 300, 300);
-         common::VisualizeField(vis2, "localhost", 19916, mat, "Materials",
-                                600, 600, 300, 300);
-         common::VisualizeField(vis3, "localhost", 19916, surf_fit_mat_gf,
-                                "Dofs to Move",
-                                900, 600, 300, 300);
-      }
-   }
-
    // Has to be after the enabling of the limiting / alignment, as it computes
    // normalization factors for these terms as well.
    if (normalization) { tmop_integ->EnableNormalization(x0); }
@@ -1033,16 +947,14 @@ int main(int argc, char *argv[])
    const double init_energy = a.GetGridFunctionEnergy(x) /
                               (hradaptivity ? mesh->GetNE() : 1);
    double init_metric_energy = init_energy;
-   if (lim_const > 0.0 || adapt_lim_const > 0.0 || surface_fit_const > 0.0)
+   if (lim_const > 0.0 || adapt_lim_const > 0.0)
    {
       lim_coeff.constant = 0.0;
       adapt_lim_coeff.constant = 0.0;
-      surf_fit_coeff.constant   = 0.0;
       init_metric_energy = a.GetGridFunctionEnergy(x) /
                            (hradaptivity ? mesh->GetNE() : 1);
       lim_coeff.constant = lim_const;
       adapt_lim_coeff.constant = adapt_lim_const;
-      surf_fit_coeff.constant   = surface_fit_const;
    }
 
    // Visualize the starting mesh and metric values.
@@ -1158,11 +1070,6 @@ int main(int argc, char *argv[])
    const IntegrationRule &ir =
       irules->Get(fespace->GetFE(0)->GetGeomType(), quad_order);
    TMOPNewtonSolver solver(ir, solver_type);
-   if (surface_fit_adapt) { solver.EnableAdaptiveSurfaceFitting(); }
-   if (surface_fit_threshold > 0)
-   {
-      solver.SetTerminationWithMaxSurfaceFittingError(surface_fit_threshold);
-   }
    // Provide all integration rules in case of a mixed mesh.
    solver.SetIntegrationRules(*irules, quad_order);
    if (solver_type == 0)
@@ -1216,12 +1123,10 @@ int main(int argc, char *argv[])
    {
       lim_coeff.constant = 0.0;
       adapt_lim_coeff.constant = 0.0;
-      surf_fit_coeff.constant  = 0.0;
       fin_metric_energy = a.GetGridFunctionEnergy(x) /
                           (hradaptivity ? mesh->GetNE() : 1);
       lim_coeff.constant = lim_const;
       adapt_lim_coeff.constant = adapt_lim_const;
-      surf_fit_coeff.constant  = surface_fit_const;
    }
    std::cout << std::scientific << std::setprecision(4);
    cout << "Initial strain energy: " << init_energy
@@ -1247,23 +1152,6 @@ int main(int argc, char *argv[])
                              600, 600, 300, 300);
    }
 
-   // Visualize fitting surfaces and report fitting errors.
-   if (surface_fit_const > 0.0)
-   {
-      if (visualization)
-      {
-         socketstream vis2, vis3;
-         common::VisualizeField(vis2, "localhost", 19916, mat, "Materials",
-                                600, 900, 300, 300);
-         common::VisualizeField(vis3, "localhost", 19916, surf_fit_mat_gf, "Surface dof",
-                                900, 900, 300, 300);
-      }
-      double err_avg, err_max;
-      tmop_integ->GetSurfaceFittingErrors(err_avg, err_max);
-      std::cout << "Avg fitting error: " << err_avg << std::endl
-                << "Max fitting error: " << err_max << std::endl;
-   }
-
    // Visualize the mesh displacement.
    if (visualization)
    {
@@ -1285,7 +1173,6 @@ int main(int argc, char *argv[])
    delete metric2;
    delete metric_coeff1;
    delete adapt_lim_eval;
-   delete adapt_surface;
    delete target_c;
    delete hr_adapt_coeff;
    delete adapt_coeff;

miniapps/meshing/mesh-optimizer.hpp

@@ -375,13 +375,13 @@ double surface_level_set(const Vector &x)
       {
          const double xc = x(0) - 0.5, yc = x(1) - 0.5;
          const double r = sqrt(xc*xc + yc*yc);
-         return std::tanh(2.0*(r-0.3));
+         return r-0.3;
       }
       else
       {
          const double xc = x(0) - 0.5, yc = x(1) - 0.5, zc = x(2) - 0.5;
          const double r = sqrt(xc*xc + yc*yc + zc*zc);
-         return std::tanh(2.0*(r-0.3));
+         return r-0.3;
       }
    }
 }
@@ -397,13 +397,23 @@ int material_id(int el_id, const GridFunction &g)
    double integral = 0.0;
    g.GetValues(el_id, ir, g_vals);
    ElementTransformation *Tr = fes->GetMesh()->GetElementTransformation(el_id);
-   for (int q = 0; q < ir.GetNPoints(); q++)
+   int approach = 1;
+   if (approach == 0)   // integral based
    {
-      const IntegrationPoint &ip = ir.IntPoint(q);
-      Tr->SetIntPoint(&ip);
-      integral += ip.weight * g_vals(q) * Tr->Weight();
+      for (int q = 0; q < ir.GetNPoints(); q++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(q);
+         Tr->SetIntPoint(&ip);
+         integral += ip.weight * g_vals(q) * Tr->Weight();
+      }
+      return (integral > 0.0) ? 1.0 : 0.0;
    }
-   return (integral > 0.0) ? 1.0 : 0.0;
+   else if (approach == 1)   // minimum value based
+   {
+      double minval = g_vals.Min();
+      return minval > 0.0 ? 1.0 : 0.0;
+   }
+   return 0.0;
 }
 
 void DiffuseField(GridFunction &field, int smooth_steps)
@@ -449,6 +459,7 @@ void DiffuseField(ParGridFunction &field, int smooth_steps)
    field.SetFromTrueDofs(fieldtrue);
 
    delete S;
+   delete A;
    delete Lap;
 }
 #endif

miniapps/meshing/pmesh-fitting.cpp

@@ -0,0 +1,887 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//    --------------------------------------------------------------
+//              Boundary and Interface Fitting Miniapp
+//    --------------------------------------------------------------
+//
+// This miniapp performs mesh optimization for controlling mesh quality and
+// aligning a selected set of nodes to boundary and/or interface of interest
+// defined using a level-set function. The mesh quality aspect is based on a
+// variational formulation of the Target-Matrix Optimization Paradigm (TMOP).
+// Boundary/interface alignment is weakly enforced using a penalization term
+// that moves a selected set of nodes towards the zero level set of a signed
+// smooth discrete function. See the following papers for more details:
+// (1) "Adaptive Surface Fitting and Tangential Relaxation for High-Order Mesh Optimization" by
+//     Knupp, Kolev, Mittal, Tomov.
+// (2) "High-Order Mesh Morphing for Boundary and Interface Fitting to Implicit Geometries" by
+//     Barrera, Kolev, Mittal, Tomov.
+// (3) "The target-matrix optimization paradigm for high-order meshes" by
+//     Dobrev, Knupp, Kolev, Mittal, Tomov.
+
+// Compile with: make pmesh-fitting
+// Sample runs:
+//  Interface fitting:
+//    mpirun -np 4 pmesh-fitting -o 3 -mid 58 -tid 1 -ni 200 -vl 1 -sfc 5e4 -rtol 1e-5
+//    mpirun -np 4 pmesh-fitting -m square01-tri.mesh -o 3 -rs 0 -mid 58 -tid 1 -ni 200 -vl 1 -sfc 1e4 -rtol 1e-5
+//  Surface fitting with weight adaptation and termination based on fitting error:
+//    mpirun -np 4 pmesh-fitting -o 2 -mid 2 -tid 1 -ni 100 -vl 2 -sfc 10 -rtol 1e-20 -st 0 -sfa 10.0 -sft 1e-5
+//  Fitting to Fischer-Tropsch reactor like domain (requires GSLIB):
+//  * mpirun -np 6 pmesh-fitting -m ../../data/inline-tri.mesh -o 2 -rs 4 -mid 2 -tid 1 -vl 2 -sfc 100 -rtol 1e-12 -ni 100 -li 40 -ae 1 -bnd -sbgmesh -slstype 2 -smtype 0 -sfa 10.0 -sft 1e-4 -amriter 5 -dist -mod-bndr-attr
+
+#include "mesh-fitting.hpp"
+
+using namespace mfem;
+using namespace std;
+
+int main (int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init(argc, argv);
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Set the method's default parameters.
+   const char *mesh_file = "square01.mesh";
+   int mesh_poly_deg     = 1;
+   int rs_levels         = 1;
+   int rp_levels         = 0;
+   int metric_id         = 2;
+   int target_id         = 1;
+   double surface_fit_const = 100.0;
+   int quad_type         = 1;
+   int quad_order        = 8;
+   int solver_type       = 0;
+   int solver_iter       = 20;
+   double solver_rtol    = 1e-10;
+   int solver_art_type   = 0;
+   int lin_solver        = 2;
+   int max_lin_iter      = 100;
+   bool move_bnd         = true;
+   bool visualization    = true;
+   int verbosity_level   = 0;
+   int adapt_eval        = 0;
+   const char *devopt    = "cpu";
+   double surface_fit_adapt = 0.0;
+   double surface_fit_threshold = -10;
+   bool adapt_marking     = false;
+   bool surf_bg_mesh     = false;
+   bool comp_dist     = false;
+   int surf_ls_type      = 1;
+   int marking_type      = 0;
+   bool mod_bndr_attr    = false;
+   bool material         = false;
+   int mesh_node_ordering = 0;
+   int amr_iters         = 0;
+
+   // 2. Parse command-line options.
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&mesh_poly_deg, "-o", "--order",
+                  "Polynomial degree of mesh finite element space.");
+   args.AddOption(&rs_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&rp_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&metric_id, "-mid", "--metric-id",
+                  "Mesh optimization metric. See list in mesh-optimizer.");
+   args.AddOption(&target_id, "-tid", "--target-id",
+                  "Target (ideal element) type:\n\t"
+                  "1: Ideal shape, unit size\n\t"
+                  "2: Ideal shape, equal size\n\t"
+                  "3: Ideal shape, initial size\n\t"
+                  "4: Given full analytic Jacobian (in physical space)\n\t"
+                  "5: Ideal shape, given size (in physical space)");
+   args.AddOption(&surface_fit_const, "-sfc", "--surface-fit-const",
+                  "Surface preservation constant.");
+   args.AddOption(&quad_type, "-qt", "--quad-type",
+                  "Quadrature rule type:\n\t"
+                  "1: Gauss-Lobatto\n\t"
+                  "2: Gauss-Legendre\n\t"
+                  "3: Closed uniform points");
+   args.AddOption(&quad_order, "-qo", "--quad_order",
+                  "Order of the quadrature rule.");
+   args.AddOption(&solver_type, "-st", "--solver-type",
+                  " Type of solver: (default) 0: Newton, 1: LBFGS");
+   args.AddOption(&solver_iter, "-ni", "--newton-iters",
+                  "Maximum number of Newton iterations.");
+   args.AddOption(&solver_rtol, "-rtol", "--newton-rel-tolerance",
+                  "Relative tolerance for the Newton solver.");
+   args.AddOption(&solver_art_type, "-art", "--adaptive-rel-tol",
+                  "Type of adaptive relative linear solver tolerance:\n\t"
+                  "0: None (default)\n\t"
+                  "1: Eisenstat-Walker type 1\n\t"
+                  "2: Eisenstat-Walker type 2");
+   args.AddOption(&lin_solver, "-ls", "--lin-solver",
+                  "Linear solver:\n\t"
+                  "0: l1-Jacobi\n\t"
+                  "1: CG\n\t"
+                  "2: MINRES\n\t"
+                  "3: MINRES + Jacobi preconditioner\n\t"
+                  "4: MINRES + l1-Jacobi preconditioner");
+   args.AddOption(&max_lin_iter, "-li", "--lin-iter",
+                  "Maximum number of iterations in the linear solve.");
+   args.AddOption(&move_bnd, "-bnd", "--move-boundary", "-fix-bnd",
+                  "--fix-boundary",
+                  "Enable motion along horizontal and vertical boundaries.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&verbosity_level, "-vl", "--verbosity-level",
+                  "Set the verbosity level - 0, 1, or 2.");
+   args.AddOption(&adapt_eval, "-ae", "--adaptivity-evaluator",
+                  "0 - Advection based (DEFAULT), 1 - GSLIB.");
+   args.AddOption(&devopt, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&surface_fit_adapt, "-sfa", "--adaptive-surface-fit",
+                  "Enable or disable adaptive surface fitting.");
+   args.AddOption(&surface_fit_threshold, "-sft", "--surf-fit-threshold",
+                  "Set threshold for surface fitting. TMOP solver will"
+                  "terminate when max surface fitting error is below this limit");
+   args.AddOption(&adapt_marking, "-marking", "--adaptive-marking", "-no-amarking",
+                  "--no-adaptive-marking",
+                  "Enable or disable adaptive marking surface fitting.");
+   args.AddOption(&surf_bg_mesh, "-sbgmesh", "--surf-bg-mesh",
+                  "-no-sbgmesh","--no-surf-bg-mesh",
+                  "Use background mesh for surface fitting.");
+   args.AddOption(&comp_dist, "-dist", "--comp-dist",
+                  "-no-dist","--no-comp-dist",
+                  "Compute distance from 0 level set or not.");
+   args.AddOption(&surf_ls_type, "-slstype", "--surf-ls-type",
+                  "1 - Circle (DEFAULT), 2 - Squircle, 3 - Butterfly.");
+   args.AddOption(&marking_type, "-smtype", "--surf-marking-type",
+                  "1 - Interface (DEFAULT), 2 - Boundary attribute.");
+   args.AddOption(&mod_bndr_attr, "-mod-bndr-attr", "--modify-boundary-attribute",
+                  "-fix-bndr-attr", "--fix-boundary-attribute",
+                  "Change boundary attribue based on alignment with Cartesian axes.");
+   args.AddOption(&material, "-mat", "--mat",
+                  "-no-mat","--no-mat", "Use default material attributes.");
+   args.AddOption(&mesh_node_ordering, "-mno", "--mesh_node_ordering",
+                  "Ordering of mesh nodes."
+                  "0 (default): byNodes, 1: byVDIM");
+   args.AddOption(&amr_iters, "-amriter", "--amr-iter",
+                  "Number of amr iterations on background mesh");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0) { args.PrintUsage(cout); }
+      return 1;
+   }
+   if (myid == 0) { args.PrintOptions(cout); }
+
+   Device device(devopt);
+   if (myid == 0) { device.Print();}
+
+   // 3. Initialize and refine the starting mesh.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1, false);
+   for (int lev = 0; lev < rs_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+   const int dim = mesh->Dimension();
+
+   // Define level-set coefficient
+   FunctionCoefficient *ls_coeff = NULL;
+   if (surf_ls_type == 1) //Circle
+   {
+      ls_coeff = new FunctionCoefficient(circle_level_set);
+   }
+   else if (surf_ls_type == 2) // reactor
+   {
+      ls_coeff = new FunctionCoefficient(reactor);
+   }
+   else if (surf_ls_type == 6) // 3D shape
+   {
+      ls_coeff = new FunctionCoefficient(csg_cubecylsph);
+   }
+   else
+   {
+      MFEM_ABORT("Surface fitting level set type not implemented yet.")
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < rp_levels; lev++) { pmesh->UniformRefinement(); }
+
+   // 4. Setup background mesh for surface fitting
+   ParMesh *pmesh_surf_fit_bg = NULL;
+   if (surf_bg_mesh)
+   {
+      Mesh *mesh_surf_fit_bg = NULL;
+      if (dim == 2)
+      {
+         mesh_surf_fit_bg =
+            new Mesh(Mesh::MakeCartesian2D(4, 4, Element::QUADRILATERAL, true));
+      }
+      else if (dim == 3)
+      {
+         mesh_surf_fit_bg =
+            new Mesh(Mesh::MakeCartesian3D(4, 4, 4, Element::HEXAHEDRON, true));
+      }
+      mesh_surf_fit_bg->EnsureNCMesh();
+      pmesh_surf_fit_bg = new ParMesh(MPI_COMM_WORLD, *mesh_surf_fit_bg);
+      delete mesh_surf_fit_bg;
+   }
+
+   // 5. Define a finite element space on the mesh. Here we use vector finite
+   //    elements which are tensor products of quadratic finite elements. The
+   //    number of components in the vector finite element space is specified by
+   //    the last parameter of the FiniteElementSpace constructor.
+   FiniteElementCollection *fec;
+   if (mesh_poly_deg <= 0)
+   {
+      fec = new QuadraticPosFECollection;
+      mesh_poly_deg = 2;
+   }
+   else { fec = new H1_FECollection(mesh_poly_deg, dim); }
+   ParFiniteElementSpace *pfespace =
+      new ParFiniteElementSpace(pmesh, fec, dim, mesh_node_ordering);
+
+   // 6. Make the mesh curved based on the above finite element space. This
+   //    means that we define the mesh elements through a fespace-based
+   //    transformation of the reference element.
+   pmesh->SetNodalFESpace(pfespace);
+
+   // 7. Get the mesh nodes (vertices and other degrees of freedom in the finite
+   //    element space) as a finite element grid function in fespace. Note that
+   //    changing x automatically changes the shapes of the mesh elements.
+   ParGridFunction x(pfespace);
+   pmesh->SetNodalGridFunction(&x);
+   x.SetTrueVector();
+
+   // 10. Save the starting (prior to the optimization) mesh to a file. This
+   //     output can be viewed later using GLVis: "glvis -m perturbed -np
+   //     num_mpi_tasks".
+   {
+      ostringstream mesh_name;
+      mesh_name << "perturbed.mesh";
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->PrintAsSerial(mesh_ofs);
+   }
+
+   // 11. Store the starting (prior to the optimization) positions.
+   ParGridFunction x0(pfespace);
+   x0 = x;
+
+   // 12. Form the integrator that uses the chosen metric and target.
+   TMOP_QualityMetric *metric = NULL;
+   switch (metric_id)
+   {
+      // T-metrics
+      case 2: metric = new TMOP_Metric_002; break;
+      case 58: metric = new TMOP_Metric_058; break;
+      case 80: metric = new TMOP_Metric_080(0.5); break;
+      case 303: metric = new TMOP_Metric_303; break;
+      case 328: metric = new TMOP_Metric_328(0.5); break;
+      default:
+         if (myid == 0) { cout << "Unknown metric_id: " << metric_id << endl; }
+         return 3;
+   }
+
+   if (metric_id < 300)
+   {
+      MFEM_VERIFY(dim == 2, "Incompatible metric for 3D meshes");
+   }
+   if (metric_id >= 300)
+   {
+      MFEM_VERIFY(dim == 3, "Incompatible metric for 2D meshes");
+   }
+
+   TargetConstructor::TargetType target_t;
+   TargetConstructor *target_c = NULL;
+   switch (target_id)
+   {
+      case 1: target_t = TargetConstructor::IDEAL_SHAPE_UNIT_SIZE; break;
+      case 2: target_t = TargetConstructor::IDEAL_SHAPE_EQUAL_SIZE; break;
+      case 3: target_t = TargetConstructor::IDEAL_SHAPE_GIVEN_SIZE; break;
+      case 4: target_t = TargetConstructor::GIVEN_SHAPE_AND_SIZE; break;
+      default:
+         if (myid == 0) { cout << "Unknown target_id: " << target_id << endl; }
+         return 3;
+   }
+
+   if (target_c == NULL)
+   {
+      target_c = new TargetConstructor(target_t, MPI_COMM_WORLD);
+   }
+   target_c->SetNodes(x0);
+   TMOP_Integrator *tmop_integ = new TMOP_Integrator(metric, target_c);
+
+   // Setup the quadrature rules for the TMOP integrator.
+   IntegrationRules *irules = NULL;
+   switch (quad_type)
+   {
+      case 1: irules = &IntRulesLo; break;
+      case 2: irules = &IntRules; break;
+      case 3: irules = &IntRulesCU; break;
+      default:
+         if (myid == 0) { cout << "Unknown quad_type: " << quad_type << endl; }
+         return 3;
+   }
+   tmop_integ->SetIntegrationRules(*irules, quad_order);
+   if (myid == 0 && dim == 2)
+   {
+      cout << "Triangle quadrature points: "
+           << irules->Get(Geometry::TRIANGLE, quad_order).GetNPoints()
+           << "\nQuadrilateral quadrature points: "
+           << irules->Get(Geometry::SQUARE, quad_order).GetNPoints() << endl;
+   }
+   if (myid == 0 && dim == 3)
+   {
+      cout << "Tetrahedron quadrature points: "
+           << irules->Get(Geometry::TETRAHEDRON, quad_order).GetNPoints()
+           << "\nHexahedron quadrature points: "
+           << irules->Get(Geometry::CUBE, quad_order).GetNPoints()
+           << "\nPrism quadrature points: "
+           << irules->Get(Geometry::PRISM, quad_order).GetNPoints() << endl;
+   }
+
+   // Modify boundary attribute for surface node movement
+   // Sets attributes of a boundary element to 1/2/3 if it is parallel to x/y/z.
+   if (mod_bndr_attr)
+   {
+      ModifyBoundaryAttributesForNodeMovement(pmesh, x);
+      pmesh->SetAttributes();
+   }
+   pmesh->ExchangeFaceNbrData();
+
+
+   // Surface fitting.
+   L2_FECollection mat_coll(0, dim);
+   H1_FECollection surf_fit_fec(mesh_poly_deg, dim);
+   ParFiniteElementSpace surf_fit_fes(pmesh, &surf_fit_fec);
+   ParFiniteElementSpace mat_fes(pmesh, &mat_coll);
+   ParGridFunction mat(&mat_fes);
+   ParGridFunction surf_fit_mat_gf(&surf_fit_fes);
+   ParGridFunction surf_fit_gf0(&surf_fit_fes);
+   Array<bool> surf_fit_marker(surf_fit_gf0.Size());
+   ConstantCoefficient surf_fit_coeff(surface_fit_const);
+   AdaptivityEvaluator *adapt_surface = NULL;
+   AdaptivityEvaluator *adapt_grad_surface = NULL;
+   AdaptivityEvaluator *adapt_hess_surface = NULL;
+
+   // Background mesh FECollection, FESpace, and GridFunction
+   H1_FECollection *surf_fit_bg_fec = NULL;
+   ParFiniteElementSpace *surf_fit_bg_fes = NULL;
+   ParGridFunction *surf_fit_bg_gf0 = NULL;
+   ParFiniteElementSpace *surf_fit_bg_grad_fes = NULL;
+   ParGridFunction *surf_fit_bg_grad = NULL;
+   ParFiniteElementSpace *surf_fit_bg_hess_fes = NULL;
+   ParGridFunction *surf_fit_bg_hess = NULL;
+
+   // If a background mesh is used, we interpolate the Gradient and Hessian
+   // from that mesh to the current mesh being optimized.
+   ParFiniteElementSpace *surf_fit_grad_fes = NULL;
+   ParGridFunction *surf_fit_grad = NULL;
+   ParFiniteElementSpace *surf_fit_hess_fes = NULL;
+   ParGridFunction *surf_fit_hess = NULL;
+
+   if (surf_bg_mesh)
+   {
+      pmesh_surf_fit_bg->SetCurvature(mesh_poly_deg);
+
+      Vector p_min(dim), p_max(dim);
+      pmesh->GetBoundingBox(p_min, p_max);
+      GridFunction &x_bg = *pmesh_surf_fit_bg->GetNodes();
+      const int num_nodes = x_bg.Size() / dim;
+      for (int i = 0; i < num_nodes; i++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            double length_d = p_max(d) - p_min(d),
+                   extra_d = 0.2 * length_d;
+            x_bg(i + d*num_nodes) = p_min(d) - extra_d +
+                                    x_bg(i + d*num_nodes) * (length_d + 2*extra_d);
+         }
+      }
+      surf_fit_bg_fec = new H1_FECollection(mesh_poly_deg+1, dim);
+      surf_fit_bg_fes = new ParFiniteElementSpace(pmesh_surf_fit_bg, surf_fit_bg_fec);
+      surf_fit_bg_gf0 = new ParGridFunction(surf_fit_bg_fes);
+   }
+
+   Array<int> vdofs;
+   if (surface_fit_const > 0.0)
+   {
+      surf_fit_gf0.ProjectCoefficient(*ls_coeff);
+      if (surf_bg_mesh)
+      {
+         OptimizeMeshWithAMRAroundZeroLevelSet(*pmesh_surf_fit_bg, *ls_coeff,
+                                               amr_iters, *surf_fit_bg_gf0);
+         pmesh_surf_fit_bg->Rebalance();
+         surf_fit_bg_fes->Update();
+         surf_fit_bg_gf0->Update();
+
+         if (comp_dist)
+         {
+            ComputeScalarDistanceFromLevelSet(*pmesh_surf_fit_bg, *ls_coeff,
+                                              *surf_fit_bg_gf0);
+         }
+         else { surf_fit_bg_gf0->ProjectCoefficient(*ls_coeff); }
+
+
+         surf_fit_bg_grad_fes =
+            new ParFiniteElementSpace(pmesh_surf_fit_bg, surf_fit_bg_fec, dim);
+         surf_fit_bg_grad = new ParGridFunction(surf_fit_bg_grad_fes);
+
+         surf_fit_grad_fes =
+            new ParFiniteElementSpace(pmesh, &surf_fit_fec, dim);
+         surf_fit_grad = new ParGridFunction(surf_fit_grad_fes);
+
+         surf_fit_bg_hess_fes =
+            new ParFiniteElementSpace(pmesh_surf_fit_bg, surf_fit_bg_fec, dim * dim);
+         surf_fit_bg_hess = new ParGridFunction(surf_fit_bg_hess_fes);
+
+         surf_fit_hess_fes =
+            new ParFiniteElementSpace(pmesh, &surf_fit_fec, dim * dim);
+         surf_fit_hess = new ParGridFunction(surf_fit_hess_fes);
+
+         //Setup gradient of the background mesh
+         const int size_bg = surf_fit_bg_gf0->Size();
+         for (int d = 0; d < pmesh_surf_fit_bg->Dimension(); d++)
+         {
+            ParGridFunction surf_fit_bg_grad_comp(
+               surf_fit_bg_fes, surf_fit_bg_grad->GetData() + d * size_bg);
+            surf_fit_bg_gf0->GetDerivative(1, d, surf_fit_bg_grad_comp);
+         }
+
+         //Setup Hessian on background mesh
+         int id = 0;
+         for (int d = 0; d < pmesh_surf_fit_bg->Dimension(); d++)
+         {
+            for (int idir = 0; idir < pmesh_surf_fit_bg->Dimension(); idir++)
+            {
+               ParGridFunction surf_fit_bg_grad_comp(
+                  surf_fit_bg_fes, surf_fit_bg_grad->GetData() + d * size_bg);
+               ParGridFunction surf_fit_bg_hess_comp(
+                  surf_fit_bg_fes, surf_fit_bg_hess->GetData()+ id * size_bg);
+               surf_fit_bg_grad_comp.GetDerivative(1, idir,
+                                                   surf_fit_bg_hess_comp);
+               id++;
+            }
+         }
+      }
+      else // !surf_bg_mesh
+      {
+         if (comp_dist)
+         {
+            ComputeScalarDistanceFromLevelSet(*pmesh, *ls_coeff, surf_fit_gf0);
+         }
+      }
+
+      // Set material gridfunction
+      for (int i = 0; i < pmesh->GetNE(); i++)
+      {
+         if (material)
+         {
+            mat(i) = pmesh->GetAttribute(i)-1;
+         }
+         else
+         {
+            mat(i) = material_id(i, surf_fit_gf0);
+            pmesh->SetAttribute(i, mat(i) + 1);
+         }
+      }
+
+      // Adapt attributes for marking such that if all but 1 face of an element
+      // are marked, the element attribute is switched.
+      if (adapt_marking)
+      {
+         ModifyAttributeForMarkingDOFS(pmesh, mat, 0);
+         ModifyAttributeForMarkingDOFS(pmesh, mat, 1);
+      }
+
+      GridFunctionCoefficient coeff_mat(&mat);
+      surf_fit_mat_gf.ProjectDiscCoefficient(coeff_mat,
+                                             GridFunction::ARITHMETIC);
+      surf_fit_mat_gf.SetTrueVector();
+      surf_fit_mat_gf.SetFromTrueVector();
+
+      // Set DOFs for fitting
+      // Strategy 1: Choose face between elements of different attributes.
+      if (marking_type == 0)
+      {
+         mat.ExchangeFaceNbrData();
+         const Vector &FaceNbrData = mat.FaceNbrData();
+         for (int j = 0; j < surf_fit_marker.Size(); j++)
+         {
+            surf_fit_marker[j] = false;
+         }
+         surf_fit_mat_gf = 0.0;
+
+         Array<int> dof_list;
+         Array<int> dofs;
+         for (int i = 0; i < pmesh->GetNumFaces(); i++)
+         {
+            auto tr = pmesh->GetInteriorFaceTransformations(i);
+            if (tr != NULL)
+            {
+               int mat1 = mat(tr->Elem1No);
+               int mat2 = mat(tr->Elem2No);
+               if (mat1 != mat2)
+               {
+                  surf_fit_gf0.ParFESpace()->GetFaceDofs(i, dofs);
+                  dof_list.Append(dofs);
+               }
+            }
+         }
+         for (int i = 0; i < pmesh->GetNSharedFaces(); i++)
+         {
+            auto tr = pmesh->GetSharedFaceTransformations(i);
+            if (tr != NULL)
+            {
+               int faceno = pmesh->GetSharedFace(i);
+               int mat1 = mat(tr->Elem1No);
+               int mat2 = FaceNbrData(tr->Elem2No-pmesh->GetNE());
+               if (mat1 != mat2)
+               {
+                  surf_fit_gf0.ParFESpace()->GetFaceDofs(faceno, dofs);
+                  dof_list.Append(dofs);
+               }
+            }
+         }
+         for (int i = 0; i < dof_list.Size(); i++)
+         {
+            surf_fit_marker[dof_list[i]] = true;
+            surf_fit_mat_gf(dof_list[i]) = 1.0;
+         }
+      }
+      // Strategy 2: Mark all boundaries with attribute marking_type
+      else if (marking_type > 0)
+      {
+         for (int i = 0; i < pmesh->GetNBE(); i++)
+         {
+            const int attr = pmesh->GetBdrElement(i)->GetAttribute();
+            if (attr == marking_type)
+            {
+               surf_fit_fes.GetBdrElementVDofs(i, vdofs);
+               for (int j = 0; j < vdofs.Size(); j++)
+               {
+                  surf_fit_marker[vdofs[j]] = true;
+                  surf_fit_mat_gf(vdofs[j]) = 1.0;
+               }
+            }
+         }
+      }
+
+      // Set AdaptivityEvaluators for transferring information from initial
+      // mesh to current mesh as it moves during adaptivity.
+      if (adapt_eval == 0)
+      {
+         adapt_surface = new AdvectorCG;
+         MFEM_VERIFY(!surf_bg_mesh, "Background meshes require GSLIB.");
+      }
+      else if (adapt_eval == 1)
+      {
+#ifdef MFEM_USE_GSLIB
+         adapt_surface = new InterpolatorFP;
+         if (surf_bg_mesh)
+         {
+            adapt_grad_surface = new InterpolatorFP;
+            adapt_hess_surface = new InterpolatorFP;
+         }
+#else
+         MFEM_ABORT("MFEM is not built with GSLIB support!");
+#endif
+      }
+      else { MFEM_ABORT("Bad interpolation option."); }
+
+      if (!surf_bg_mesh)
+      {
+         tmop_integ->EnableSurfaceFitting(surf_fit_gf0, surf_fit_marker,
+                                          surf_fit_coeff,
+                                          *adapt_surface);
+      }
+      else
+      {
+         tmop_integ->EnableSurfaceFittingFromSource(
+            *surf_fit_bg_gf0, surf_fit_gf0,
+            surf_fit_marker, surf_fit_coeff, *adapt_surface,
+            *surf_fit_bg_grad, *surf_fit_grad, *adapt_grad_surface,
+            *surf_fit_bg_hess, *surf_fit_hess, *adapt_hess_surface);
+      }
+
+      if (visualization)
+      {
+         socketstream vis1, vis2, vis3, vis4, vis5;
+         common::VisualizeField(vis1, "localhost", 19916, surf_fit_gf0,
+                                "Level Set", 0, 0, 300, 300);
+         common::VisualizeField(vis2, "localhost", 19916, mat,
+                                "Materials", 300, 0, 300, 300);
+         common::VisualizeField(vis3, "localhost", 19916, surf_fit_mat_gf,
+                                "Surface DOFs", 600, 0, 300, 300);
+         if (surf_bg_mesh)
+         {
+            common::VisualizeField(vis4, "localhost", 19916, *surf_fit_bg_gf0,
+                                   "Level Set - Background",
+                                   0, 400, 300, 300);
+         }
+      }
+   }
+
+   // 13. Setup the final NonlinearForm (which defines the integral of interest,
+   //     its first and second derivatives). Here we can use a combination of
+   //     metrics, i.e., optimize the sum of two integrals, where both are
+   //     scaled by used-defined space-dependent weights.  Note that there are
+   //     no command-line options for the weights and the type of the second
+   //     metric; one should update those in the code.
+   ParNonlinearForm a(pfespace);
+   ConstantCoefficient *metric_coeff1 = NULL;
+   a.AddDomainIntegrator(tmop_integ);
+
+   // Compute the minimum det(J) of the starting mesh.
+   double min_detJ = infinity();
+   const int NE = pmesh->GetNE();
+   for (int i = 0; i < NE; i++)
+   {
+      const IntegrationRule &ir =
+         irules->Get(pfespace->GetFE(i)->GetGeomType(), quad_order);
+      ElementTransformation *transf = pmesh->GetElementTransformation(i);
+      for (int j = 0; j < ir.GetNPoints(); j++)
+      {
+         transf->SetIntPoint(&ir.IntPoint(j));
+         min_detJ = min(min_detJ, transf->Jacobian().Det());
+      }
+   }
+   MPI_Allreduce(MPI_IN_PLACE, &min_detJ, 1,
+                 MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
+   if (myid == 0)
+   { cout << "Minimum det(J) of the original mesh is " << min_detJ << endl; }
+
+   MFEM_VERIFY(min_detJ > 0, "The input mesh is inverted, use mesh-optimizer.");
+
+   const double init_energy = a.GetParGridFunctionEnergy(x);
+   double init_metric_energy = init_energy;
+   if (surface_fit_const > 0.0)
+   {
+      surf_fit_coeff.constant   = 0.0;
+      init_metric_energy = a.GetParGridFunctionEnergy(x);
+      surf_fit_coeff.constant  = surface_fit_const;
+   }
+
+   // 14. Fix all boundary nodes, or fix only a given component depending on the
+   //     boundary attributes of the given mesh.  Attributes 1/2/3 correspond to
+   //     fixed x/y/z components of the node.  Attribute dim+1 corresponds to
+   //     an entirely fixed node.
+   if (move_bnd == false)
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      if (marking_type > 0)
+      {
+         ess_bdr[marking_type-1] = 0;
+      }
+      a.SetEssentialBC(ess_bdr);
+   }
+   else
+   {
+      int n = 0;
+      for (int i = 0; i < pmesh->GetNBE(); i++)
+      {
+         const int nd = pfespace->GetBE(i)->GetDof();
+         const int attr = pmesh->GetBdrElement(i)->GetAttribute();
+         MFEM_VERIFY(!(dim == 2 && attr == 3),
+                     "Boundary attribute 3 must be used only for 3D meshes. "
+                     "Adjust the attributes (1/2/3/4 for fixed x/y/z/all "
+                     "components, rest for free nodes), or use -fix-bnd.");
+         if (attr == 1 || attr == 2 || attr == 3) { n += nd; }
+         if (attr == 4) { n += nd * dim; }
+      }
+      Array<int> ess_vdofs(n);
+      n = 0;
+      for (int i = 0; i < pmesh->GetNBE(); i++)
+      {
+         const int nd = pfespace->GetBE(i)->GetDof();
+         const int attr = pmesh->GetBdrElement(i)->GetAttribute();
+         pfespace->GetBdrElementVDofs(i, vdofs);
+         if (attr == 1) // Fix x components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j]; }
+         }
+         else if (attr == 2) // Fix y components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j+nd]; }
+         }
+         else if (attr == 3) // Fix z components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j+2*nd]; }
+         }
+         else if (attr == 4) // Fix all components.
+         {
+            for (int j = 0; j < vdofs.Size(); j++)
+            { ess_vdofs[n++] = vdofs[j]; }
+         }
+      }
+      a.SetEssentialVDofs(ess_vdofs);
+   }
+
+   // 15. As we use the Newton method to solve the resulting nonlinear system,
+   //     here we setup the linear solver for the system's Jacobian.
+   Solver *S = NULL, *S_prec = NULL;
+   const double linsol_rtol = 1e-12;
+   if (lin_solver == 0)
+   {
+      S = new DSmoother(1, 1.0, max_lin_iter);
+   }
+   else if (lin_solver == 1)
+   {
+      CGSolver *cg = new CGSolver(MPI_COMM_WORLD);
+      cg->SetMaxIter(max_lin_iter);
+      cg->SetRelTol(linsol_rtol);
+      cg->SetAbsTol(0.0);
+      cg->SetPrintLevel(verbosity_level >= 2 ? 3 : -1);
+      S = cg;
+   }
+   else
+   {
+      MINRESSolver *minres = new MINRESSolver(MPI_COMM_WORLD);
+      minres->SetMaxIter(max_lin_iter);
+      minres->SetRelTol(linsol_rtol);
+      minres->SetAbsTol(0.0);
+      if (verbosity_level > 2) { minres->SetPrintLevel(1); }
+      else { minres->SetPrintLevel(verbosity_level == 2 ? 3 : -1); }
+      if (lin_solver == 3 || lin_solver == 4)
+      {
+         auto hs = new HypreSmoother;
+         hs->SetType((lin_solver == 3) ? HypreSmoother::Jacobi
+                     /* */             : HypreSmoother::l1Jacobi, 1);
+         hs->SetPositiveDiagonal(true);
+         S_prec = hs;
+         minres->SetPreconditioner(*S_prec);
+      }
+      S = minres;
+   }
+
+   // Perform the nonlinear optimization.
+   const IntegrationRule &ir =
+      irules->Get(pfespace->GetFE(0)->GetGeomType(), quad_order);
+   TMOPNewtonSolver solver(pfespace->GetComm(), ir, solver_type);
+   if (surface_fit_adapt > 0.0)
+   {
+      solver.SetAdaptiveSurfaceFittingScalingFactor(surface_fit_adapt);
+   }
+   if (surface_fit_threshold > 0)
+   {
+      solver.SetTerminationWithMaxSurfaceFittingError(surface_fit_threshold);
+   }
+   // Provide all integration rules in case of a mixed mesh.
+   solver.SetIntegrationRules(*irules, quad_order);
+   if (solver_type == 0)
+   {
+      // Specify linear solver when we use a Newton-based solver.
+      solver.SetPreconditioner(*S);
+   }
+   solver.SetMaxIter(solver_iter);
+   solver.SetRelTol(solver_rtol);
+   solver.SetAbsTol(0.0);
+   solver.SetMinimumDeterminantThreshold(0.001*min_detJ);
+   if (solver_art_type > 0)
+   {
+      solver.SetAdaptiveLinRtol(solver_art_type, 0.5, 0.9);
+   }
+   solver.SetPrintLevel(verbosity_level >= 1 ? 1 : -1);
+   solver.SetOperator(a);
+   Vector b(0);
+   solver.Mult(b, x.GetTrueVector());
+   x.SetFromTrueVector();
+
+   // 16. Save the optimized mesh to a file. This output can be viewed later
+   //     using GLVis: "glvis -m optimized -np num_mpi_tasks".
+   {
+      ostringstream mesh_name;
+      mesh_name << "optimized.mesh";
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->PrintAsSerial(mesh_ofs);
+   }
+
+   // Compute the final energy of the functional.
+   const double fin_energy = a.GetParGridFunctionEnergy(x);
+   double fin_metric_energy = fin_energy;
+   if (surface_fit_const > 0.0)
+   {
+      surf_fit_coeff.constant  = 0.0;
+      fin_metric_energy  = a.GetParGridFunctionEnergy(x);
+      surf_fit_coeff.constant  = surface_fit_const;
+   }
+
+   if (myid == 0)
+   {
+      std::cout << std::scientific << std::setprecision(4);
+      cout << "Initial strain energy: " << init_energy
+           << " = metrics: " << init_metric_energy
+           << " + extra terms: " << init_energy - init_metric_energy << endl;
+      cout << "  Final strain energy: " << fin_energy
+           << " = metrics: " << fin_metric_energy
+           << " + extra terms: " << fin_energy - fin_metric_energy << endl;
+      cout << "The strain energy decreased by: "
+           << (init_energy - fin_energy) * 100.0 / init_energy << " %." << endl;
+   }
+
+   if (surface_fit_const > 0.0)
+   {
+      if (visualization)
+      {
+         socketstream vis2, vis3;
+         common::VisualizeField(vis2, "localhost", 19916, mat,
+                                "Materials", 300, 400, 300, 300);
+         common::VisualizeField(vis3, "localhost", 19916, surf_fit_mat_gf,
+                                "Surface DOFs", 600, 400, 300, 300);
+      }
+      double err_avg, err_max;
+      tmop_integ->GetSurfaceFittingErrors(err_avg, err_max);
+      if (myid == 0)
+      {
+         std::cout << "Avg fitting error: " << err_avg << std::endl
+                   << "Max fitting error: " << err_max << std::endl;
+      }
+   }
+
+   // 18. Visualize the mesh displacement.
+   if (visualization)
+   {
+      x0 -= x;
+      socketstream vis;
+      common::VisualizeField(vis, "localhost", 19916, x0,
+                             "Displacements", 900, 400, 300, 300, "jRmclA");
+   }
+
+   delete S;
+   delete S_prec;
+   delete metric_coeff1;
+   delete adapt_surface;
+   delete adapt_grad_surface;
+   delete adapt_hess_surface;
+   delete ls_coeff;
+   delete surf_fit_hess;
+   delete surf_fit_hess_fes;
+   delete surf_fit_bg_hess;
+   delete surf_fit_bg_hess_fes;
+   delete surf_fit_grad;
+   delete surf_fit_grad_fes;
+   delete surf_fit_bg_grad;
+   delete surf_fit_bg_grad_fes;
+   delete surf_fit_bg_gf0;
+   delete surf_fit_bg_fes;
+   delete surf_fit_bg_fec;
+   delete target_c;
+   delete metric;
+   delete pfespace;
+   delete fec;
+   delete pmesh_surf_fit_bg;
+   delete pmesh;
+
+   return 0;
+}

miniapps/meshing/pmesh-optimizer.cpp

@@ -69,12 +69,6 @@
 //   Adaptive limiting through FD (requires GSLIB):
 //   * mpirun -np 4 pmesh-optimizer -m stretched2D.mesh -o 2 -mid 2 -tid 1 -ni 50 -qo 5 -nor -vl 1 -alc 0.5 -fd -ae 1
 //
-//  Adaptive surface fitting:
-//    mpirun -np 4 pmesh-optimizer -m square01.mesh -o 3 -rs 1 -mid 58 -tid 1 -ni 200 -vl 1 -sfc 5e4 -rtol 1e-5
-//    mpirun -np 4 pmesh-optimizer -m square01-tri.mesh -o 3 -rs 0 -mid 58 -tid 1 -ni 200 -vl 1 -sfc 1e4 -rtol 1e-5
-//  Surface fitting with weight adaptation and termination based on fitting error
-//    mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 1 -mid 2 -tid 1 -ni 100 -vl 2 -sfc 10 -rtol 1e-20 -st 0 -sfa -sft 1e-5
-//
 //   Blade shape:
 //     mpirun -np 4 pmesh-optimizer -m blade.mesh -o 4 -mid 2 -tid 1 -ni 30 -ls 3 -art 1 -bnd -qt 1 -qo 8
 //     (requires CUDA):
@@ -137,7 +131,6 @@ int main (int argc, char *argv[])
    int target_id         = 1;
    double lim_const      = 0.0;
    double adapt_lim_const   = 0.0;
-   double surface_fit_const = 0.0;
    int quad_type         = 1;
    int quad_order        = 8;
    int solver_type       = 0;
@@ -161,8 +154,6 @@ int main (int argc, char *argv[])
    bool pa               = false;
    int n_hr_iter         = 5;
    int n_h_iter          = 1;
-   bool surface_fit_adapt = false;
-   double surface_fit_threshold = -10;
    int mesh_node_ordering = 0;
    int barrier_type       = 0;
    int worst_case_type    = 0;
@@ -232,8 +223,6 @@ int main (int argc, char *argv[])
    args.AddOption(&lim_const, "-lc", "--limit-const", "Limiting constant.");
    args.AddOption(&adapt_lim_const, "-alc", "--adapt-limit-const",
                   "Adaptive limiting coefficient constant.");
-   args.AddOption(&surface_fit_const, "-sfc", "--surface-fit-const",
-                  "Surface preservation constant.");
    args.AddOption(&quad_type, "-qt", "--quad-type",
                   "Quadrature rule type:\n\t"
                   "1: Gauss-Lobatto\n\t"
@@ -303,12 +292,6 @@ int main (int argc, char *argv[])
    args.AddOption(&n_h_iter, "-nh", "--n_h_iter",
                   "Number of h-adaptivity iterations per r-adaptivity"
                   "iteration.");
-   args.AddOption(&surface_fit_adapt, "-sfa", "--adaptive-surface-fit", "-no-sfa",
-                  "--no-adaptive-surface-fit",
-                  "Enable or disable adaptive surface fitting.");
-   args.AddOption(&surface_fit_threshold, "-sft", "--surf-fit-threshold",
-                  "Set threshold for surface fitting. TMOP solver will"
-                  "terminate when max surface fitting error is below this limit");
    args.AddOption(&mesh_node_ordering, "-mno", "--mesh_node_ordering",
                   "Ordering of mesh nodes."
                   "0 (default): byNodes, 1: byVDIM");
@@ -903,75 +886,6 @@ int main (int argc, char *argv[])
       }
    }
 
-   // Surface fitting.
-   L2_FECollection mat_coll(0, dim);
-   H1_FECollection surf_fit_fec(mesh_poly_deg, dim);
-   ParFiniteElementSpace surf_fit_fes(pmesh, &surf_fit_fec);
-   ParFiniteElementSpace mat_fes(pmesh, &mat_coll);
-   ParGridFunction mat(&mat_fes);
-   ParGridFunction surf_fit_mat_gf(&surf_fit_fes);
-   ParGridFunction surf_fit_gf0(&surf_fit_fes);
-   Array<bool> surf_fit_marker(surf_fit_gf0.Size());
-   ConstantCoefficient surf_fit_coeff(surface_fit_const);
-   AdaptivityEvaluator *adapt_surface = NULL;
-   if (surface_fit_const > 0.0)
-   {
-      MFEM_VERIFY(hradaptivity == false,
-                  "Surface fitting with HR is not implemented yet.");
-      MFEM_VERIFY(pa == false,
-                  "Surface fitting with PA is not implemented yet.");
-
-      FunctionCoefficient ls_coeff(surface_level_set);
-      surf_fit_gf0.ProjectCoefficient(ls_coeff);
-
-      for (int i = 0; i < pmesh->GetNE(); i++)
-      {
-         mat(i) = material_id(i, surf_fit_gf0);
-         pmesh->SetAttribute(i, static_cast<int>(mat(i) + 1));
-      }
-
-      GridFunctionCoefficient coeff_mat(&mat);
-      surf_fit_mat_gf.ProjectDiscCoefficient(coeff_mat, GridFunction::ARITHMETIC);
-      for (int j = 0; j < surf_fit_marker.Size(); j++)
-      {
-         if (surf_fit_mat_gf(j) > 0.1 && surf_fit_mat_gf(j) < 0.9)
-         {
-            surf_fit_marker[j] = true;
-            surf_fit_mat_gf(j) = 1.0;
-         }
-         else
-         {
-            surf_fit_marker[j] = false;
-            surf_fit_mat_gf(j) = 0.0;
-         }
-      }
-
-      if (adapt_eval == 0) { adapt_surface = new AdvectorCG; }
-      else if (adapt_eval == 1)
-      {
-#ifdef MFEM_USE_GSLIB
-         adapt_surface = new InterpolatorFP;
-#else
-         MFEM_ABORT("MFEM is not built with GSLIB support!");
-#endif
-      }
-      else { MFEM_ABORT("Bad interpolation option."); }
-
-      tmop_integ->EnableSurfaceFitting(surf_fit_gf0, surf_fit_marker, surf_fit_coeff,
-                                       *adapt_surface);
-      if (visualization)
-      {
-         socketstream vis1, vis2, vis3;
-         common::VisualizeField(vis1, "localhost", 19916, surf_fit_gf0, "Level Set 0",
-                                300, 600, 300, 300);
-         common::VisualizeField(vis2, "localhost", 19916, mat, "Materials",
-                                600, 600, 300, 300);
-         common::VisualizeField(vis3, "localhost", 19916, surf_fit_mat_gf,
-                                "Dofs to Move",
-                                900, 600, 300, 300);
-      }
-   }
-
    // Has to be after the enabling of the limiting / alignment, as it computes
    // normalization factors for these terms as well.
    if (normalization) { tmop_integ->ParEnableNormalization(x0); }
@@ -1074,16 +988,14 @@ int main (int argc, char *argv[])
    const double init_energy = a.GetParGridFunctionEnergy(x) /
                               (hradaptivity ? pmesh->GetGlobalNE() : 1);
    double init_metric_energy = init_energy;
-   if (lim_const > 0.0 || adapt_lim_const > 0.0 || surface_fit_const > 0.0)
+   if (lim_const > 0.0 || adapt_lim_const > 0.0)
    {
       lim_coeff.constant = 0.0;
       adapt_lim_coeff.constant = 0.0;
-      surf_fit_coeff.constant   = 0.0;
       init_metric_energy = a.GetParGridFunctionEnergy(x) /
                            (hradaptivity ? pmesh->GetGlobalNE() : 1);
       lim_coeff.constant = lim_const;
       adapt_lim_coeff.constant = adapt_lim_const;
-      surf_fit_coeff.constant  = surface_fit_const;
    }
 
    // Visualize the starting mesh and metric values.
@@ -1200,11 +1112,6 @@ int main (int argc, char *argv[])
    const IntegrationRule &ir =
       irules->Get(pfespace->GetFE(0)->GetGeomType(), quad_order);
    TMOPNewtonSolver solver(pfespace->GetComm(), ir, solver_type);
-   if (surface_fit_adapt) { solver.EnableAdaptiveSurfaceFitting(); }
-   if (surface_fit_threshold > 0)
-   {
-      solver.SetTerminationWithMaxSurfaceFittingError(surface_fit_threshold);
-   }
    // Provide all integration rules in case of a mixed mesh.
    solver.SetIntegrationRules(*irules, quad_order);
    if (solver_type == 0)
@@ -1256,16 +1163,14 @@ int main (int argc, char *argv[])
    const double fin_energy = a.GetParGridFunctionEnergy(x) /
                              (hradaptivity ? pmesh->GetGlobalNE() : 1);
    double fin_metric_energy = fin_energy;
-   if (lim_const > 0.0 || adapt_lim_const > 0.0 || surface_fit_const > 0.0)
+   if (lim_const > 0.0 || adapt_lim_const > 0.0)
    {
       lim_coeff.constant = 0.0;
       adapt_lim_coeff.constant = 0.0;
-      surf_fit_coeff.constant  = 0.0;
       fin_metric_energy  = a.GetParGridFunctionEnergy(x) /
                            (hradaptivity ? pmesh->GetGlobalNE() : 1);
       lim_coeff.constant = lim_const;
       adapt_lim_coeff.constant = adapt_lim_const;
-      surf_fit_coeff.constant  = surface_fit_const;
    }
    if (myid == 0)
    {
@@ -1294,26 +1199,6 @@ int main (int argc, char *argv[])
                              600, 600, 300, 300);
    }
 
-   // Visualize fitting surfaces and report fitting errors.
-   if (surface_fit_const > 0.0)
-   {
-      if (visualization)
-      {
-         socketstream vis2, vis3;
-         common::VisualizeField(vis2, "localhost", 19916, mat,
-                                "Materials", 600, 900, 300, 300);
-         common::VisualizeField(vis3, "localhost", 19916, surf_fit_mat_gf,
-                                "Surface dof", 900, 900, 300, 300);
-      }
-      double err_avg, err_max;
-      tmop_integ->GetSurfaceFittingErrors(err_avg, err_max);
-      if (myid == 0)
-      {
-         std::cout << "Avg fitting error: " << err_avg << std::endl
-                   << "Max fitting error: " << err_max << std::endl;
-      }
-   }
-
    // Visualize the mesh displacement.
    if (visualization)
    {
@@ -1341,7 +1226,6 @@ int main (int argc, char *argv[])
    delete metric2;
    delete metric_coeff1;
    delete adapt_lim_eval;
-   delete adapt_surface;
    delete target_c;
    delete hr_adapt_coeff;
    delete adapt_coeff;

miniapps/nurbs/CMakeLists.txt

@@ -14,6 +14,14 @@ add_mfem_miniapp(nurbs_ex1
   LIBRARIES mfem)
 
 if (MFEM_ENABLE_TESTING)
+  add_test(NAME nurbs_ex1_1d_r1_o2_ser
+    COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+    -m ${PROJECT_SOURCE_DIR}/data/segment-nurbs.mesh -r 1 -o 2 -lod 3)
+
+  add_test(NAME nurbs_ex1_1d_r1_o2_wbc_ser
+    COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+    -m ${PROJECT_SOURCE_DIR}/data/segment-nurbs.mesh -r 1 -o 2 -wbc -lod 3)
+
   add_test(NAME nurbs_ex1_r0_o4_ser
     COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis -r 0 -o 4)
 

miniapps/nurbs/nurbs_ex1.cpp

@@ -10,6 +10,7 @@
 //               nurbs_ex1 -m ../../data/disc-nurbs.mesh -o -1
 //               nurbs_ex1 -m ../../data/pipe-nurbs.mesh -o -1
 //               nurbs_ex1 -m ../../data/beam-hex-nurbs.mesh -pm 1 -ps 2
+//               nurbs_ex1 -m ../../data/segment-nurbs.mesh -r 2 -o 2 -lod 3
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -30,10 +31,20 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include <list>
 
 using namespace std;
 using namespace mfem;
 
+class Data
+{
+public:
+   double x,val;
+   Data(double x_, double val_) {x=x_; val=val_;};
+};
+
+inline bool operator==(const Data& d1,const Data& d2) { return (d1.x == d2.x); };
+inline bool operator <(const Data& d1,const Data& d2) { return (d1.x  < d2.x); };
 
 /** Class for integrating the bilinear form a(u,v) := (Q Laplace u, v) where Q
     can be a scalar coefficient. */
@@ -131,6 +142,7 @@ int main(int argc, char *argv[])
    Array<int> slave(0);
    bool static_cond = false;
    bool visualization = 1;
+   int lod = 0;
    bool ibp = 1;
    bool strongBC = 1;
    double kappa = -1;
@@ -165,6 +177,8 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&lod, "-lod", "--level-of-detail",
+                  "Refinement level for 1D solution output (0 means no output).");
    args.Parse();
    if (!args.Good())
    {
@@ -390,6 +404,7 @@ int main(int argc, char *argv[])
    ofstream sol_ofs("sol.gf");
    sol_ofs.precision(8);
    x.Save(sol_ofs);
+   sol_ofs.close();
 
    // 13. Send the solution by socket to a GLVis server.
    if (visualization)
@@ -401,6 +416,43 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *mesh << x << flush;
    }
 
+   if (mesh->Dimension() == 1 && lod > 0)
+   {
+      std::list<Data> sol;
+
+      Vector      vals,coords;
+      GridFunction *nodes = mesh->GetNodes();
+      if (!nodes)
+      {
+         nodes = new GridFunction(fespace);
+         mesh->GetNodes(*nodes);
+      }
+
+      for (int i = 0; i <  mesh->GetNE(); i++)
+      {
+         int geom       = mesh->GetElementBaseGeometry(i);
+         RefinedGeometry *refined_geo = GlobGeometryRefiner.Refine(( Geometry::Type)geom,
+                                                                   lod, 1);
+
+         x.GetValues(i, refined_geo->RefPts, vals);
+         nodes->GetValues(i, refined_geo->RefPts, coords);
+
+         for (int j = 0; j < vals.Size(); j++)
+         {
+            sol.push_back(Data(coords[j],vals[j]));
+         }
+      }
+      sol.sort();
+      sol.unique();
+      ofstream sol_ofs("solution.dat");
+      for (std::list<Data>::iterator d = sol.begin(); d != sol.end(); ++d)
+      {
+         sol_ofs<<d->x <<"\t"<<d->val<<endl;
+      }
+
+      sol_ofs.close();
+   }
+
    // 14. Save data in the VisIt format
    VisItDataCollection visit_dc("Example1", mesh);
    visit_dc.RegisterField("solution", &x);

miniapps/shifted/CMakeLists.txt

@@ -11,13 +11,11 @@
 
 if (MFEM_USE_MPI)
   list(APPEND DIST_COMMON_SOURCES
-        dist_solver.cpp
         sbm_solver.cpp
         marking.cpp
         extrapolator.cpp
 	integ_algoim.cpp)
   list(APPEND DIST_COMMON_HEADERS
-        dist_solver.hpp
         sbm_solver.hpp
         sbm_aux.hpp
         marking.hpp

miniapps/shifted/diffusion.cpp

@@ -76,10 +76,10 @@
 #include "sbm_aux.hpp"
 #include "sbm_solver.hpp"
 #include "marking.hpp"
-#include "dist_solver.hpp"
 
 using namespace mfem;
 using namespace std;
+using namespace common;
 
 int main(int argc, char *argv[])
 {

miniapps/shifted/distance.cpp

@@ -91,11 +91,11 @@
 #include <fstream>
 #include <iostream>
 #include "../common/mfem-common.hpp"
-#include "dist_solver.hpp"
 #include "sbm_aux.hpp"
 
 using namespace std;
 using namespace mfem;
+using namespace common;
 
 double sine_ls(const Vector &x)
 {

miniapps/shifted/makefile

@@ -25,9 +25,9 @@ include $(DEFAULTS_MK)
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-DIFFUSION_SRC = diffusion.cpp dist_solver.cpp sbm_solver.cpp marking.cpp
+DIFFUSION_SRC = diffusion.cpp sbm_solver.cpp marking.cpp
 DIFFUSION_OBJ = $(DIFFUSION_SRC:.cpp=.o)
-DISTANCE_SRC = distance.cpp dist_solver.cpp
+DISTANCE_SRC = distance.cpp
 DISTANCE_OBJ = $(DISTANCE_SRC:.cpp=.o)
 EXTRAPOLATE_SRC = extrapolate.cpp extrapolator.cpp marking.cpp
 EXTRAPOLATE_OBJ = $(EXTRAPOLATE_SRC:.cpp=.o)

miniapps/toys/rubik.cpp

@@ -1106,8 +1106,6 @@ void mark_elements(Mesh & mesh, char axis, int tier)
    double xData[3];
    Vector x(xData,3);
 
-   int count = 0;
-
    Array<int> v;
    for (int i=0; i<mesh.GetNE(); i++)
    {
@@ -1127,7 +1125,6 @@ void mark_elements(Mesh & mesh, char axis, int tier)
             if ( x[0] > -2.5 + tier && x[0] < -1.5 + tier )
             {
                mesh.SetAttribute(i, 2);
-               count++;
             }
             else
             {
@@ -1138,7 +1135,6 @@ void mark_elements(Mesh & mesh, char axis, int tier)
             if ( x[1] > -2.5 + tier && x[1] < -1.5 + tier )
             {
                mesh.SetAttribute(i, 2);
-               count++;
             }
             else
             {
@@ -1149,7 +1145,6 @@ void mark_elements(Mesh & mesh, char axis, int tier)
             if ( x[2] > -2.5 + tier && x[2] < -1.5 + tier )
             {
                mesh.SetAttribute(i, 2);
-               count++;
             }
             else
             {

tests/unit/CMakeLists.txt

@@ -73,6 +73,7 @@ set(UNIT_TESTS_SRCS
   fem/test_fa_determinism.cpp
   fem/test_face_elem_trans.cpp
   fem/test_face_permutation.cpp
+  fem/test_face_restriction.cpp
   fem/test_fe.cpp
   fem/test_get_value.cpp
   fem/test_getderivative.cpp
@@ -351,7 +352,7 @@ if (MFEM_USE_HIP)
    set_property(SOURCE ${DEBUG_DEVICE_SRCS}
                 PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
-mfem_add_executable(debug_device_tests unit_test_main.cpp ${DEBUG_DEVICE_SRCS})
+mfem_add_executable(debug_device_tests ${DEBUG_DEVICE_SRCS})
 target_link_libraries(debug_device_tests mfem)
 add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} debug_device_tests)
 add_test(NAME debug_device_tests COMMAND debug_device_tests)

tests/unit/fem/test_face_restriction.cpp

@@ -0,0 +1,118 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include "unit_tests.hpp"
+
+using namespace mfem;
+
+Mesh MakeCartesianMesh(int nx, int dim)
+{
+   if (dim == 2)
+   {
+      return Mesh::MakeCartesian2D(nx, nx, Element::QUADRILATERAL, true);
+   }
+   else
+   {
+      return Mesh::MakeCartesian3D(nx, nx, nx, Element::HEXAHEDRON);
+   }
+}
+
+namespace face_restriction_test { enum class SpaceType {RT, ND}; }
+
+TEST_CASE("Vector FE Face Restriction", "[FaceRestriction]")
+{
+   using namespace face_restriction_test;
+
+   const auto space_type = GENERATE(SpaceType::RT, SpaceType::ND);
+   const int dim = GENERATE(2, 3);
+   const int nx = 3;
+   const int order = 4;
+
+   CAPTURE(dim);
+
+   Mesh mesh = MakeCartesianMesh(nx, dim);
+
+   int ndof_per_face;
+   std::unique_ptr<FiniteElementCollection> fec;
+   if (space_type == SpaceType::RT)
+   {
+      fec.reset(new RT_FECollection(order-1, dim));
+      ndof_per_face = int(pow(order, dim-1));
+   }
+   else
+   {
+      fec.reset(new ND_FECollection(order, dim));
+      ndof_per_face = (dim - 1)*order*int(pow(order + 1, dim - 2));
+   }
+
+   FiniteElementSpace fes(&mesh, fec.get());
+
+   auto ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   auto ftype = FaceType::Boundary;
+   const int nfaces = fes.GetNFbyType(FaceType::Boundary);
+   const FaceRestriction *face_restr =
+      fes.GetFaceRestriction(ordering, ftype);
+
+   REQUIRE(face_restr != nullptr);
+
+   Array<int> bdr_dofs;
+   fes.GetBoundaryTrueDofs(bdr_dofs);
+
+   // Set gf to have random values on the boundary, zero on the interior
+   GridFunction gf(&fes);
+   gf.Randomize(0);
+   gf.SetSubVectorComplement(bdr_dofs, 0.0);
+
+   // Mapping to face E-vector and back to L-vector should give back the
+   // original grid function.
+   Vector face_vec(face_restr->Height());
+   REQUIRE(face_vec.Size() == nfaces*ndof_per_face);
+   face_restr->Mult(gf, face_vec);
+
+   if (space_type == SpaceType::ND && dim == 3)
+   {
+      // Adjust for multiplicity. In all other cases, each boundary DOF is
+      // unique (not shared between faces). In the case of 3D ND elements, some
+      // boundary DOFs are shared between two faces (i.e. those that lie on
+      // element edges).
+      //
+      // This adjustment will ensure that the original vector is recovered after
+      // multiplying by the transpose of the face restriction operator.
+      //
+      // Note that this assumes the mesh contains only hexahedral elements.
+      const int n = order*(order+1);
+      for (int f = 0; f < fes.GetNFbyType(ftype); ++f)
+      {
+         for (int d = 0; d < 2; ++d)
+         {
+            const int mx = (d == 0) ? order : order + 1;
+            const int my = (d == 0) ? order + 1 : order;
+            for (int i = 0; i < n; ++i)
+            {
+               const int ix = i % mx;
+               const int iy = i / mx;
+               if ((d == 0 && (iy == 0 || iy == my - 1)) ||
+                   (d == 1 && (ix == 0 || ix == mx - 1)))
+               {
+                  face_vec[f*ndof_per_face + d*n + i] *= 0.5;
+               }
+            }
+         }
+      }
+   }
+
+   GridFunction gf2(&fes);
+   face_restr->MultTranspose(face_vec, gf2);
+
+   gf2 -= gf;
+   REQUIRE(gf2.Normlinf() == MFEM_Approx(0.0));
+}

tests/unit/fem/test_linearform_ext.cpp

@@ -282,4 +282,68 @@ TEST_CASE("Linear Form Extension", "[LinearFormExtension], [CUDA]")
 
       REQUIRE(d1.Norml2() == MFEM_Approx(0.0));
    }
+
+   SECTION("VectorFE")
+   {
+      Mesh mesh(mesh_file);
+      const int dim = mesh.Dimension();
+
+      CAPTURE(mesh_file, dim, p);
+
+      RT_FECollection fec(p-1, dim);
+      FiniteElementSpace fes(&mesh, &fec);
+
+      FunctionCoefficient coeff(f);
+
+      LinearForm d1(&fes);
+      d1.AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(coeff));
+      d1.UseFastAssembly(true);
+      d1.Assemble();
+
+      LinearForm d2(&fes);
+      d2.AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(coeff));
+      d2.UseFastAssembly(false);
+      d2.Assemble();
+
+      CAPTURE(d1.Norml2(), d2.Norml2());
+
+      d1 -= d2;
+
+      REQUIRE(d1.Norml2() == MFEM_Approx(0.0));
+   }
+}
+
+TEST_CASE("H(div) Linear Form Extension", "[LinearFormExtension], [CUDA]")
+{
+   const bool all = launch_all_non_regression_tests;
+
+   const auto mesh_file =
+      all ? GENERATE("../../data/star.mesh", "../../data/star-q3.mesh",
+                     "../../data/fichera.mesh", "../../data/fichera-q3.mesh") :
+      GENERATE("../../data/star-q3.mesh", "../../data/fichera-q3.mesh");
+   const auto p = all ? GENERATE(1,2,3,4,5,6) : GENERATE(1,3);
+
+   Mesh mesh(mesh_file);
+   const int dim = mesh.Dimension();
+
+   CAPTURE(mesh_file, dim, p);
+
+   RT_FECollection fec(p, dim);
+   FiniteElementSpace fes(&mesh, &fec);
+
+   VectorFunctionCoefficient coeff(dim, fvec_dim);
+
+   LinearForm d1(&fes);
+   d1.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coeff));
+   d1.UseFastAssembly(true);
+   d1.Assemble();
+
+   LinearForm d2(&fes);
+   d2.AddDomainIntegrator(new VectorFEDomainLFIntegrator(coeff));
+   d2.UseFastAssembly(false);
+   d2.Assemble();
+
+   CAPTURE(d1.Norml2(), d2.Norml2());
+   d1 -= d2;
+   REQUIRE(d1.Norml2() == MFEM_Approx(0.0));
 }

tests/unit/fem/test_project_bdr.cpp

@@ -32,7 +32,6 @@ TEST_CASE("3D ProjectBdrCoefficientTangent",
    int n = 1;
    int dim = 3;
    int order = 1;
-   int npts = 0;
 
    double tol = 1e-6;
 
@@ -84,7 +83,6 @@ TEST_CASE("3D ProjectBdrCoefficientTangent",
 
                for (int j=0; j<ir.GetNPoints(); j++)
                {
-                  npts++;
                   const IntegrationPoint &ip = ir.IntPoint(j);
                   T->SetIntPoint(&ip);