minor edits

Fix warning from assigned and unused variables

.gitignore

@@ -113,6 +113,12 @@ examples/ex25p-*.*
 examples/ex28_*
 examples/ex28p_*
 examples/flux.*
+examples/dsol.*
+examples/cond.*
+examples/cond_j.*
+examples/cond_mesh.*
+examples/port_mesh.*
+examples/port_mode.*
 
 examples/amgx/ex1
 examples/amgx/ex1p

CHANGELOG

@@ -13,6 +13,9 @@ Version 4.5.3 (development)
 
 New and updated examples and miniapps
 -------------------------------------
+- Added a new example code, Example 36/36p, to demonstrate the solution of
+  the obstacle problem with a new finite element method.
+
 - Added a new miniapp, Mesh Quality, for evaluating mesh quality using size,
   skewness, and aspect-ratio computed from the Jacobian of the transformation.
 
@@ -26,6 +29,9 @@ New and updated examples and miniapps
   integrators are added in support of DPG systems: TraceIntegrator,
   NormalTraceIntegrator and TangentTraceIntegrator.
 
+- Added new SubMesh examples demonstrating source terms and boundary conditions
+  transferred from SubMesh objects.
+
 - Added a new H(div) solvers miniapp in miniapps/hdiv-linear-solver,
   demonstrating the use of a matrix-free saddle-point solver methodology,
   suitable for high-order discretizations and for GPU acceleration. Examples
@@ -41,6 +47,8 @@ Meshing improvements
 - Added new methods in the Mesh class to set and get attributes on NURBS patches
   and patch boundaries.
 
+- Added HIP support to the SUNDIALS interface.
+
 - TMOP improvement: added asymptotically-balanced compound metrics 90, 94, 328,
   338. Added the tmop-metric-magnitude tool for tracking how metrics change
   under geometric perturbations.
@@ -50,6 +58,9 @@ Discretization improvements
 - Face restriction operators for Nedelec and Raviart-Thomas finite element
   spaces are now supported through the ConformingFaceRestriction class.
 
+- SubMesh and ParSubMesh have been extended to support the transfer of
+  Nedelec and Raviart-Thomas finite element spaces.
+
 - VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
 
 - Added support for p-refined meshes in FindPointsGSLIB.

CMakeLists.txt

@@ -317,6 +317,9 @@ if (MFEM_USE_SUNDIALS)
   if (MFEM_USE_CUDA)
     list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
   endif()
+  if (MFEM_USE_HIP)
+    list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
+  endif()
   find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 

INSTALL

@@ -628,9 +628,13 @@ The specific libraries and their options are:
   both MPI and hypre.
   If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
   for CUDA.
-  URL: http://computation.llnl.gov/projects/sundials/sundials-software
+  If MFEM_USE_HIP is enabled, we expect that SUNDIALS is built with support
+  for HIP.
+  URL: http://computing.llnl.gov/projects/sundials/sundials-software
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.
+  Versions: SUNDIALS >= 5.0.0,
+            SUNDIALS >= 5.4.0 for CUDA support, and
+            SUNDIALS >= 5.7.0 for HIP support.
 
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
   URL: http://faculty.cse.tamu.edu/davis/suitesparse.html

config/cmake/MFEMConfig.cmake.in

@@ -55,6 +55,8 @@ set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
 set(MFEM_USE_MOONOLITH @MFEM_USE_MOONOLITH@)
 set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
+set(MFEM_USE_MKL_CPARDISO @MFEM_USE_MKL_CPARDISO@)
+set(MFEM_USE_MKL_PARDISO @MFEM_USE_MKL_PARDISO@)
 set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)

config/cmake/config.hpp.in

@@ -186,6 +186,9 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO
 
+// Enable interface to the MKL Pardiso library.
+#cmakedefine MFEM_USE_MKL_PARDISO
+
 // Use forward mode for automatic differentiation.
 #cmakedefine MFEM_USE_ADFORWARD
 

config/cmake/modules/FindSUNDIALS.cmake

@@ -22,8 +22,8 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
     "include" nvector/nvector_serial.h "lib" sundials_nvecserial
   ADD_COMPONENT NVector_Cuda
     "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_ParHyp
-    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
+  ADD_COMPONENT NVector_Hip
+    "include" nvector/nvector_hip.h "lib" sundials_nvechip
   ADD_COMPONENT NVector_Parallel
     "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
   ADD_COMPONENT NVector_MPIPlusX

config/defaults.mk

@@ -267,6 +267,9 @@ endif
 ifeq ($(MFEM_USE_CUDA),YES)
    SUNDIALS_LIB += -lsundials_nveccuda
 endif
+ifeq ($(MFEM_USE_HIP),YES)
+   SUNDIALS_LIB += -lsundials_nvechip
+endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 

data/fichera-quad-mixed.mesh

@@ -0,0 +1,48 @@
+MFEM mesh v1.0
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+6
+1 3 0 1 4 3
+1 3 2 3 6 5
+1 2 3 4 8
+1 2 4 7 8
+1 2 7 6 8
+1 2 6 3 8
+
+boundary
+8
+1 1 0 1
+2 1 1 4
+3 1 4 7
+4 1 7 6
+5 1 6 5
+6 1 5 2
+7 1 2 3
+8 1 3 0
+
+vertices
+9
+2
+0.5 0
+1 0
+0 0.5
+0.5 0.5
+1 0.5
+0 1
+0.5 1
+1 1
+0.75 0.75

data/fichera-quad.mesh

@@ -0,0 +1,44 @@
+MFEM mesh v1.0
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+3
+1 3 0 1 4 3
+1 3 2 3 6 5
+1 3 3 4 7 6
+
+boundary
+8
+1 1 0 1
+2 1 1 4
+3 1 4 7
+4 1 7 6
+5 1 6 5
+6 1 5 2
+7 1 2 3
+8 1 3 0
+
+vertices
+8
+2
+0.5 0
+1 0
+0 0.5
+0.5 0.5
+1 0.5
+0 1
+0.5 1
+1 1

data/llnl-p3.mesh

@@ -0,0 +1,322 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+#
+
+dimension
+2
+
+elements
+26
+1 2 1 18 0
+1 3 1 3 19 18
+2 3 3 6 20 19
+1 3 6 9 21 20
+2 3 9 12 22 21
+1 3 12 15 23 22
+2 2 23 15 24
+1 2 1 4 3
+2 3 4 7 6 3
+1 3 7 10 9 6
+2 3 10 13 12 9
+1 3 13 16 15 12
+2 3 16 25 24 15
+1 3 2 5 4 1
+1 3 5 8 7 4
+1 3 8 11 10 7
+1 3 11 14 13 10
+1 3 14 17 16 13
+1 2 25 16 17
+1 3 18 19 27 26
+2 3 19 20 28 27
+1 3 20 21 29 28
+2 3 21 22 30 29
+1 3 22 23 31 30
+2 3 23 24 32 31
+1 3 24 25 33 32
+
+boundary
+18
+1 1 28 27
+2 1 30 29
+3 1 32 31
+4 1 0 1
+4 1 1 2
+4 1 2 5
+4 1 5 8
+4 1 8 11
+4 1 11 14
+4 1 14 17
+4 1 17 25
+4 1 25 33
+4 1 33 32
+4 1 31 30
+4 1 29 28
+4 1 27 26
+4 1 26 18
+4 1 18 0
+
+vertices
+34
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P3
+VDim: 2
+Ordering: 1
+
+0 0
+0.53125 0
+1 0
+0.53125 0.09375
+0.5625 0.09375
+1 0.09375
+0.53125 0.21875
+0.6875 0.21875
+1 0.1875
+0.53125 0.25
+0.71875 0.25
+1 0.25
+0.53125 0.375
+0.84375 0.375
+1 0.34375
+0.53125 0.40625
+0.875 0.40625
+1 0.40625
+0 0.53125
+0.09375 0.53125
+0.21875 0.53125
+0.25 0.53125
+0.375 0.53125
+0.40625 0.53125
+0.53125 0.53125
+1 0.53125
+0 1
+0.09375 1
+0.21875 1
+0.25 1
+0.375 1
+0.40625 1
+0.53125 1
+1 1
+
+0.33175106835972 0.094168845750364
+0.094168845750364 0.33175106835972
+-5.1759634627347e-17 0.14683388869532
+6.5255471622478e-17 0.38441611130468
+0.14683388869532 6.0713766400335e-17
+0.38441611130468 8.2458945395444e-19
+0.53125 0.025911862710939
+0.53125 0.067838137289061
+0.34721731046049 0.13433915461926
+0.13433915461926 0.34721731046049
+0.025911862710939 0.53125
+0.067838137289061 0.53125
+0.53125 0.12829915028125
+0.53125 0.18420084971875
+0.39979807890035 0.24774225329947
+0.24774225329947 0.39979807890035
+0.12829915028125 0.53125
+0.18420084971875 0.53125
+0.53125 0.22738728757031
+0.53125 0.24136271242969
+0.41294327101031 0.27609302796952
+0.27609302796952 0.41294327101031
+0.22738728757031 0.53125
+0.24136271242969 0.53125
+0.53125 0.28454915028125
+0.53125 0.34045084971875
+0.46552403945017 0.38949612664974
+0.38949612664974 0.46552403945017
+0.28454915028125 0.53125
+0.34045084971875 0.53125
+0.53125 0.38363728757031
+0.53125 0.39761271242969
+0.47866923156014 0.41784690131979
+0.41784690131979 0.47866923156014
+0.38363728757031 0.53125
+0.39761271242969 0.53125
+0.53125 0.44079915028125
+0.53125 0.49670084971875
+0.44079915028125 0.53125
+0.49670084971875 0.53125
+0.53988728757031 0.025911862710939
+0.55386271242969 0.067838137289061
+0.53988728757031 0.09375
+0.55386271242969 0.09375
+0.59704915028125 0.12829915028125
+0.65295084971875 0.18420084971875
+0.57443643785157 0.21875
+0.64431356214843 0.21875
+0.69613728757031 0.22738728757031
+0.71011271242969 0.24136271242969
+0.58307372542188 0.25
+0.66692627457812 0.25
+0.75329915028125 0.28454915028125
+0.80920084971875 0.34045084971875
+0.61762287570313 0.375
+0.75737712429687 0.375
+0.85238728757031 0.38363728757031
+0.86636271242969 0.39761271242969
+0.62626016327344 0.40625
+0.77998983672656 0.40625
+0.90954915028125 0.44079915028125
+0.96545084971875 0.49670084971875
+0.6608093135547 0.53125
+0.8704406864453 0.53125
+1 0.025911862710939
+1 0.067838137289061
+0.68342202598438 0.09375
+0.87907797401562 0.09375
+0.6608093135547 0
+0.8704406864453 0
+1 0.11966186271094
+1 0.16158813728906
+0.77387287570313 0.21011271242969
+0.91362712429687 0.19613728757031
+1 0.20477457514063
+1 0.23272542485937
+0.79648558813282 0.25
+0.92226441186718 0.25
+1 0.27591186271094
+1 0.31783813728906
+0.88693643785157 0.36636271242969
+0.95681356214843 0.35238728757031
+1 0.36102457514063
+1 0.38897542485937
+0.90954915028125 0.40625
+0.96545084971875 0.40625
+1 0.44079915028125
+1 0.49670084971875
+0.09375 0.6608093135547
+0.09375 0.8704406864453
+0.025911862710939 1
+0.067838137289061 1
+0 0.6608093135547
+0 0.8704406864453
+0.21875 0.6608093135547
+0.21875 0.8704406864453
+0.12829915028125 1
+0.18420084971875 1
+0.25 0.6608093135547
+0.25 0.8704406864453
+0.22738728757031 1
+0.24136271242969 1
+0.375 0.6608093135547
+0.375 0.8704406864453
+0.28454915028125 1
+0.34045084971875 1
+0.40625 0.6608093135547
+0.40625 0.8704406864453
+0.38363728757031 1
+0.39761271242969 1
+0.53125 0.6608093135547
+0.53125 0.8704406864453
+0.44079915028125 1
+0.49670084971875 1
+1 0.6608093135547
+1 0.8704406864453
+0.6608093135547 1
+0.8704406864453 1
+0.14782497614169 0.14782497614169
+0.3364183509774 0.10629113008478
+0.3439701728879 0.12590539815918
+0.10629113008478 0.3364183509774
+0.12590539815918 0.3439701728879
+0.36175027742635 0.16568300020856
+0.38526511193449 0.21639840771017
+0.16568300020856 0.36175027742635
+0.21639840771017 0.38526511193449
+0.40343132064181 0.2555782146968
+0.40931002926885 0.2682570665722
+0.2555782146968 0.40343132064181
+0.2682570665722 0.40931002926885
+0.42747623797617 0.30743687355882
+0.45099107248431 0.35815228106044
+0.30743687355882 0.42747623797617
+0.35815228106044 0.45099107248431
+0.46915728119164 0.39733208804706
+0.47503598981867 0.41001093992246
+0.39733208804706 0.46915728119164
+0.41001093992246 0.47503598981867
+0.47232443490112 0.47232443490112
+0.54166666666667 0.0625
+0.57886271242969 0.12829915028125
+0.61931356214843 0.18420084971875
+0.54943643785157 0.12829915028125
+0.56488728757031 0.18420084971875
+0.65056356214843 0.22738728757031
+0.66067627457812 0.24136271242969
+0.57682372542188 0.22738728757031
+0.58068643785157 0.24136271242969
+0.69192627457812 0.28454915028125
+0.73237712429687 0.34045084971875
+0.59262287570313 0.28454915028125
+0.60807372542188 0.34045084971875
+0.76362712429687 0.38363728757031
+0.77373983672656 0.39761271242969
+0.62001016327344 0.38363728757031
+0.62387287570313 0.39761271242969
+0.80498983672656 0.44079915028125
+0.8454406864453 0.49670084971875
+0.6358093135547 0.44079915028125
+0.65126016327344 0.49670084971875
+0.87282797401562 0.025911862710939
+0.8766906864453 0.067838137289061
+0.6670593135547 0.025911862710939
+0.67717202598438 0.067838137289061
+0.88862712429687 0.12204915028125
+0.90407797401562 0.16783813728906
+0.70842202598438 0.12591186271094
+0.74887287570313 0.17795084971875
+0.91601441186718 0.21102457514063
+0.91987712429687 0.23511271242969
+0.78012287570313 0.22113728757031
+0.79023558813282 0.23897542485937
+0.93181356214843 0.27829915028125
+0.94726441186718 0.32408813728906
+0.82148558813282 0.28216186271094
+0.86193643785157 0.33420084971875
+0.95920084971875 0.36727457514063
+0.96306356214843 0.39136271242969
+0.89318643785157 0.37738728757031
+0.90329915028125 0.39522542485937
+0.95833333333333 0.44791666666667
+0.025911862710939 0.6608093135547
+0.067838137289061 0.6608093135547
+0.025911862710939 0.8704406864453
+0.067838137289061 0.8704406864453
+0.12829915028125 0.6608093135547
+0.18420084971875 0.6608093135547
+0.12829915028125 0.8704406864453
+0.18420084971875 0.8704406864453
+0.22738728757031 0.6608093135547
+0.24136271242969 0.6608093135547
+0.22738728757031 0.8704406864453
+0.24136271242969 0.8704406864453
+0.28454915028125 0.6608093135547
+0.34045084971875 0.6608093135547
+0.28454915028125 0.8704406864453
+0.34045084971875 0.8704406864453
+0.38363728757031 0.6608093135547
+0.39761271242969 0.6608093135547
+0.38363728757031 0.8704406864453
+0.39761271242969 0.8704406864453
+0.44079915028125 0.6608093135547
+0.49670084971875 0.6608093135547
+0.44079915028125 0.8704406864453
+0.49670084971875 0.8704406864453
+0.6608093135547 0.6608093135547
+0.8704406864453 0.6608093135547
+0.6608093135547 0.8704406864453
+0.8704406864453 0.8704406864453

doc/CodeDocumentation.dox

@@ -105,6 +105,8 @@ namespace mfem {
  * - <a class="el" href="ex32p_8cpp_source.html">Example 32p</a>: parallel anisotropic Maxwell eigensolver
  * - <a class="el" href="ex33_8cpp_source.html">Example 33</a>: nodal H1 FEM for the fractional Laplacian problem
  * - <a class="el" href="ex33p_8cpp_source.html">Example 33p</a>: parallel nodal H1 FEM for the fractional Laplacian problem
+ * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
+ * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples

examples/CMakeLists.txt

@@ -41,7 +41,7 @@ list(APPEND ALL_EXE_SRCS
   ex31.cpp
   ex33.cpp
   ex34.cpp
-  ex35.cpp
+  ex36.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -79,6 +79,9 @@ if (MFEM_USE_MPI)
     ex31p.cpp
     ex32p.cpp
     ex33p.cpp
+    ex34p.cpp
+    ex35p.cpp
+    ex36p.cpp
     )
 endif()
 
@@ -121,9 +124,10 @@ if (MFEM_ENABLE_TESTING)
   # Add CUDA/HIP tests.
   set(DEVICE_EXAMPLES
       # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
       # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
+      ex34p ex35p)
   set(MFEM_TEST_DEVICE)
   if (MFEM_USE_CUDA)
     set(MFEM_TEST_DEVICE "cuda")

examples/contact/IPsolver.cpp

@@ -0,0 +1,907 @@
+#include "mfem.hpp"
+#include "IPsolver.hpp"
+#include "problems.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstdlib>
+
+using namespace std;
+using namespace mfem;
+
+
+
+InteriorPointSolver::InteriorPointSolver(OptProblem * Problem, ParFiniteElementSpace *Vhin) 
+: problem(Problem), block_offsetsumlz(5), block_offsetsuml(4), block_offsetsx(3),
+ saveLogBarrierIterates(false), Vh(Vhin)
+{
+  tol  = 1.e-2;
+  max_iter = 20;
+  mu_k     = 1.0;
+
+  sMax     = 1.e2;
+  kSig     = 1.e10;   // control deviation from primal Hessian
+  tauMin   = 0.8;     // control rate at which iterates can approach the boundary
+  eta      = 1.e-4;   // backtracking constant
+  thetaMin = 1.e-4;   // allowed violation of the equality constraints
+
+  // constants in line-step A-5.4
+  delta    = 1.0;
+  sTheta   = 1.1;
+  sPhi     = 2.3;
+
+  // control the rate at which the penalty parameter is decreased
+  kMu     = 0.2;
+  thetaMu = 1.5;
+
+
+  thetaMax = 1.e6; // maximum constraint violation
+  // data for the second order correction
+  kSoc     = 0.99;
+
+  // equation (18)
+  gTheta = 1.e-5;
+  gPhi   = 1.e-5;
+
+  kEps   = 1.e1;
+
+  dimU = problem->GetDimU();
+  dimM = problem->GetDimM();
+  dimC = problem->GetDimC();
+  ckSoc.SetSize(dimC);
+  
+  block_offsetsumlz[0] = 0;
+  block_offsetsumlz[1] = dimU; // u
+  block_offsetsumlz[2] = dimM; // m
+  block_offsetsumlz[3] = dimC; // lambda
+  block_offsetsumlz[4] = dimM; // zl
+  block_offsetsumlz.PartialSum();
+
+  for(int i = 0; i < block_offsetsuml.Size(); i++)  { block_offsetsuml[i] = block_offsetsumlz[i]; }
+  for(int i = 0; i < block_offsetsx.Size(); i++)    { block_offsetsx[i]   = block_offsetsuml[i] ; }
+
+  // lower-bound for the inequality constraint m >= ml
+  ml = problem->Getml();
+  
+  lk.SetSize(dimC);   lk  = 0.0;
+  zlk.SetSize(dimM); zlk = 0.0;
+  mf.SetSize(dimM);   mf = 0.0;
+
+  linSolver = 0;
+  MyRank = 0;
+  iAmRoot = MyRank == 0 ? true : false;
+}
+
+double InteriorPointSolver::MaxStepSize(Vector &x, Vector &xl, Vector &xhat, double tau)
+{
+  double alphaMaxloc = 1.0;
+  double alphaTmp;
+  for(int i = 0; i < x.Size(); i++)
+  {
+    if( xhat(i) < 0. )
+    {
+      alphaTmp = -1. * tau * (x(i) - xl(i)) / xhat(i);
+      alphaMaxloc = min(alphaMaxloc, alphaTmp);
+    } 
+  }
+
+  // alphaMaxloc is the local maximum step size which is
+  // distinct on each MPI process. Need to compute
+  // the global maximum step size 
+  double alphaMaxglb;
+  alphaMaxglb = alphaMaxloc;
+  return alphaMaxglb;
+}
+
+double InteriorPointSolver::MaxStepSize(Vector &x, Vector &xhat, double tau)
+{
+  Vector zero(x.Size()); zero = 0.0;
+  return MaxStepSize(x, zero, xhat, tau);
+}
+
+
+void InteriorPointSolver::Mult(const Vector &x0, Vector &xf)
+{
+  BlockVector x0block(block_offsetsx); x0block = 0.0;
+  x0block.GetBlock(0).Set(1.0, x0);
+  // hard coded initialization :(
+  x0block.GetBlock(1) = 1.0;
+  x0block.GetBlock(1).Add(1.0, ml);
+  BlockVector xfblock(block_offsetsx); xfblock = 0.0;
+  Mult(x0block, xfblock);
+  xf.Set(1.0, xfblock.GetBlock(0));
+  mf.Set(1.0, xfblock.GetBlock(1));
+}
+
+void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
+{
+  converged = false;
+  IPNewtonKrylovIters.open("IPNewtonKrylovIters.dat", ios::out | ios::trunc);  
+  BlockVector xk(block_offsetsx), xhat(block_offsetsx); xk = 0; xhat = 0.0;
+  BlockVector Xk(block_offsetsumlz), Xhat(block_offsetsumlz); Xk = 0.0; Xhat = 0.0;
+  BlockVector Xhatuml(block_offsetsuml); Xhatuml = 0.0;
+  Vector zlhat(dimM); zlhat = 0.0;
+
+  xk.GetBlock(0).Set(1.0, x0.GetBlock(0));
+  xk.GetBlock(1).Set(1.0, x0.GetBlock(1));
+  // running estimate of the final values of the Lagrange multipliers
+  lk  = 0.0;
+  zlk = 0.0;
+
+  for(int i = 0; i < dimM; i++)
+  {
+    zlk(i) = 1.e1 * mu_k / (xk(i+dimU) - ml(i));
+  }
+
+  Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+  Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+  Xk.GetBlock(2).Set(1.0, lk);
+  Xk.GetBlock(3).Set(1.0, zlk);
+
+  /* set theta0 = theta(x0)
+   *     thetaMin
+   *     thetaMax
+   * when theta(xk) < thetaMin and the switching condition holds
+   * then we ask for the Armijo sufficient decrease of the barrier
+   * objective to be satisfied, in order to accept the trial step length alphakl
+   * 
+   * thetaMax controls how the filter is initialized for each log-barrier subproblem
+   * F0 = {(th, phi) s.t. th > thetaMax}
+   * that is the filter does not allow for iterates where the constraint violation
+   * is larger than that of thetaMax
+   */
+  double theta0 = theta(xk);
+  thetaMin = 1.e-4 * max(1.0, theta0);
+  thetaMax = 1.e8  * thetaMin;
+
+  double Eeval, maxBarrierSolves, Eevalmu0;
+  bool printOptimalityError; // control optimality error print to console for log-barrier subproblems
+  
+  maxBarrierSolves = 10;
+
+  for(jOpt = 0; jOpt < max_iter; jOpt++)
+  {
+    if(iAmRoot)
+    {
+      cout << "interior-point solve step " << jOpt << endl;
+    }
+    // A-2. Check convergence of overall optimization problem
+    printOptimalityError = false;
+    Eevalmu0 = E(xk, lk, zlk, printOptimalityError);
+    if(Eevalmu0 < tol)
+    {
+      converged = true;
+      if(iAmRoot)
+      {
+	      IPNewtonKrylovIters.close();
+        cout << "solved optimization problem :)\n";
+      }
+      break;
+    }
+    
+    if(jOpt > 0) { maxBarrierSolves = 1; }
+    
+    for(int i = 0; i < maxBarrierSolves; i++)
+    {
+      // A-3. Check convergence of the barrier subproblem
+      printOptimalityError = true;
+      Eeval = E(xk, lk, zlk, mu_k, printOptimalityError);
+      if(Eeval < kEps * mu_k)
+      {
+        if(iAmRoot)
+        {
+          cout << "solved barrier subproblem :), for mu = " << mu_k << endl;
+        }
+        // A-3.1. Recompute the barrier parameter
+        mu_k  = max(tol / 10., min(kMu * mu_k, pow(mu_k, thetaMu)));
+        // A-3.2. Re-initialize the filter
+        F1.DeleteAll();
+        F2.DeleteAll();
+      }
+      else
+      {
+        break;
+      }
+    }
+    
+    // A-4. Compute the search direction
+    // solve for (uhat, mhat, lhat)
+    if(iAmRoot)
+    {
+      cout << "\n** A-4. IP-Newton solve **\n";
+    }
+    zlhat = 0.0; Xhatuml = 0.0;
+    // why do we have Xhatuml ....???
+    // TO DO: remove Xhatuml in favor of passing Xhat
+    IPNewtonSolve(xk, lk, zlk, zlhat, Xhatuml, mu_k, false); 
+
+
+    // assign data stack, X = (u, m, l, zl)
+    Xk = 0.0;
+    Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+    Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+    Xk.GetBlock(2).Set(1.0, lk);
+    Xk.GetBlock(3).Set(1.0, zlk);
+
+    // assign data stack, Xhat = (uhat, mhat, lhat, zlhat)
+    Xhat = 0.0;
+    for(int i = 0; i < 3; i++)
+    {
+      Xhat.GetBlock(i).Set(1.0, Xhatuml.GetBlock(i));
+    }
+    Xhat.GetBlock(3).Set(1.0, zlhat);
+
+
+    // A-5. Backtracking line search.
+    if(iAmRoot)
+    {
+      cout << "\n** A-5. Linesearch **\n";
+      cout << "mu = " << mu_k << endl;
+    }
+    lineSearch(Xk, Xhat, mu_k);
+
+    if(lineSearchSuccess)
+    {
+      if(iAmRoot)
+      {
+        cout << "lineSearch successful :)\n";
+      }
+      if(!switchCondition || !sufficientDecrease)
+      {
+        F1.Append( (1. - gTheta) * thx0);
+        F2.Append( phx0 - gPhi * thx0);
+      }
+      // ----- A-6: Accept the trial point
+      // print info regarding zl...
+      xk.GetBlock(0).Add(alpha, Xhat.GetBlock(0));
+      xk.GetBlock(1).Add(alpha, Xhat.GetBlock(1));
+      lk.Add(alpha,   Xhat.GetBlock(2));
+      zlk.Add(alphaz, Xhat.GetBlock(3));
+      projectZ(xk, zlk, mu_k);
+    }
+    else
+    {
+      if(iAmRoot)
+      {
+        cout << "lineSearch not successful :(\n";
+        cout << "attempting feasibility restoration with theta = " << thx0 << endl;
+        cout << "no feasibility restoration implemented, exiting now \n";
+      }
+      break;
+      //cout << "feasibility restoration!!! :( :( :(\n";
+      //problem->feasibilityRestoration(x, 1.e-12);
+      //      break;
+    }
+    //
+    if(jOpt + 1 == max_iter && iAmRoot) 
+    { 
+      cout << "maximum optimization iterations :(\n";
+      IPNewtonKrylovIters.close();
+    }
+  }
+  // done with optimization routine, just reassign data to xf reference so
+  // that the application code has access to the optimal point
+  xf = 0.0;
+  xf.GetBlock(0).Set(1.0, xk.GetBlock(0));
+  xf.GetBlock(1).Set(1.0, xk.GetBlock(1));
+}
+
+void InteriorPointSolver::FormIPNewtonMat(BlockVector & x, Vector & l, Vector &zl, BlockOperator &Ak)
+{
+  // WARNING: Huu, Hum, Hmu, Hmm should all be Hessian terms of the Lagrangian, currently we 
+  //          them by Hessian terms of the objective function and neglect the Hessian of l^T c
+
+  Huu = problem->Duuf(x); Hum = problem->Dumf(x);
+  Hmu = problem->Dmuf(x); Hmm = problem->Dmmf(x);
+
+  Vector DiagLogBar(dimM); DiagLogBar = 0.0;
+  for(int ii = 0; ii < dimM; ii++)
+  {
+    DiagLogBar(ii) = zl(ii) / (x(ii+dimU) - ml(ii));
+  }
+  if(saveLogBarrierIterates)
+  {
+    std::ofstream diagStream;
+    char diagString[100];
+    snprintf(diagString, 100, "logBarrierHessiandata/D%d.dat", jOpt);
+    diagStream.open(diagString, ios::out | ios::trunc);
+    for(int ii = 0; ii < dimM; ii++)
+    {
+      diagStream << setprecision(30) << DiagLogBar(ii) << endl;
+    }
+    diagStream.close();
+  } 
+
+  delete Wmm;
+  if(Hmm != nullptr)
+  {
+    SparseMatrix * D = new SparseMatrix(DiagLogBar);
+    Wmm = Add(*Hmm, *D);
+    delete D;
+  }
+  else
+  {
+    Wmm = new SparseMatrix(DiagLogBar);
+  }
+
+  delete JuT;
+  delete JmT;
+  Ju = problem->Duc(x); JuT = Transpose(*Ju);
+  Jm = problem->Dmc(x); JmT = Transpose(*Jm);
+  
+  //         IP-Newton system matrix
+  //    Ak = [[H_(u,u)  H_(u,m)   J_u^T]
+  //          [H_(m,u)  W_(m,m)   J_m^T]
+  //          [ J_u      J_m       0  ]]
+
+  Ak.SetBlock(0, 0, Huu);                         Ak.SetBlock(0, 2, JuT);
+                          Ak.SetBlock(1, 1, Wmm); Ak.SetBlock(1, 2, JmT);
+  Ak.SetBlock(2, 0,  Ju); Ak.SetBlock(2, 1,  Jm);
+
+  if(Hum != nullptr) { Ak.SetBlock(0, 1, Hum); Ak.SetBlock(1, 0, Hmu); }
+}
+
+
+// perturbed KKT system solve
+// determine the search direction
+void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, Vector &zlhat, BlockVector &Xhat, double mu, bool socSolve)
+{
+  // solve A x = b, where A is the IP-Newton matrix
+  BlockOperator A(block_offsetsuml, block_offsetsuml); BlockVector b(block_offsetsuml); b = 0.0;
+  FormIPNewtonMat(x, l, zl, A);
+
+  //       [grad_u phi + Ju^T l]
+  // b = - [grad_m phi + Jm^T l]
+  //       [          c        ]
+  BlockVector gradphi(block_offsetsx); gradphi = 0.0;
+  BlockVector JTl(block_offsetsx); JTl = 0.0;
+  Dxphi(x, mu, gradphi);
+  
+  (A.GetBlock(0,2)).Mult(l, JTl.GetBlock(0));
+  (A.GetBlock(1,2)).Mult(l, JTl.GetBlock(1));
+
+  for(int ii = 0; ii < 2; ii++)
+  {
+    b.GetBlock(ii).Set(1.0, gradphi.GetBlock(ii));
+    b.GetBlock(ii).Add(1.0, JTl.GetBlock(ii));
+  }
+  if(!socSolve) 
+  {
+    problem->c(x, b.GetBlock(2));
+  }
+  else
+  {
+    b.GetBlock(2).Set(1.0, ckSoc);
+  }
+  b *= -1.0; 
+  Xhat = 0.0;
+
+  #ifdef MFEM_USE_SUITESPARSE
+    // Direct solve for IP-Newton saddle-point system
+    // A = [ [ Huu   0    Ju^T]
+    //       [  0    D     -I ]
+    //       [ Ju   -I      0 ]]
+    if(linSolver == 0)
+    {
+      BlockMatrix ABlockMatrix(block_offsetsuml, block_offsetsuml);
+      for(int ii = 0; ii < 3; ii++)
+      {
+        for(int jj = 0; jj < 3; jj++)
+        {
+          if(!A.IsZeroBlock(ii, jj))
+          {
+            ABlockMatrix.SetBlock(ii, jj, dynamic_cast<SparseMatrix *>(&(A.GetBlock(ii, jj))));
+          }
+        }
+      }
+      /* direct solve of the 3x3 IP-Newton linear system */
+      UMFPackSolver ASolver;
+      SparseMatrix *ASparse = ABlockMatrix.CreateMonolithic();
+      ASolver.SetOperator(*ASparse);
+      ASolver.Mult(b, Xhat);
+
+      Vector residual(Xhat.Size());
+      ASparse->Mult(Xhat, residual);
+      residual.Add(-1.0, b);
+      delete ASparse;
+    }
+    else if(linSolver == 1)
+    {
+      // Direct solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+      // where Wmm = D for contact problems
+      SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+      SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+      SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+      SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+      Vector Dvec(dimM); Dvec = 0.0;
+      Vector one(dimM); one = 1.0;
+      Wmmloc->Mult(one, Dvec);
+      SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
+      SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+
+      
+      /* prepare the reduced rhs */
+      // breduced = bu + Ju^T (bm + Wmm bl)
+      Vector breduced(dimU); breduced = 0.0;
+      Vector tempVec(dimM); tempVec = 0.0;
+      Wmmloc->Mult(b.GetBlock(2), tempVec);
+      tempVec.Add(1.0, b.GetBlock(1));
+      JuTloc->Mult(tempVec, breduced);
+      breduced.Add(1.0, b.GetBlock(0));
+
+      // solve the reduced linear system
+      UMFPackSolver AreducedSolver;
+      AreducedSolver.SetOperator(*Areduced);
+      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+
+      // now propagate solved uhat to obtain mhat and lhat
+      // xm = Ju xu - bl
+      Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+      Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+      // xl = Wmm xm - bm
+      Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+      Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+      delete Wmmloc;
+      delete Huuloc;
+      delete JuTDJu;
+      delete Juloc;
+      delete Areduced;
+    }
+  #else
+    MFEM_VERIFY(linSolver > 1, "linSolver = 0, 1 require MFEM_USE_SUITESPARSE=YES");
+  #endif
+  if (linSolver == 2 || linSolver == 3)
+  { 
+    // Iterative solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+    // where Wmm = D for contact problems
+    // here the iterative solver is a Jacobi-preconditioned CG-solve
+    SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+    SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+    SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+    SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+    // Vector Dvec(dimM); Dvec = 0.0;
+    // Vector one(dimM); one = 1.0;
+    // Wmmloc->Mult(one, Dvec);
+
+    // SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
+    SparseMatrix *JuTDJu   = RAP(*Juloc,*Wmmloc,*Juloc);     // Ju^T D Ju
+    SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+    /* prepare the reduced rhs */
+    // breduced = bu + Ju^T (bm + Wmm bl)
+    Vector breduced(dimU); breduced = 0.0;
+    Vector tempVec(dimM); tempVec = 0.0;
+    Wmmloc->Mult(b.GetBlock(2), tempVec);
+    tempVec.Add(1.0, b.GetBlock(1));
+    JuTloc->Mult(tempVec, breduced);
+    breduced.Add(1.0, b.GetBlock(0));
+    
+    /* set up an iterative solver */
+    int globalNumRows = dimU;
+    HYPRE_BigInt rowStarts[2];
+    rowStarts[0] = 0;
+    rowStarts[1] = dimU;
+    HypreParMatrix * Ahypre = new HypreParMatrix(MPI_COMM_WORLD, globalNumRows, rowStarts, Areduced);
+    // CGSolver Asolver(MPI_COMM_WORLD);
+    HyprePCG Asolver(MPI_COMM_WORLD);
+    HypreBoomerAMG * Aprec = new HypreBoomerAMG(*Ahypre);
+    Aprec->SetPrintLevel(0);
+    if(linSolver == 3)
+    {
+      Aprec->SetElasticityOptions(Vh);
+    }
+    Aprec->SetSystemsOptions(3,false);
+
+    Asolver.SetOperator(*Ahypre);
+    Asolver.SetPrintLevel(2);
+    Asolver.SetMaxIter(1000);
+    // Asolver.SetResidualConvergenceOptions();
+    Asolver.SetTol(1.e-6);
+    Asolver.SetPreconditioner(*Aprec);
+    // Asolver.SetResidualConvergenceOptions();
+
+    Asolver.Mult(breduced, Xhat.GetBlock(0));
+    int num_iterations;
+    Asolver.GetNumIterations(num_iterations);
+    cgnum_iterations.Append(num_iterations);
+    // int numNewtonKrylovIters = -1;
+    // numNewtonKrylovIters = Asolver.GetNumIterations();
+    // IPNewtonKrylovIters << numNewtonKrylovIters << endl;
+
+    delete Aprec;
+    delete Ahypre;
+    
+    // now propagate solved uhat to obtain mhat and lhat
+    // xm = Ju xu - bl
+    Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+    Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+    // // xl = Wmm xm - bm
+    Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+    Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+    
+
+    delete Wmmloc;
+    delete Huuloc;
+    delete JuTDJu;
+    delete Juloc;
+    delete Areduced;
+  }
+  else if(linSolver > 2)
+  { 
+    // Iterative solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+    // where Wmm = D for contact problems
+    // here the iterative solver is a Jacobi-preconditioned CG-solve
+    SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+    SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+    SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+    SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+    Vector Dvec(dimM); Dvec = 0.0;
+    Vector one(dimM); one = 1.0;
+    Wmmloc->Mult(one, Dvec);
+    SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
+    SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+
+    /* prepare the reduced rhs */
+    // breduced = bu + Ju^T (bm + Wmm bl)
+    Vector breduced(dimU); breduced = 0.0;
+    Vector tempVec(dimM); tempVec = 0.0;
+    Wmmloc->Mult(b.GetBlock(2), tempVec);
+    tempVec.Add(1.0, b.GetBlock(1));
+    JuTloc->Mult(tempVec, breduced);
+    breduced.Add(1.0, b.GetBlock(0));
+    
+    /* set up an iterative solver */
+    GSSmoother AreducedPrec((SparseMatrix &)(*Areduced));
+    GMRESSolver AreducedSolver;
+    AreducedSolver.SetOperator(*Areduced);
+    AreducedSolver.SetAbsTol(1.e-12);
+    AreducedSolver.SetRelTol(1.e-8);
+    AreducedSolver.SetMaxIter(500);
+    AreducedSolver.SetPreconditioner(AreducedPrec);
+    AreducedSolver.SetPrintLevel(1);
+    AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+    
+    // now propagate solved uhat to obtain mhat and lhat
+    // xm = Ju xu - bl
+    Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+    Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+    // xl = Wmm xm - bm
+    Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+    Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+    delete Wmmloc;
+    delete Huuloc;
+    delete JuTDJu;
+    delete Juloc;
+    delete Areduced;
+  }
+    
+  
+  /* backsolve to determine zlhat */
+  for(int ii = 0; ii < dimM; ii++)
+  {
+    zlhat(ii) = -1.*(zl(ii) + (zl(ii) * Xhat(ii + dimU) - mu) / (x(ii + dimU) - ml(ii)) );
+  }
+}
+
+// here Xhat, X will be BlockVectors w.r.t. the 4 partitioning X = (u, m, l, zl)
+
+void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double mu)
+{
+  double tau  = max(tauMin, 1.0 - mu);
+  Vector u0   = X0.GetBlock(0);
+  Vector m0   = X0.GetBlock(1);
+  Vector l0   = X0.GetBlock(2);
+  Vector z0   = X0.GetBlock(3);
+  Vector uhat = Xhat.GetBlock(0);
+  Vector mhat = Xhat.GetBlock(1);
+  Vector lhat = Xhat.GetBlock(2);
+  Vector zhat = Xhat.GetBlock(3);
+  double alphaMax  = MaxStepSize(m0, ml, mhat, tau);
+  double alphaMaxz = MaxStepSize(z0, zhat, tau);
+  alphaz = alphaMaxz;
+
+
+  BlockVector x0(block_offsetsx); x0 = 0.0;
+  x0.GetBlock(0).Set(1.0, u0);
+  x0.GetBlock(1).Set(1.0, m0);
+  
+  BlockVector xhat(block_offsetsx); xhat = 0.0;
+  xhat.GetBlock(0).Set(1.0, uhat);
+  xhat.GetBlock(1).Set(1.0, mhat);
+  
+  BlockVector xtrial(block_offsetsx); xtrial = 0.0;
+  BlockVector Dxphi0(block_offsetsx); Dxphi0 = 0.0;
+  int maxBacktrack = 20;
+  alpha = alphaMax;
+
+
+  Vector ck0(dimC); ck0 = 0.0;
+  Vector zhatsoc(dimM); zhatsoc = 0.0;
+  BlockVector Xhatumlsoc(block_offsetsuml); Xhatumlsoc = 0.0;
+  BlockVector xhatsoc(block_offsetsx); xhatsoc = 0.0;
+  Vector uhatsoc(dimU); uhatsoc = 0.0;
+  Vector mhatsoc(dimM); mhatsoc = 0.0;
+
+  Dxphi(x0, mu, Dxphi0);
+  Dxphi0_xhat = InnerProduct(Dxphi0, xhat);
+  double xhat_L2norm = sqrt(InnerProduct(xhat, xhat));
+  double Dxphi_L2norm = sqrt(InnerProduct(Dxphi0, Dxphi0));
+  descentDirection = Dxphi0_xhat < 0. ? true : false;
+  if(descentDirection)
+  {
+    cout << "is a descent direction for the log-barrier objective\n";
+  }
+  else
+  {
+    cout << "is not a descent direction for the log-barrier objective\n";
+  }
+  cout << "Dxphi^T xhat / (|| Dxphi ||_2 * || xhat ||_2) = " << Dxphi0_xhat / (xhat_L2norm * Dxphi_L2norm) << endl;
+  thx0 = theta(x0);
+  phx0 = phi(x0, mu);
+
+  lineSearchSuccess = false;
+  for(int i = 0; i < maxBacktrack; i++)
+  {
+    cout << "\n--------- alpha = " << alpha << " ---------\n";
+
+    // ----- A-5.2. Compute trial point: xtrial = x0 + alpha_i xhat
+    xtrial.Set(1.0, x0);
+    xtrial.Add(alpha, xhat);
+
+    // ------ A-5.3. if not in filter region go to A.5.4 otherwise go to A-5.5.
+    thxtrial = theta(xtrial);
+    phxtrial = phi(xtrial, mu);
+    filterCheck(thxtrial, phxtrial);    
+    if(!inFilterRegion)
+    {
+      cout << "not in filter region :)\n";
+      // ------ A.5.4: Check sufficient decrease
+      if(!descentDirection)
+      {
+        switchCondition = false;
+      }
+      else
+      {
+        switchCondition = (alpha * pow(abs(Dxphi0_xhat), sPhi) > delta * pow(thx0, sTheta)) ? true : false;
+      }
+      cout << "alpha |Dxphi(x0)^T xhat|^sPhi = " << alpha * pow(abs(Dxphi0_xhat), sPhi) << endl;
+      cout << "delta * theta(x0)^sTheta = " << delta * pow(thx0, sTheta) << endl;
+      cout << "theta(x0) = "     << thx0     << ", thetaMin = "                  << thetaMin             << endl;
+      cout << "theta(xtrial) = " << thxtrial << ", (1-gTheta) *theta(x0) = "     << (1. - gTheta) * thx0 << endl;
+      cout << "phi(xtrial) = "   << phxtrial << ", phi(x0) - gPhi *theta(x0) = " << phx0 - gPhi * thx0   << endl;
+      
+      // Case I      
+      if(thx0 <= thetaMin && switchCondition)
+      {
+        sufficientDecrease = phxtrial <= phx0 + eta * alpha * Dxphi0_xhat ? true : false;
+        if(sufficientDecrease)
+        {
+          if(iAmRoot) { cout << "A-5.4. Case I -- accepted step length.\n"; } 
+          // accept the trial step
+          lineSearchSuccess = true;
+          break;
+        }
+      }
+      else
+      {
+        if(thxtrial <= (1. - gTheta) * thx0 || phxtrial <= phx0 - gPhi * thx0)
+        {
+          if(iAmRoot) { cout << "A-5.4. Case II -- accepted step length.\n"; } 
+          // accept the trial step
+          lineSearchSuccess = true;
+          break;
+        }
+      }
+      // A-5.5: Initialize the second-order correction
+      if((!(thx0 < thxtrial)) && i == 0)
+      {
+        cout << "second order correction\n";
+        problem->c(xtrial, ckSoc);
+        problem->c(x0, ck0);
+        ckSoc.Add(alphaMax, ck0);
+        // A-5.6 Compute the second-order correction.
+        IPNewtonSolve(x0, l0, z0, zhatsoc, Xhatumlsoc, mu, true);
+        mhatsoc.Set(1.0, Xhatumlsoc.GetBlock(1));
+        // alphasoc = MaxStepSize(m0, ml, mhatsoc, tau);
+        //WARNING: not complete but currently solver isn't entering this region
+      }
+    }
+    else
+    {
+      cout << "in filter region :(\n"; 
+    }
+
+    // include more if needed
+    alpha *= 0.5;
+
+  } 
+}
+
+void InteriorPointSolver::projectZ(const Vector &x, Vector &z, double mu)
+{
+  double zi;
+  double mudivmml;
+  for(int i = 0; i < dimM; i++)
+  {
+    zi = z(i);
+    mudivmml = mu / (x(i + dimU) - ml(i));
+    z(i) = max(min(zi, kSig * mudivmml), mudivmml / kSig);
+  }
+}
+
+void InteriorPointSolver::filterCheck(double th, double ph)
+{
+  inFilterRegion = false;
+  if(th > thetaMax)
+  {
+    inFilterRegion = true;
+  }
+  else
+  {
+    for(int i = 0; i < F1.Size(); i++)
+    {
+      if(th >= F1[i] && ph >= F2[i])
+      {
+        inFilterRegion = true;
+        break;
+      }
+    }
+  }
+}
+
+double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, double mu, bool print)
+{
+  double E1, E2, E3;
+  double sc, sd;
+  BlockVector gradL(block_offsetsx); gradL = 0.0; // stationarity grad L = grad f + J^T l - z
+  Vector cx(dimC); cx = 0.0;     // feasibility c = c(x)
+  Vector comp(dimM); comp = 0.0; // complementarity M Z - mu 1
+
+  DxL(x, l, zl, gradL);
+  E1 = gradL.Normlinf();
+
+  problem->c(x, cx);
+  E2 = cx.Normlinf();
+
+  for(int ii = 0; ii < dimM; ii++) 
+  { 
+    comp(ii) = x(dimU + ii) * zl(ii) - mu;
+  }
+  E3 = comp.Normlinf();
+
+  double ll1, zl1;
+  zl1 = zl.Norml1() / double(dimC + dimM);
+  ll1 =  l.Norml1();
+  sc = max(sMax, zl1 / (double(dimM)) ) / sMax;
+  sd = max(sMax, (ll1 + zl1) / (double(dimC + dimM))) / sMax;
+  if(iAmRoot && print)
+  {
+    cout << "evaluating optimality error for mu = " << mu << endl;
+    cout << "stationarity measure = "    << E1 / sd << endl;
+    cout << "feasibility measure  = "    << E2      << endl;
+    cout << "complimentarity measure = " << E3 / sc << endl;
+  }
+  return max(max(E1 / sd, E2), E3 / sc);
+}
+
+double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, bool print)
+{
+  return E(x, l, zl, 0.0, print);
+}
+
+double InteriorPointSolver::theta(const BlockVector &x)
+{
+  Vector cx(dimC); cx = 0.0;
+  problem->c(x, cx);
+  return sqrt(InnerProduct(cx, cx));
+}
+
+// log-barrier objective
+double InteriorPointSolver::phi(const BlockVector &x, double mu)
+{
+  double fx = problem->CalcObjective(x); 
+  double logBarrierLoc = 0.0;
+  for(int i = 0; i < dimM; i++) 
+  { 
+    logBarrierLoc += log(x(dimU+i)-ml(i));
+  }
+  double logBarrierGlb = 0.0;
+  logBarrierGlb = logBarrierLoc;
+  return fx - mu * logBarrierGlb;
+}
+
+
+// gradient of log-barrier objective with respect to x = (u, m)
+void InteriorPointSolver::Dxphi(const BlockVector &x, double mu, BlockVector &y)
+{
+  problem->CalcObjectiveGrad(x, y);
+  for(int i = 0; i < dimM; i++) 
+  { 
+    y(dimU + i) -= mu / (x(dimU + i) - ml(i));
+  } 
+}
+
+// Lagrangian function evaluation
+// L(x, l, zl) = f(x) + l^T c(x) - zl^T m
+double InteriorPointSolver::L(const BlockVector &x, const Vector &l, const Vector &zl)
+{
+  double fx = problem->CalcObjective(x);
+  Vector cx(dimC); problem->c(x, cx);
+  return (fx + InnerProduct(cx, l) - InnerProduct(x.GetBlock(1), zl));
+}
+
+void InteriorPointSolver::DxL(const BlockVector &x, const Vector &l, const Vector &zl, BlockVector &y)
+{
+  // evaluate the gradient of the objective with respect to the primal variables x = (u, m)
+  BlockVector gradxf(block_offsetsx); gradxf = 0.0;
+  problem->CalcObjectiveGrad(x, gradxf);
+  
+  SparseMatrix *Jacu, *Jacm, *JacuT, *JacmT;
+  Jacu = problem->Duc(x); Jacm = problem->Dmc(x);
+  JacuT = Transpose(*Jacu);
+  JacmT = Transpose(*Jacm);
+  JacuT->Mult(l, y.GetBlock(0));
+  JacmT->Mult(l, y.GetBlock(1));
+  delete Jacu; delete JacuT;
+  delete Jacm; delete JacmT;
+  y.Add(1.0, gradxf);
+  (y.GetBlock(1)).Add(-1.0, zl);
+}
+
+
+bool InteriorPointSolver::GetConverged() const
+{
+  return converged;
+}
+
+void InteriorPointSolver::SetTol(double Tol)
+{
+  tol = Tol;
+}
+
+void InteriorPointSolver::SetMaxIter(int max_it)
+{
+  max_iter = max_it;
+}
+
+void InteriorPointSolver::SetBarrierParameter(double mu_0)
+{
+  mu_k = mu_0;
+}
+
+void InteriorPointSolver::SaveLogBarrierHessianIterates(bool save)
+{
+  MFEM_ASSERT(MyRank == 0 || save == false, "currently can only save logbarrier hessian in serial codes");
+  saveLogBarrierIterates = save;
+}
+
+void InteriorPointSolver::SetLinearSolver(int LinSolver)
+{
+  linSolver = LinSolver;
+}
+
+
+
+InteriorPointSolver::~InteriorPointSolver() 
+{
+  delete Wmm;
+  delete Huu;
+  delete Hum;
+  delete Hmu;
+  delete Hmm;
+  delete Hum;
+  delete Ju;
+  delete Jm;
+  delete JuT;
+  delete JmT;
+
+  F1.DeleteAll();
+  F2.DeleteAll();
+  block_offsetsx.DeleteAll();
+  block_offsetsumlz.DeleteAll();
+  block_offsetsuml.DeleteAll();
+  ml.SetSize(0);
+}

examples/contact/IPsolver.hpp

@@ -0,0 +1,103 @@
+#include "mfem.hpp"
+#include "problems.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+#ifndef IPSOLVER 
+#define IPSOLVER
+
+class InteriorPointSolver
+{
+protected:
+    OptProblem* problem;
+    double tol;
+    int  max_iter;
+    double mu_k; // \mu_k
+    Vector lk, zlk, mf;
+
+    double sMax, kSig, tauMin, eta, thetaMin, delta, sTheta, sPhi, kMu, thetaMu;
+    double thetaMax, kSoc, gTheta, gPhi, kEps;
+	
+    // filter
+    Array<double> F1, F2;
+	
+    // quantities computed in lineSearch
+    double alpha, alphaz;
+    double thx0, thxtrial;
+    double phx0, phxtrial;
+    bool descentDirection, switchCondition, sufficientDecrease, lineSearchSuccess, inFilterRegion;
+    double Dxphi0_xhat;
+
+    int dimU, dimM, dimC;
+    Array<int> block_offsetsumlz, block_offsetsuml, block_offsetsx;
+    Vector ml;
+
+    Vector ckSoc;
+    SparseMatrix * Huu = nullptr; 
+    SparseMatrix * Hum = nullptr; 
+    SparseMatrix * Hmu = nullptr; 
+    SparseMatrix * Hmm = nullptr; 
+    SparseMatrix * Wmm = nullptr; 
+    SparseMatrix * Ju  = nullptr; 
+    SparseMatrix * Jm  = nullptr; 
+    SparseMatrix * JuT = nullptr; 
+    SparseMatrix * JmT  = nullptr;;
+    
+    int jOpt;
+    bool converged;
+    
+    int MyRank;
+    bool iAmRoot;
+
+    bool saveLogBarrierIterates;
+
+    int linSolver;
+    std::ofstream IPNewtonKrylovIters;
+
+    ParFiniteElementSpace *Vh;
+    Array<int> cgnum_iterations;
+
+
+    // not sure if this data is needed or if it can 
+    // all be accounted for in the problem class
+    // which variables have equality constraints
+    //Array<int> eqConstrainedVariables;
+    //Array<double> eqConstrainedValues;
+
+
+
+public:
+    InteriorPointSolver(OptProblem*, ParFiniteElementSpace *);
+    void Mult(const BlockVector& , BlockVector&); // used when the user wants to be aware of bound-constrained variable m >= ml
+    void Mult(const Vector&, Vector &); // useful when the user doesn't need to know about bound-constrained variable   m >= ml
+    double MaxStepSize(Vector& , Vector& , Vector& , double);
+    double MaxStepSize(Vector& , Vector& , double);
+    void FormIPNewtonMat(BlockVector& , Vector& , Vector& , BlockOperator &);
+    void IPNewtonSolve(BlockVector& , Vector& , Vector& , Vector&, BlockVector& , double, bool);
+    void lineSearch(BlockVector& , BlockVector& , double);
+    void projectZ(const Vector & , Vector &, double);
+    void filterCheck(double, double);
+    double E(const BlockVector &, const Vector &, const Vector &, double, bool);
+    double E(const BlockVector &, const Vector &, const Vector &, bool);
+    bool GetConverged() const;
+    // TO DO: include Hessian of Lagrangian
+    double theta(const BlockVector &);
+    double phi(const BlockVector &, double);
+    void Dxphi(const BlockVector &, double, BlockVector &);
+    double L(const BlockVector &, const Vector &, const Vector &);
+    void DxL(const BlockVector &, const Vector &, const Vector &, BlockVector &);
+    void SetTol(double);
+    void SetMaxIter(int);
+    void SetBarrierParameter(double);    
+    void SaveLogBarrierHessianIterates(bool);
+    void SetLinearSolver(int);
+    Vector GetBoundConstrainedVariable() {return mf;}
+    Array<int> & GetCGIterNumbers() {return cgnum_iterations;}
+    virtual ~InteriorPointSolver();
+};
+
+#endif

examples/contact/README.md

@@ -0,0 +1,17 @@
+# OneProcessAMGContact
+
+
+
+Be sure to edit the makefile so that it points to a parallel MFEM build
+
+specifically the MFEM_BUILD_DIR
+
+
+after building exQPContactBlockTL one can
+
+1. run the bash script scalingJobArray.bat via `source scalingJobArray.bat' which will populate the CG iterations required to solve
+      various linear systems into the data/ subdirectory
+2. run the python script data/process.py in order to put the scaling information into the single files algorithmicScaling_Elasticity.dat and algorithmicScaling_noElasticity.dat
+      in order to see the number of average AMG-CG iterations per optimization solve. 
+
+

examples/contact/exQPContactBlockTL.cpp

@@ -0,0 +1,274 @@
+//                                Contact example
+//
+// Compile with: make contact
+//
+// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
+// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
+
+#include <fstream>
+#include <iostream>
+#include <array>
+
+#include "mfem.hpp"
+#include "problems.hpp"
+#include "IPsolver.hpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+  Mpi::Init(argc, argv);
+  Hypre::Init();
+  int linSolver = 2;
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  int ref_levels = 0;
+
+  OptionsParser args(argc, argv);
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+
+   
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+	
+  // Create an instance of the nlp
+  ExContactBlockTL * contact = new ExContactBlockTL(ref_levels);
+  int ndofs = contact->GetDimD();
+  int nconstraints = contact->GetDimS();
+  std::ofstream problemDimStream;
+  problemDimStream.open("problemDim.dat", ios::out | ios::trunc);
+  problemDimStream << ndofs << endl;
+  problemDimStream.close();
+  std::ofstream problemDimConstraintsStream;
+  problemDimConstraintsStream.open("problemDimConstraints.dat", ios::out | ios::trunc);
+  problemDimConstraintsStream << nconstraints << endl;
+  problemDimConstraintsStream.close();
+  
+  // set up a QP-problem
+  // E(d) = 1 / 2 d^T K d + f^T d
+  // g(d) = J d + g0
+  // where K, J, f and g0 are evaluated at d0 (a valid configuration)
+  
+  // to do: seems more appropriate to evaluate at a valid configuration...
+  //        that is one where the Dirichlet conditions hold... need to pull
+  //        this data from contactBlockTL...
+  Vector d0(ndofs); d0 = 0.0;
+  Array<int> DirichletDofs = contact->GetDirichletDofs();
+  Array<double> DirichletVals = contact->GetDirichletVals();
+  SparseMatrix *K;
+  Vector f(ndofs); f = 0.0;
+  contact->DdE(d0, f); K = contact->DddE(d0);
+  for(int i = 0; i < DirichletDofs.Size(); i++)
+  {
+    d0(DirichletDofs[i]) = DirichletVals[i];
+  }
+  SparseMatrix *J;
+  Vector g0(nconstraints); g0 = 0.0;
+  J = contact->Ddg(d0); contact->g(d0, g0);
+  Vector temp(nconstraints);
+  J->Mult(d0, temp);
+  g0.Add(-1.0, temp);
+
+  // check which rows of the Jacobian are zero!
+  Vector ei(nconstraints); ei = 0.0;
+  Vector JTei(ndofs); JTei = 0.0;
+
+  double normJTei;
+
+  int reduced_nconstraints = 0; // find actual number of constraints
+  
+  
+  Array<int> nonZeroRows;
+  for(int i = 0; i < nconstraints; i++)
+  {
+    ei(i) = 1.0;
+    J->MultTranspose(ei, JTei);
+    // nullify contributions from Dirichlet constrined dofs
+    for(int j = 0; j < DirichletDofs.Size(); j++)
+    {
+      JTei(DirichletDofs[j]) = 0.0;
+    }
+    normJTei = sqrt(InnerProduct(JTei, JTei));
+    if (normJTei > 1.e-12)
+    {
+      reduced_nconstraints += 1;
+      nonZeroRows.Append(i);
+    }
+    ei(i) = 0.0;
+  }
+  cout << "number of linearized constraints = " << reduced_nconstraints << endl; // 9 constraints 
+  
+  // remove zero rows of the gap function Jacobian and corresponding gap function entries
+  SparseMatrix * Jreduced = new SparseMatrix(reduced_nconstraints, ndofs);
+  Vector g0reduced(reduced_nconstraints); g0reduced = 0.0; 
+  
+  
+  for(int i = 0; i < reduced_nconstraints; i++)
+  {
+    Array<int> col_tmp;
+    Vector v_tmp; v_tmp = 0.0;
+    J->GetRow(nonZeroRows[i], col_tmp, v_tmp);
+    
+    /* obtain subset of columns of the given nonZero Jacobian row that are not Dirichlet constrained */
+    bool freeDof;
+    Array<int> loc_indicies;
+    for(int j = 0; j < col_tmp.Size(); j++)
+    {
+      freeDof = true;
+      for(int k = 0; k < DirichletDofs.Size(); k++)
+      {
+        if(col_tmp[j] == DirichletDofs[k])
+        {
+          freeDof = false;
+        }
+      }
+      if(freeDof)
+      {
+        loc_indicies.Append(j);
+      }
+    }
+
+    Array<int> col_tmp_reduced(loc_indicies.Size());
+    Vector v_tmp_reduced(loc_indicies.Size());
+    for(int j = 0; j < loc_indicies.Size(); j++)
+    {
+      col_tmp_reduced[j] = col_tmp[loc_indicies[j]];
+      v_tmp_reduced(j)   = v_tmp(loc_indicies[j]);
+    }
+
+    Jreduced->SetRow(i, col_tmp_reduced, v_tmp_reduced);
+    g0reduced(i) = g0(nonZeroRows[i]);
+  }
+
+
+  QPContactProblem *QPContact = new QPContactProblem(*K, *Jreduced, f, g0reduced);
+  
+  Mesh * mesh1 = new Mesh("meshes/block1.mesh", 1, 1);
+  Mesh * mesh2 = new Mesh("meshes/rotatedblock2.mesh", 1, 1);
+  for(int i = 0; i < ref_levels; i++)
+  {
+     mesh1->UniformRefinement();
+     mesh2->UniformRefinement(); 
+  }
+  
+  int numMeshes = 2;
+  Mesh *meshArray[numMeshes];
+  meshArray[0] = mesh1;
+  meshArray[1] = mesh2;
+  Mesh mesh(meshArray, numMeshes);
+  
+  ParMesh pmesh(MPI_COMM_WORLD, mesh);  
+  H1_FECollection fec(1, mesh.Dimension());
+  ParFiniteElementSpace fespace(&pmesh, &fec, mesh.Dimension(), Ordering::byVDIM);
+
+  InteriorPointSolver * QPContactOptimizer = new InteriorPointSolver(QPContact, &fespace);
+  QPContactOptimizer->SetTol(1.e-6);
+  QPContactOptimizer->SetLinearSolver(linSolver);
+  QPContactOptimizer->SetMaxIter(50);
+  Vector x0(ndofs); x0 = 0.0;
+  for(int i = 0; i < DirichletDofs.Size(); i++)
+  {
+    x0(DirichletDofs[i]) = DirichletVals[i];
+  }
+  Vector xf(ndofs); xf = 0.0;
+  QPContactOptimizer->Mult(x0, xf);
+  
+  double Einitial = QPContact->E(x0);
+  double Efinal = QPContact->E(xf);
+  cout << "Energy objective at initial point = " << Einitial << endl;
+  cout << "Energy objective at QP optimizer = " << Efinal << endl;
+  QPContactOptimizer->GetCGIterNumbers().Print(mfem::out, 20);
+  MFEM_VERIFY(QPContactOptimizer->GetConverged(), "Interior point solver did not converge.");
+  
+  
+  //Mesh * mesh1 = new Mesh("meshes/block1.mesh", 1, 1);
+  //Mesh * mesh2 = new Mesh("meshes/rotatedblock2.mesh", 1, 1);
+  //for(int i = 0; i < ref_levels; i++)
+  //{
+  //   mesh1->UniformRefinement();
+  //   mesh2->UniformRefinement(); 
+  //}
+  //int gdim = mesh1->Dimension();
+  //FiniteElementCollection * fec = new H1_FECollection(1, gdim);
+  //FiniteElementSpace * fespace1 = new FiniteElementSpace(mesh1, fec, gdim, Ordering::byVDIM);
+  //FiniteElementSpace * fespace2 = new FiniteElementSpace(mesh2, fec, gdim, Ordering::byVDIM);
+  // 
+  //GridFunction x1_gf(fespace1);
+  //GridFunction x2_gf(fespace2);
+
+  //int ndof1 = fespace1->GetTrueVSize();
+  //int ndof2 = fespace2->GetTrueVSize();
+  //int ndof  = ndof1 + ndof2;
+  //for(int i = 0; i < ndof1; i++)
+  //{
+  //  x1_gf(i) = xf(i);
+  //}
+  //for(int i = ndof1; i < ndof; i++)
+  //{
+  //  x2_gf(i - ndof1) = xf(i);
+  //}
+
+  //mesh1->SetNodalFESpace(fespace1);
+  //mesh2->SetNodalFESpace(fespace2);
+  //GridFunction *nodes1 = mesh1->GetNodes();
+  //GridFunction *nodes2 = mesh2->GetNodes();
+
+  //{
+  //  *nodes1 += x1_gf;
+  //  *nodes2 += x2_gf;
+  //}
+  //
+
+  //ParaViewDataCollection paraview_dc1("QPContactBody1", mesh1);
+  //paraview_dc1.SetPrefixPath("ParaView");
+  //paraview_dc1.SetLevelsOfDetail(1);
+  //paraview_dc1.SetDataFormat(VTKFormat::BINARY);
+  //paraview_dc1.SetHighOrderOutput(true);
+  //paraview_dc1.SetCycle(0);
+  //paraview_dc1.SetTime(0.0);
+  //paraview_dc1.RegisterField("Body1", &x1_gf);
+  //paraview_dc1.Save();
+  //
+  //ParaViewDataCollection paraview_dc2("QPContactBody2", mesh2);
+  //paraview_dc2.SetPrefixPath("ParaView");
+  //paraview_dc2.SetLevelsOfDetail(1);
+  //paraview_dc2.SetDataFormat(VTKFormat::BINARY);
+  //paraview_dc2.SetHighOrderOutput(true);
+  //paraview_dc2.SetCycle(0);
+  //paraview_dc2.SetTime(0.0);
+  //paraview_dc2.RegisterField("Body2", &x2_gf);
+  //paraview_dc2.Save();
+
+  //delete fespace1;
+  //delete fespace2;
+  //delete fec;
+  //delete mesh1;
+  //delete mesh2;
+  
+  delete QPContact;
+  delete QPContactOptimizer;
+  
+  delete K;
+  delete J;
+  delete Jreduced;
+  delete contact;
+  return 0;
+}

examples/contact/makefile

@@ -0,0 +1,36 @@
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = ./
+
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+# Remove built-in rule
+#%: %.cpp
+
+exQPContactBlockTL: exQPContactBlockTL.o problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) exQPContactBlockTL.o problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+
+
+
+exQPContactBlockTL.o: exQPContactBlockTL.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+problems.o: problems.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
+
+IPsolver.o: IPsolver.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+.PHONY: clean
+clean:
+	rm -f *.o exQPContactBlockTL
+
+

examples/contact/meshes/block1.mesh

@@ -0,0 +1,103 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+9
+1   5     0      1     3     2     8    9     11    10
+1   5     2      3     5     4    10    11    13    12
+1   5     4      5     7     6    12    13    15    14
+1   5     8      9    11    10    16    17    19    18
+1   5     10    11    13    12    18    19    21    20
+1   5     12    13    15    14    20    21    23    22
+1   5     16    17    19    18    24    25    27    26
+1   5     18    19    21    20    26    27    29    28
+1   5     20    21    23    22    28    29    31    30
+
+
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+30
+1   3    1     0     2     3
+1   3    3     2     4     5
+1   3    5     4     6     7
+1   3    24    25    27    26
+1   3    26    27    29    28
+1   3    28    29    31    30
+2   3    2     0     8    10
+2   3    4     2    10    12
+2   3    6     4    12    14
+2  3    10    8    16    18
+2  3    12    10    18    20
+2  3    14    12    20    22
+2  3    18    16    24    26
+2  3    20    18    26    28
+2  3    22    20    28    30
+3  3    1     3    11    9
+3  3     3     5    13    11
+3  3     5     7    15    13
+3  3    9    11    19    17
+3  3    11    13    21    19
+3  3    13    15    23    21
+3  3    17    19    27    25
+3  3    19    21    29    27
+3  3    21    23    31    29
+1  3     8     0     1    9
+1  3    16     8    9    17
+1  3    24    16    17    25
+1  3     6    14    15     7
+1  3    14    22    23    15
+1  3    22    30    31    23
+
+
+vertices
+32
+3
+   -1.0000         0         0
+         0         0         0
+   -1.0000    0.3000         0
+         0    0.3000         0
+   -1.0000    0.6500         0
+         0    0.6500         0
+   -1.0000    1.0000         0
+         0    1.0000         0
+   -1.0000         0    0.3000
+         0         0    0.3000
+   -1.0000    0.3000    0.3500
+         0    0.3000    0.3500
+   -1.0000    0.6500    0.3000
+         0    0.6500    0.3000
+   -1.0000    1.0000    0.3000
+         0    1.0000    0.3000
+   -1.0000         0    0.6500
+         0         0    0.6500
+   -1.0000    0.3000    0.6500
+         0    0.3000    0.6500
+   -1.0000    0.6500    0.6500
+         0    0.6500    0.6500
+   -1.0000    1.0000    0.6500
+         0    1.0000    0.6500
+   -1.0000         0    1.0000
+         0         0    1.0000
+   -1.0000    0.3000    1.0000
+         0    0.3000    1.0000
+   -1.0000    0.6500    1.0000
+         0    0.6500    1.0000
+   -1.0000    1.0000    1.0000
+         0    1.0000    1.0000

examples/contact/meshes/rotatedblock2.mesh

@@ -0,0 +1,70 @@
+
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+# 1 nothing
+elements
+4
+1   5     0     1     3     2     6     7    9     8
+1   5     2     3     5     4     8     9    11    10
+1   5     6     7     9     8    12    13    15    14
+1   5     8     9    11    10    14    15    17    16
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+16
+1   3     1     0     2     3
+1   3     3     2     4     5
+1   3    12    13    15    14
+1   3    14    15    17    16
+3   3     2     0     6     8
+3   3     4     2     8    10
+3   3     8     6    12    14
+3   3    10     8    14    16
+2   3     1     3     9     7
+2   3     3     5    11    9
+2   3     7    9    15    13
+2   3     9    11    17    15
+1   3     6     0     1     7
+1   3    12     6     7    13
+1   3     4    10    11     5
+1   3    10    16    17    11
+
+vertices
+18
+3
+
+0.000000000000 0.145770950245 0.443895630208
+0.507100000000 0.145770950245 0.443895630208
+0.000000000000 0.350937660019 0.294833290227
+0.507100000000 0.350937660019 0.294833290227
+0.000000000000 0.556104369792 0.145770950245
+0.507100000000 0.556104369792 0.145770950245
+0.000000000000 0.294833290227 0.649062339981
+0.507100000000 0.294833290227 0.649062339981
+0.000000000000 0.500000000000 0.500000000000
+0.507100000000 0.500000000000 0.500000000000
+0.000000000000 0.705166709773 0.350937660019
+0.507100000000 0.705166709773 0.350937660019
+0.000000000000 0.443895630208 0.854229049755
+0.507100000000 0.443895630208 0.854229049755
+0.000000000000 0.649062339981 0.705166709773
+0.507100000000 0.649062339981 0.705166709773
+0.000000000000 0.854229049755 0.556104369792
+0.507100000000 0.854229049755 0.556104369792

examples/contact/nodepair.hpp

@@ -0,0 +1,897 @@
+
+using namespace std;
+using namespace mfem;
+
+void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]);
+   dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);
+   dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]);
+   dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);
+   dNdxi(1,3) = 0.25*(1-xi[0]);
+}
+
+
+void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
+                     DenseMatrix& dN2dxi)
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(2,4); dNdxi = 0.0;
+   dN2dxi.SetSize(3,4);
+   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
+
+   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
+   dN2dxi(1,3) = -0.25;
+}
+
+// returns the vector and matrix form of the shape functions and its derivative
+void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
+                       DenseMatrix& ddNdxi)
+{
+   N.SetSize(3,12); N = 0.0;
+   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
+   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
+   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
+   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
+   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
+   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
+   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
+
+   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
+   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
+   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
+   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
+   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
+   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
+
+   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
+   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
+   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
+   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
+   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
+   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
+   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
+
+   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
+   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
+   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
+   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
+   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
+   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
+}
+
+
+void cross(const Vector a, const Vector b, Vector& c)
+{
+   assert(a.Size()==3);
+   c.SetSize(3);
+   c[0] = a[1]*b[2] - a[2]*b[1];
+   c[1] = -a[0]*b[2] + b[0]*a[2];
+   c[2] = a[0]*b[1] - a[1]*b[0];
+
+}
+// a outer b
+void outer(const Vector a, const Vector b, DenseMatrix& c)
+{
+   int m = a.Size();
+   int n = b.Size();
+   assert(c.Height()==m);
+   assert(c.Width() ==n);
+   for (int i=0; i<m; i++)
+   {
+      for (int j=0; j<n; j++)
+      {
+         c(i,j) = a[i]*b[j];
+      }
+   }
+}
+// dphidxi 2*4
+// coords 4*3
+void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
+                   Vector& normal, double& nnorm)
+{
+
+   DenseMatrix dxdxi(2,3);
+   Mult(dphidxi, coords, dxdxi);
+   Vector dxdxi1(3);
+   Vector dxdxi2(3);
+
+   dxdxi.GetRow(0,dxdxi1);
+   dxdxi.GetRow(1,dxdxi2);
+
+   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
+   // VectorCrossProductCoefficient::Eval has hard-coded cross product
+   nnorm = normal.Norml2( );
+   normal /=  nnorm;
+}
+
+void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
+{
+   bool converged = false;
+   bool pt_on_elem = false;
+   int dim = 3;
+   xi.SetSize(dim-1);
+   xi = 0.0;
+   int max_iter = 15;
+   double off_el_xi = 1e-2;
+   double proj_newton_tol = 1e-13;
+   double proj_max_gap = 0.5;
+   Vector gap_v(dim);
+   // warm start from linear solution
+
+   for (int it=0; it<max_iter; it++)
+   {
+      //cout<<it<<endl;
+      Vector m_N(4);
+      m_N = 0.;
+      DenseMatrix m_dN(2,4);
+      m_dN = 0.;
+      DenseMatrix m_dN2(3,4);
+      m_dN2 = 0.;
+      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+      Vector x_c(dim);
+      m_coords.MultTranspose(m_N, x_c);
+
+      gap_v = s_x;
+      gap_v -= x_c;
+
+      DenseMatrix m_dx(2,3);
+      m_dx = 0.;
+      Mult(m_dN, m_coords, m_dx);
+
+      Vector r(dim-1);
+      r = 0.0;
+      m_dx.Mult(gap_v, r);
+
+      if (r.Normlinf() < proj_newton_tol)
+      {
+         converged = true;
+         break;
+      }
+
+      DenseMatrix drdxi(dim-1,dim-1);
+      drdxi = 0.;
+      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
+      drdxi *= -1.0;
+
+      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+      Mult(m_dN2,m_coords, m_dx2);
+
+      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
+      for (int d=0; d<3; d++)
+      {
+         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+         drdxi.Add(gap_v[d], Mtemp);
+      }
+
+      //cond_num = rcond(drdxi);  condition number?
+      //drdxi.TestInversion();
+      DenseMatrixInverse drdxi_inv(drdxi);
+      Vector xi_tmp(dim-1);
+
+      drdxi_inv.Mult(r,xi_tmp);
+      xi -= xi_tmp;
+   }
+   if (!converged)
+   {
+      xi = 0.0;
+   }
+   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
+
+   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
+   //
+   // Discuss with Frank... what is happening here
+   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
+   {
+      pt_on_elem = true;
+   }
+
+   if (pt_on_elem)
+   {
+     //cout << "convergence of node to segment projection? " << converged << endl;
+     //for(int i = 0; i < 2; i++)
+     //{
+     //  cout << "xi_" << i << " = " << xi(i) << endl;
+     //}
+   }
+   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
+   MFEM_VERIFY(converged == true, "projection didn't converge");
+}
+
+
+
+// m_coords is expected to be 4 * 3
+void  ComputeGapJacobian(const Vector x_s, const Vector xi,
+                         const DenseMatrix m_coords,
+                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N, x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   double nnorm = 0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   gap = gap_v * normal; // gap function value, dot product between vectors
+
+   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
+
+   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
+   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
+
+   Vector m_dxrow1(3);
+   m_dx.GetRow(0, m_dxrow1);
+   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
+   dr_dx_res1 *= -1.0;
+
+   Vector m_dxrow2(3);
+   m_dx.GetRow(1, m_dxrow2);
+   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
+   dr_dx_res2 *= -1.0;
+
+   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
+   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
+
+   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
+   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
+   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
+   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
+
+   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
+   dr_dx_res2 += dr_dx_res2_tmp;
+
+
+   DenseMatrix K_dxidx1(2,2); // 2*2
+   K_dxidx1 = 0.;
+   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   // how to get 2nd order? multidimensional matrix?
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= K_dxidx1;
+   K_dxidx += K_dxidx2;
+
+   // resize the vectors and matrices
+   Vector dxidx(24); dxidx = 0.0;
+   Vector drdx_r(24); drdx_r = 0.0;
+
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         drdx_r[4*j+i] = dr_dx_res1(i,j);
+         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
+
+      }
+   }
+   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
+   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
+   DenseMatrix drdx_K(24,24); drdx_K = 0.;
+   for (int i =0; i<12; i++)
+   {
+      drdx_K(i,i) = K_dxidx(0,0);
+      drdx_K(i,12+i) = K_dxidx(0,1);
+      drdx_K(12+i,i) = K_dxidx(1,0);
+      drdx_K(12+i,12+i) = K_dxidx(1,1);
+   }
+
+   DenseMatrixInverse drdxK_inv(drdx_K);
+   drdxK_inv.Mult(drdx_r,dxidx);
+   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
+   dxidx *= -1.0;
+
+
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+
+   Vector dxidxs(6); dxidxs = 0.0;
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
+
+   dgdxm.SetSize(12); dgdxm = 0.;
+   DenseMatrix dgdxm_tmp(4,3);
+   outer(m_N, normal,dgdxm_tmp);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
+      }
+   }
+   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
+
+   dgdxs.SetSize(3);
+   dgdxs += normal;
+   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
+};
+
+void ComputeGapHessian(const Vector x_s, const Vector xi,
+                       const DenseMatrix m_coords,
+                       DenseMatrix& dg2dx)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   int dim = 3;
+   int num_dofs1 = dim;
+   int num_dofs2 = 4*dim;
+   int num_dofs = num_dofs1 + num_dofs2;
+   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N,x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+   Mult(m_dN2,m_coords, m_dx2);
+   double nnorm = 0.0;
+   Vector normal(3); normal = 0.0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   double gap = gap_v * normal; // gap function value, dot product between vectors
+
+   DenseMatrix M(2,2); M = 0.0;
+   MultABt(m_dx, m_dx, M);
+
+   DenseMatrix f(2, num_dofs2); f = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+      M.Add(-gap_v[d], Mtemp);
+
+      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
+      DenseMatrix ftmp(2,4);
+      outer(m_dxcol, m_N, ftmp);
+      ftmp *= -1;
+      ftmp.Add( gap_v[d], m_dN);   // 2*4
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         f(0,d+j*3) = ftmp(0,j);
+         f(1,d+j*3) = ftmp(1,j);
+      }
+   }
+   //fprintf('hess dxidxm\n');
+   DenseMatrixInverse Minv(M);
+   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
+   Minv.Mult(f, dxidxm);
+   //LinearSolve??
+   //dxidxm = M\f;
+
+   DenseMatrix nde2(2,2); nde2 = 0.0;
+   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
+      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
+      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
+
+      nde2 += ndetmp;
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
+         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
+      }
+
+   }
+
+   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
+   Ndn += Nndx2;
+   AddMult(nde2, dxidxm, Ndn);
+
+
+   DenseMatrix M2(2,2); M2 = 0.0;
+   MultABt(m_dx, m_dx, M2);
+   DenseMatrixInverse M2inv(M2);
+   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
+   DenseMatrix m_con(2,2); m_con = 0.0;
+
+   M2inv.Mult(diag2, m_con);
+
+   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
+
+   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
+   MultAtB(Ndn, m_con, dg2dxm_tmp);
+   Mult(dg2dxm_tmp, Ndn, dg2dxm);
+   dg2dxm *= gap;
+
+   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
+   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   dg2dxm_tmp = 0.0;
+   MultAtB(dxidxm, nde2, dg2dxm_tmp);
+
+   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
+
+   dg2dxm_tmp2 = 0.0;
+   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= M2;
+   K_dxidx += K_dxidx2;
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+   Vector dxidxs(6);
+
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
+   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
+   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
+
+   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
+   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
+
+   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
+   dxidxs_row2 = 0.0;
+   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
+   Vector mdx2_row3(3); mdx2_row3 = 0.0;
+   dxidxs_m.GetRow(0,dxidxs_row1);
+   dxidxs_m.GetRow(1,dxidxs_row2);
+   m_dx2.GetRow(0,mdx2_row1);
+   m_dx2.GetRow(1,mdx2_row2);
+   m_dx2.GetRow(2,mdx2_row3);
+
+   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
+   outer(mdx2_row1, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row2, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+
+   outer(mdx2_row2, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row3, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+
+   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
+
+   for (int d=0; d<3; d++)
+   {
+      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
+      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
+      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
+
+      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
+      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
+
+      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
+      m_dx.GetRow(0, m_dxrow);
+      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
+      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
+      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
+
+      dtaodxs_tmp2 += dtaodxs_tmp;
+      dtaodxs.SetCol(d, dtaodxs_tmp2);
+   }
+
+   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
+   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
+   outer(normal, normal, dndxs_tmp);
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
+
+   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
+   MultAtB(m_dx, dxidxs_m, dgvdxs);
+   dgvdxs *= -1;
+   for (int d=0; d<3; d++)
+   {
+      dgvdxs(d,d) += 1.0;
+   }
+   //dxidxs: 2*3
+
+   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
+   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
+   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
+   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
+   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
+   dg2dxs += dg2dxs_tmp2;
+   dg2dxs_tmp2 = 0.0;
+   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
+   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
+
+   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
+   BasisVectorDerivs(xi, Ne, Be, dBe);
+
+   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
+   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
+
+   Vector m_coords_v(12);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         m_coords_v[i*3+j] = m_coords(i,j);
+      }
+   }
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix dBe_tmp(3,12);
+      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
+
+      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
+
+      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
+      dBe_tmp = 0.0;
+      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
+
+   }
+
+   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
+
+   for (int d=0; d<12; d++)
+   {
+      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
+      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
+      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
+      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
+      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
+      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
+
+      cross(tmp1, m_dxrow2, dtaodxm_tmp);
+      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
+      dtaodxm_tmp += dtaodxm_tmp2;
+
+      dtaodxm.SetCol(d, dtaodxm_tmp);
+   }
+
+   DenseMatrix dndxm(3,12); dndxm = 0.0;
+   dndxm += dtaodxm;
+   dndxm *= 1.0/nnorm;
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
+
+   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
+   dgvdxm -= Ne;
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
+      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
+
+   }
+
+   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
+   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
+   MultAtB(dgvdxs, dndxm, dg2dxsxm);
+
+   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
+   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
+
+   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
+   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
+
+   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
+
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
+      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
+      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
+   }
+
+   dg2dxsxm += dgvdxsxmn;
+
+   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
+   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
+   MultAtB(dgvdxm, dndxs, dg2dxmxs);
+   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
+   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
+
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
+
+   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
+   dgvdxmxsn_tmp *= -1.0;
+   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+      Be_tmp.Transpose(); // Be is now 12*3
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
+      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
+      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
+
+   }
+
+   dg2dxmxs += dgvdxmxsn;
+
+   dg2dx.CopyMN(dg2dxs, 0, 0);
+   dg2dx.CopyMN(dg2dxm, 3, 3);
+   dg2dx.CopyMN(dg2dxsxm, 0, 3);
+   dg2dx.CopyMN(dg2dxmxs, 3, 0);
+
+};
+
+
+
+void NodeSegConPairs(const Vector x1, const Vector xi2,
+                     const DenseMatrix coords2,
+                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
+{
+   double gap = 0.0;
+   Vector normal(3); normal = 0.0;
+   Vector dgdxm(12); dgdxm = 0.0;
+   Vector dgdxs(3); dgdxs = 0.0;
+
+   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
+   node_g = gap;
+
+   node_dg.SetSize(12+3);
+   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
+   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
+
+   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   ComputeGapHessian(x1, xi2, coords2, dg2dx);
+
+   node_dg2.SetSize(15,15);
+   node_dg2 = dg2dx;
+
+   /*
+   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
+
+   v1 = 1:3;
+   v2 = 1:12;
+   %v1 = ones(1,3)
+   %v2 = ones(1,12)
+   v2 = reshape(v2,4,3);
+   x1n1 = x1 + 0.01*v1;
+   coords2n1 = coords2 + 0.001*v2;
+      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
+      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
+   x1n2 = x1 - 0.01*v1;
+   coords2n2 = coords2 - 0.001*v2;
+      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
+      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
+   fprintf('fd\n');
+      %gapv1-gapv2
+   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
+
+   %dgdxsn1-dgdxsn2
+   fprintf('code\n');
+   v2n = v2';
+   %dg2dx(1:3,1:3)*0.04*ones(3,1)
+   temp = zeros(12,3);
+   for i = 1:4
+   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
+   temp((i-1)*3+1:i*3,:) = temp1';
+   end
+   temp2 = zeros(3,12);
+   for i = 1:4
+   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
+   temp2(:,(i-1)*3+1:i*3) = temp3';
+   end
+   %dg2dx
+   %dg2dx(4:end,1:3) = temp;
+   %dg2dx(1:3,4:end) = temp2;
+   %dgvdxm * 0.002*v2n(:)
+   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
+   %dg2dx(4:end,1:3)
+   end*/
+
+};
+
+
+// coordsm : (npoints*4, 3) use what class?
+// m_conn: (npoints*4)
+void Assemble_Contact(const int m, const int npoints, const int ndofs,
+                      const Vector x_s,
+                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
+                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
+                      std::vector<SparseMatrix>& dM)
+{
+   int ndim = 3;
+
+   g.SetSize(m);
+   g = 0.0;
+
+   //SparseMatrix M(m, n);  // M needs to be the correct size
+
+   //dM.resize(m); // needs to clear?
+
+   double g_tmp = 0.;
+   Vector dg(4*ndim+ndim);
+   dg = 0.;
+   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
+   dg2 = 0.;
+
+   for (int i=0; i<npoints; i++)
+   {
+      Vector x1(ndim);
+      x1[0] = x_s[i*ndim];
+      x1[1] = x_s[i*ndim+1];
+      x1[2] = x_s[i*ndim+2];
+
+      Vector xi2(ndim-1);
+      xi2[0] = xi[i*(ndim-1)];
+      xi2[1] = xi[i*(ndim-1)+1];
+
+      DenseMatrix coords2(4,3);
+      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
+
+      //how to get coords2?
+      dg = 0.0;
+      dg2 = 0.;
+      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
+      g[s_conn[i]] = g_tmp; // should be unique
+      Array<int> m_conn_i(4);
+      m_conn.GetSubArray(4*i, 4, m_conn_i);
+
+      Array<int> node_conn(5);
+      node_conn[0] = s_conn[i];
+      for (int j=0; j<4; j++)
+      {
+         node_conn[j+1] = m_conn_i[j];
+      }
+
+      Array<int>  M_i_tmp(1);
+      M_i_tmp[0] = s_conn[i];
+
+      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
+      Array<int> j_idx(5*ndim); j_idx = 0;
+      for (int j=0; j< 5; j++)
+      {
+         for (int k=0; k<ndim; k++)
+         {
+            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
+         }
+      }
+      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
+      M_v_tmp.SetRow(0, dg);
+
+      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
+
+      Array<int> dM_i(ndim*(4+1));
+      Array<int> dM_j(ndim*(4+1));
+
+      for (int j=0; j< ndim*(4+1); j++)
+      {
+         dM_i[j] = j_idx[j];
+         dM_j[j] = j_idx[j];
+      }
+      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
+      dM[s_conn[i]].Finalize();
+      dM[s_conn[i]].Threshold(0.0);
+      dM[s_conn[i]].SortColumnIndices();
+   }
+   M.Finalize();
+   M.Threshold(0.0);
+   M.SortColumnIndices();
+};
+

examples/contact/problems.hpp

@@ -0,0 +1,396 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <set>
+using namespace std;
+using namespace mfem;
+
+#ifndef PROBLEM_DEFS
+#define PROBLEM_DEFS
+
+
+
+// abstract OptProblem class
+// of the form
+// min_(u,m) f(u,m) s.t. c(u,m)=0 and m>=ml
+// the primal variable (u, m) is represented as a BlockVector
+
+class OptProblem
+{
+protected:
+    int dimU, dimM, dimC;
+    Array<int> block_offsetsx;
+    Vector ml;
+public:
+    OptProblem();
+    virtual double CalcObjective(const BlockVector &) const = 0;
+    virtual void Duf(const BlockVector &, Vector &) const = 0;
+    virtual void Dmf(const BlockVector &, Vector &) const = 0;
+    void CalcObjectiveGrad(const BlockVector &, BlockVector &) const;
+    virtual SparseMatrix* Duuf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dumf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmuf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmmf(const BlockVector &) = 0;
+    virtual void c(const BlockVector &, Vector &) const = 0;
+    virtual SparseMatrix* Duc(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmc(const BlockVector &) = 0;
+    // TO DO: include Hessian terms of constraint c
+    // TO DO: include log-barrier lumped-mass and pass that
+    // to the optimizer
+    //virtual SparseMatrix* GetLogBarrierLumpedMass() = 0;
+    int GetDimU() const { return dimU; };
+    int GetDimM() const { return dimM; }; 
+    int GetDimC() const { return dimC; };
+    Vector Getml() const { return ml; };
+    ~OptProblem();
+};
+
+
+// abstract ContactProblem class
+// of the form
+// min_d e(d) s.t. g(d) >= 0
+// TO DO: add functionality for gap function Hessian apply 
+class ContactProblem : public OptProblem
+{
+protected:
+    int dimD;
+    int dimS;
+    Array<int> block_offsetsx;
+public:
+    //ContactProblem(int, int);        // constructor
+    ContactProblem();
+    void InitializeParentData(int, int);
+    double CalcObjective(const BlockVector &) const; // objective e
+    void Duf(const BlockVector &, Vector &) const;
+    void Dmf(const BlockVector &, Vector &) const;
+    SparseMatrix* Duuf(const BlockVector &);
+    SparseMatrix* Dumf(const BlockVector &);
+    SparseMatrix* Dmuf(const BlockVector &);
+    SparseMatrix* Dmmf(const BlockVector &);
+    void c(const BlockVector &, Vector &) const;
+    SparseMatrix* Duc(const BlockVector &);
+    SparseMatrix* Dmc(const BlockVector &);
+    virtual double E(const Vector &) const = 0;           // objective e(d) (energy function)
+    virtual void DdE(const Vector &, Vector &) const = 0; // gradient of objective De / Dd
+    virtual SparseMatrix* DddE(const Vector &) = 0;       // Hessian of objective D^2 e / D d^2
+    virtual void g(const Vector &, Vector &) const = 0;   // inequality constraint g(d) >= 0 (gap function)
+    virtual SparseMatrix* Ddg(const Vector &) = 0;        // Jacobian of inequality constraint Dg / Dd
+    int GetDimD() const { return dimD; };
+    int GetDimS() const { return dimS; };
+    virtual ~ContactProblem();
+};
+
+
+class ObstacleProblem : public ContactProblem
+{
+protected:
+   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d >= 0
+   // stiffness matrix used to define objective
+   BilinearForm *Kform;
+   LinearForm   *fform;
+   Array<int> empty_tdof_list; // needed for calls to FormSystemMatrix
+   SparseMatrix  K;
+   SparseMatrix  *J;
+   FiniteElementSpace *Vh;
+   Vector f;
+public : 
+   ObstacleProblem(FiniteElementSpace* , double (*fSource)(const Vector &));
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   // TO DO: include lumped-mass for the log-barrier term
+   //SparseMatrix* GetLogBarrierLumpedMass();
+   virtual ~ObstacleProblem();
+};
+
+class DirichletObstacleProblem : public ContactProblem
+{
+protected:
+   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d + \psi >= 0
+   // stiffness matrix used to define objective
+   BilinearForm *Kform;
+   LinearForm   *fform;
+   Array<int> ess_tdof_list; // needed for calls to FormSystemMatrix
+   SparseMatrix  *K;
+   SparseMatrix  *J;
+   FiniteElementSpace *Vh;
+   Vector f;
+   Vector psi;
+   Vector xDC;
+public : 
+   DirichletObstacleProblem(FiniteElementSpace*, Vector&,  double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list, bool);
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   virtual ~DirichletObstacleProblem();
+};
+
+
+// abstract out technology for removing null rows of the Jacobian from an existing contact problem
+class ReducedContactProblem : public ContactProblem
+{
+protected:
+   Array<int> activeConstraints;
+   Array<int> fixedDofs;
+   ContactProblem * contact;
+   int dimSin;
+public:
+   ReducedContactProblem(ContactProblem * contact, Array<int> activeConstraints, Array<int> fixedDofs);
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   virtual ~ReducedContactProblem();
+};
+
+
+class QPContactProblem : public ContactProblem
+{
+protected:
+  SparseMatrix *K;
+  SparseMatrix *J;
+  Vector f;
+  Vector g0;
+public:
+  QPContactProblem(const SparseMatrix, const SparseMatrix, const Vector, const Vector);
+  double E(const Vector &) const;
+  void DdE(const Vector &, Vector &) const;
+  SparseMatrix* DddE(const Vector &);
+  void g(const Vector &, Vector &) const;
+  SparseMatrix* Ddg(const Vector &);
+  virtual ~QPContactProblem();
+};
+
+
+typedef int Index;
+typedef double Number;
+
+class ExContactBlockTL : public ContactProblem
+{
+public:
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   FiniteElementSpace GetVh1();
+   FiniteElementSpace GetVh2();
+
+public:
+   /** default constructor */
+   ExContactBlockTL(int );
+   
+
+   /** default destructor */
+   virtual ~ExContactBlockTL();
+
+   ///**@name Overloaded from TNLP */
+   ///** Method to return some info about the nlp */
+   //virtual bool get_nlp_info(
+   //   Index&          n,
+   //   Index&          m,
+   //   Index&          nnz_jac_g,
+   //   Index&          nnz_h_lag,
+   //   IndexStyleEnum& index_style
+   //);
+
+   ///** Method to return the bounds for my problem */
+   //virtual bool get_bounds_info(
+   //   Index   n,
+   //   Number* x_l,
+   //   Number* x_u,
+   //   Index   m,
+   //   Number* g_l,
+   //   Number* g_u
+   //);
+
+   ///** Method to return the starting point for the algorithm */
+   //virtual bool get_starting_point(
+   //   Index   n,
+   //   bool    init_x,
+   //   Number* x,
+   //   bool    init_z,
+   //   Number* z_L,
+   //   Number* z_U,
+   //   Index   m,
+   //   bool    init_lambda,
+   //   Number* lambda
+   //);
+
+   /* Method to return the objective value */
+   virtual bool eval_f(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number&       obj_value
+   ) const;
+
+   /* Method to return the gradient of the objective */
+   virtual bool eval_grad_f(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number*       grad_f
+   ) const;
+
+   /* Method to return the constraint residuals */
+   virtual bool eval_g(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Index         m,
+      Number*       cons
+   ) const;
+
+   /* Method to return:
+      1) The structure of the Jacobian (if "values" is NULL)
+      2) The values of the Jacobian (if "values" is not NULL)
+   */
+   virtual bool eval_jac_g(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Index         m,
+      Index         nele_jac,
+      Index*        iRow,
+      Index*        jCol,
+      Number*       values
+   ) const;
+
+   /* Method to return:
+    *   1) The structure of the Hessian of the Lagrangian (if "values" is NULL)
+    *   2) The values of the Hessian of the Lagrangian (if "values" is not NULL)
+   */
+   virtual bool eval_h(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number        obj_factor,
+      Index         m,
+      const Number* lambda,
+      bool          new_lambda,
+      Index         nele_hess,
+      Index*        iRow,
+      Index*        jCol,
+      Number*       values
+   );
+
+   ///** This method is called when the algorithm is complete so the TNLP can store/write the solution */
+   //virtual void finalize_solution(
+   //   SolverReturn               status,
+   //   Index                      n,
+   //   const Number*              x,
+   //   const Number*              z_L,
+   //   const Number*              z_U,
+   //   Index                      m,
+   //   const Number*              g,
+   //   const Number*              lambda,
+   //   Number                     obj_value,
+   //   const IpoptData*           ip_data,
+   //   IpoptCalculatedQuantities* ip_cq
+   //);
+
+private:
+   void update_g() const;
+   void update_jac();
+   void update_hess();
+
+private:
+   /**@name Methods to block default compiler methods.
+    *
+    * The compiler automatically generates the following three methods.
+    *  Since the default compiler implementation is generally not what
+    *  you want (for all but the most simple classes), we usually
+    *  put the declarations of these methods in the private section
+    *  and never implement them. This prevents the compiler from
+    *  implementing an incorrect "default" behavior without us
+    *  knowing. (See Scott Meyers book, "Effective C++")
+    */
+   ExContactBlockTL(
+      const ExContactBlockTL&
+   );
+
+   ExContactBlockTL& operator=(
+      const ExContactBlockTL&
+   );
+
+   Array<int> attr;
+   Array<int> m_attr;
+   Array<int> s_conn; // connectivity of the second/slave mesh
+   std::string mesh_file1;
+   std::string mesh_file2;
+   Mesh* mesh1;
+   Mesh* mesh2;
+   FiniteElementCollection* fec1;
+   FiniteElementCollection* fec2;
+   FiniteElementSpace* fespace1;
+   FiniteElementSpace* fespace2;
+   Array<int> ess_tdof_list1;
+   Array<int> ess_tdof_list2;
+   GridFunction nodes0;
+   GridFunction* nodes1;
+   GridFunction* nodes2;
+   mutable GridFunction* x1;
+   mutable GridFunction* x2;
+   LinearForm* b1;
+   LinearForm* b2;
+   PWConstCoefficient* lambda1_func;
+   PWConstCoefficient* lambda2_func;
+   PWConstCoefficient* mu1_func;
+   PWConstCoefficient* mu2_func;
+   BilinearForm* a1;
+   BilinearForm* a2;
+
+   mfem::Vector lambda1;
+   mfem::Vector lambda2;
+   mfem::Vector mu1;
+   mfem::Vector mu2;
+   mutable mfem::Vector xyz;
+
+   std::set<int> bdryVerts2;
+
+   int dim;
+   // degrees of freedom of both meshes
+   int ndof_1;
+   int ndof_2;
+   int ndofs;
+   // number of nodes for each mesh
+   int nnd_1;
+   int nnd_2;
+   int nnd;
+
+   int npoints;
+
+   SparseMatrix A1;
+   mfem::Vector B1, X1;
+   SparseMatrix A2;
+   mfem::Vector B2, X2;
+
+   SparseMatrix* K;
+   mutable mfem::Vector gapv;
+   mutable mfem::Vector m_xi;
+   mutable mfem::Vector xs;
+
+   mutable Array<int> m_conn; // only works for linear elements that have 4 vertices!
+   mutable DenseMatrix* coordsm;
+   mutable SparseMatrix* M;
+
+   mutable std::vector<SparseMatrix>* dM;
+
+   Array<int> Dirichlet_dof;
+   Array<double> Dirichlet_val;
+
+public: 
+   Mesh * GetMesh1() {return mesh1;}
+   Mesh * GetMesh2() {return mesh2;}
+   Array<int> GetDirichletDofs() {return Dirichlet_dof;}
+   Array<double> GetDirichletVals() {return Dirichlet_val;}
+
+};
+
+#endif

examples/ex34.cpp

@@ -2,17 +2,43 @@
 //
 // Compile with: make ex34
 //
-// Sample runs:  ex34
+// Sample runs:  ex34 -o 2
+//               ex34 -o 2 -pa -hex
 //
-// Description:  This example code demonstrates the use of MFEM to define a
-//               discontinuous Galerkin (DG) finite element discretization of
-//               the Laplace problem -Delta u = f with Dirichlet boundary
-//               conditions. Finite element spaces of any order, including zero
-//               on regular grids, are supported. The example highlights the
-//               use of coupling solution domains though custom physics defined
-//               on internal boundaries.
+// Device sample runs:
+//               ex34 -o 2 -pa -hex -d cuda
+//               ex34 -o 2 -no-pa -d cuda
 //
-//               We recommend viewing examples 1 and 14 before viewing this
+// Description:  This example code solves a simple magnetostatic problem
+//               curl curl A = J where the current density J is computed on a
+//               subset of the domain as J = -sigma grad phi. We discretize the
+//               vector potential with Nedelec finite elements, the scalar
+//               potential with Lagrange finite elements, and the current
+//               density with Raviart-Thomas finite elements.
+//
+//               The example demonstrates the use of a SubMesh to compute the
+//               scalar potential and its associated current density which is
+//               then transferred to the original mesh and used as a source
+//               function.
+//
+//               Note that this example takes certain liberties with the
+//               current density which is not necessarily divergence free
+//               as it should be. This was done to focus on the use of the
+//               SubMesh to transfer information between a full mesh and a
+//               sub-domain. A more rigorous implementation might employ an
+//               H(div) saddle point solver to obtain a divergence free J on
+//               the SubMesh. It would then also need to ensure that the r.h.s.
+//               of curl curl A = J does in fact lie in the range of the weak
+//               curl operator by performing a divergence cleaning procedure
+//               before the solve. After divergence cleaning the delta
+//               parameter would probably not be needed.
+//
+//               This example is designed to make use of a specific mesh which
+//               has a known configuration of elements and boundary attributes.
+//               Other meshes could be used but extra care would be required to
+//               properly define the SubMesh and the necessary boundaries.
+//
+//               We recommend viewing examples 1 and 3 before viewing this
 //               example.
 
 #include "mfem.hpp"
@@ -22,391 +48,575 @@
 using namespace std;
 using namespace mfem;
 
-class InteriorLFIntegrator : public LinearFormIntegrator
-{
-public:
-   InteriorLFIntegrator(Coefficient &Q)
-      : Q(Q)
-   {}
+static bool pa_ = false;
+static bool algebraic_ceed_ = false;
 
-   void AssembleRHSElementVect(const FiniteElement &el1,
-                               const FiniteElement &el2,
-                               FaceElementTransformations &trans,
-                               Vector &mesh_coords_bar) override;
-
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               ElementTransformation &trans,
-                               Vector &elvect) override
-   {
-      mfem_error("AssembleRHSElementVect(...)");
-   }
-
-private:
-   Coefficient &Q;
-
-#ifndef MFEM_THREAD_SAFE
-   Vector shape1;
-   Vector shape2;
-#endif
-};
-
-Mesh generate_mesh(int ref, int internal_bdr_attr = 5);
+void ComputeCurrentDensityOnSubMesh(int order,
+                                    const Array<int> &phi0_attr,
+                                    const Array<int> &phi1_attr,
+                                    const Array<int> &jn_zero_attr,
+                                    GridFunction &j_cond);
 
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options.
-   int ref_levels = 0;
+   const char *mesh_file = "../data/fichera-mixed.mesh";
+   Array<int> cond_attr;
+   Array<int> submesh_elems;
+   Array<int> sym_plane_attr;
+   Array<int> phi0_attr;
+   Array<int> phi1_attr;
+   Array<int> jn_zero_attr;
+   int ref_levels = 1;
    int order = 1;
-   int sol_order = 3;
-   double jump = -2;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   double eta = 0.0;
-   bool visualization = 1;
+   double delta_const = 1e-6;
+   bool mixed = true;
+   bool static_cond = false;
+   const char *device_config = "cpu";
+   bool visualization = true;
 
    OptionsParser args(argc, argv);
    args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly, -1 for auto.");
+                  "Number of times to refine the mesh uniformly.");
    args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) >= 0.");
-   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the three DG penalty parameters, typically +1/-1."
-                  " See the documentation of class DGDiffusionIntegrator.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the three DG penalty parameters, should be positive."
-                  " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
-   args.AddOption(&sol_order, "-so", "--solution_order",
-                  "Polynomial order of the exact solution >= 0.");
-   args.AddOption(&jump, "-j", "--jump",
-                  "Value of the discontinuity between the material regions.");
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_const, "-mc", "--magnetic-cond",
+                  "Magnetic Conductivity");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&mixed, "-mixed", "--mixed-mesh", "-hex",
+                  "--hex-mesh", "Mixed mesh of hexahedral mesh.");
+   args.AddOption(&pa_, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+#ifdef MFEM_USE_CEED
+   args.AddOption(&algebraic_ceed_, "-a", "--algebraic", "-no-a", "--no-algebraic",
+                  "Use algebraic Ceed solver");
+#endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+
    args.Parse();
    if (!args.Good())
    {
       args.PrintUsage(cout);
       return 1;
    }
-   if (kappa < 0)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   if (sol_order < 0)
-   {
-      sol_order = 1;
-   }
    args.PrintOptions(cout);
 
-   // 2. Construct the (serial) mesh and refine it if requested.
-   auto mesh = generate_mesh(ref_levels);
+   if (!mixed || pa_)
+   {
+      mesh_file = "../data/fichera.mesh";
+   }
 
+   if (submesh_elems.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0)
+      {
+         submesh_elems.SetSize(5);
+         submesh_elems[0] = 0;
+         submesh_elems[1] = 2;
+         submesh_elems[2] = 3;
+         submesh_elems[3] = 4;
+         submesh_elems[4] = 9;
+      }
+      else if (strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         submesh_elems.SetSize(7);
+         submesh_elems[0] = 10;
+         submesh_elems[1] = 14;
+         submesh_elems[2] = 34;
+         submesh_elems[3] = 36;
+         submesh_elems[4] = 37;
+         submesh_elems[5] = 38;
+         submesh_elems[6] = 39;
+      }
+   }
+   if (sym_plane_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         sym_plane_attr.SetSize(8);
+         sym_plane_attr[0] =  9;
+         sym_plane_attr[1] = 10;
+         sym_plane_attr[2] = 11;
+         sym_plane_attr[3] = 12;
+         sym_plane_attr[4] = 13;
+         sym_plane_attr[5] = 14;
+         sym_plane_attr[6] = 15;
+         sym_plane_attr[7] = 16;
+      }
+   }
+   if (phi0_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         phi0_attr.Append(2);
+      }
+   }
+   if (phi1_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         phi1_attr.Append(23);
+      }
+   }
+   if (jn_zero_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         jn_zero_attr.Append(25);
+      }
+      for (int i=0; i<sym_plane_attr.Size(); i++)
+      {
+         jn_zero_attr.Append(sym_plane_attr[i]);
+      }
+   }
+
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
    int dim = mesh.Dimension();
 
-   if (mesh.NURBSext)
+   if (!mixed || pa_)
    {
-      mesh.SetCurvature(max(order, 1));
-   }
+      mesh.UniformRefinement();
 
-   // 3. Define a finite element space on the mesh. Here we use discontinuous
-   //    finite elements of the specified order >= 0.
-   DG_FECollection fec(order, dim);
-   FiniteElementSpace fespace(&mesh, &fec);
-   cout << "Number of unknowns: " << fespace.GetVSize() << endl;
-
-   // 4. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system.
-   LinearForm b(&fespace);
-
-   Array<int> p1_attr_marker(mesh.attributes.Max());
-   p1_attr_marker = 0;
-   p1_attr_marker[0] = 1;
-
-   FunctionCoefficient p1_source([sol_order](const Vector &p)
-   {
-      const double x = p(0);
-      const double val = -(sol_order - 1)*sol_order*pow(x, sol_order-2);
-      return val;
-   });
-   b.AddDomainIntegrator(new DomainLFIntegrator(p1_source), p1_attr_marker);
-
-   Array<int> p2_attr_marker(mesh.attributes.Max());
-   p2_attr_marker = 0;
-   p2_attr_marker[1] = 1;
-
-   FunctionCoefficient p2_source([sol_order](const Vector &p)
-   {
-      const double x = p(0);
-      double val = -(sol_order - 1)*sol_order*pow(x - 2, sol_order-2);
-      if (sol_order % 2 == 0)
+      if (ref_levels > 0)
       {
-         val *= -1.0;
+         ref_levels--;
       }
-      return val;
-   });
-   b.AddDomainIntegrator(new DomainLFIntegrator(p2_source), p2_attr_marker);
-
-   ConstantCoefficient one(1.0);
-
-   Array<int> p1_bdr_attr_marker(mesh.bdr_attributes.Max());
-   p1_bdr_attr_marker = 0;
-   p1_bdr_attr_marker[0] = 1;
-
-   ConstantCoefficient left_bc_val(0.0);
-   b.AddBdrFaceIntegrator(
-      new DGDirichletLFIntegrator(left_bc_val, one, sigma, kappa),
-      p1_bdr_attr_marker);
-
-   Array<int> p2_bdr_attr_marker(mesh.bdr_attributes.Max());
-   p2_bdr_attr_marker = 0;
-   p2_bdr_attr_marker[1] = 1;
-
-   ConstantCoefficient right_bc_val(2.0 + jump);
-   b.AddBdrFaceIntegrator(
-      new DGDirichletLFIntegrator(right_bc_val, one, sigma, kappa),
-      p2_bdr_attr_marker);
-
-   Array<int> internal_bdr_attr_marker(mesh.bdr_attributes.Max());
-   internal_bdr_attr_marker = 0;
-   internal_bdr_attr_marker[4] = 1;
-
-   ConstantCoefficient interface_flux(sol_order);
-   b.AddInternalBoundaryFaceIntegrator(
-      new InteriorLFIntegrator(interface_flux),
-      internal_bdr_attr_marker);
-
-   b.Assemble();
-
-   // 5. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 6. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator and the interior and boundary DG face integrators.
-   //    Note that boundary conditions are imposed weakly in the form, so there
-   //    is no need for dof elimination. After assembly and finalizing we
-   //    extract the corresponding sparse matrix A.
-   BilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa),
-                          p1_bdr_attr_marker);
-   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa),
-                          p2_bdr_attr_marker);
-   if (eta > 0)
-   {
-      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
    }
 
-   // 7. Negate the DG interface terms along the internal boundary so that the
-   //    only coupling between domains is from the chosen model (constant flux
-   //    in this case).
-   ProductCoefficient neg_one(-1.0, one);
-   a.AddInternalBoundaryFaceIntegrator(new DGDiffusionIntegrator(neg_one, sigma,
-                                                                 kappa),
-                                       internal_bdr_attr_marker);
-   if (eta > 0)
+   int submesh_attr = -1;
+   if (cond_attr.Size() == 0 && submesh_elems.Size() > 0)
    {
-      a.AddInternalBoundaryFaceIntegrator(new DGDiffusionBR2Integrator(fespace,
-                                                                       neg_one, eta),
-                                          internal_bdr_attr_marker);
+      int max_attr = mesh.attributes.Max();
+      submesh_attr = max_attr + 1;
+
+      for (int i=0; i<submesh_elems.Size(); i++)
+      {
+         mesh.SetAttribute(submesh_elems[i], submesh_attr);
+      }
+      mesh.SetAttributes();
+
+      if (cond_attr.Size() == 0)
+      {
+         cond_attr.Append(submesh_attr);
+      }
    }
 
-   a.Assemble();
-   a.Finalize();
-   const SparseMatrix &A = a.SpMat();
-
-#ifndef MFEM_USE_SUITESPARSE
-   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
-   //    non-symmetric one.
-   GSSmoother M(A);
-   if (sigma == -1.0)
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement.
    {
-      PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
    }
-   else
+
+   // 5b. Extract a submesh covering a portion of the domain
+   SubMesh mesh_cond(SubMesh::CreateFromDomain(mesh, cond_attr));
+
+   // 6. Define a suitable finite element space on the SubMesh and compute
+   //    the current density as an H(div) field.
+   RT_FECollection fec_cond_rt(order - 1, dim);
+   FiniteElementSpace fes_cond_rt(&mesh_cond, &fec_cond_rt);
+   GridFunction j_cond(&fes_cond_rt);
+
+   ComputeCurrentDensityOnSubMesh(order, phi0_attr, phi1_attr, jn_zero_attr,
+                                  j_cond);
+
+   // 6a. Save the SubMesh and associated current density in parallel. This
+   //     output can be viewed later using GLVis:
+   //        "glvis -np <np> -m cond_mesh -g cond_j"
    {
-      GMRES(A, M, b, x, 1, 500, 500, 1e-24, 0.0);
+      ostringstream mesh_name, cond_name;
+      mesh_name << "cond.mesh";
+      cond_name << "cond_j.gf";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh_cond.Print(mesh_ofs);
+
+      ofstream cond_ofs(cond_name.str().c_str());
+      cond_ofs.precision(8);
+      j_cond.Save(cond_ofs);
+   }
+   // 6b. Send the current density, computed on the SubMesh, to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock.precision(8);
+      port_sock << "solution\n" << mesh_cond << j_cond
+                << "window_title 'Conductor J'"
+                << "window_geometry 400 0 400 350" << flush;
    }
-#else
-   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(b, x);
-#endif
 
-   // 9. Save the refined mesh and the solution. This output can be viewed later
-   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
+   // 7. Define a parallel finite element space on the full mesh. Here we
+   //    use the H(curl) finite elements for the vector potential and H(div)
+   //    for the current density.
+   ND_FECollection fec_nd(order, dim);
+   RT_FECollection fec_rt(order - 1, dim);
+   FiniteElementSpace fespace_nd(&mesh, &fec_nd);
+   FiniteElementSpace fespace_rt(&mesh, &fec_rt);
 
-   // 10. Send the solution by socket to a GLVis server.
+   GridFunction j_full(&fespace_rt);
+   j_full = 0.0;
+   mesh_cond.Transfer(j_cond, j_full);
+
+   // 7a. Send the transferred current density to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock(vishost, visport);
       sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << mesh << j_full
+               << "window_title 'J Full'"
+               << "window_geometry 400 430 400 350" << flush;
    }
 
+   // 8. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes except for those on a symmetry
+   //    plane as essential (Dirichlet) and converting them to a list of
+   //    true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      for (int i=0; i<sym_plane_attr.Size(); i++)
+      {
+         ess_bdr[sym_plane_attr[i]-1] = 0;
+      }
+      fespace_nd.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   //  9. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system, which in this case is
+   //     (J,W_i) where J is given by the function H(div) field transferred
+   //     from the SubMesh and W_i are the basis functions in the finite
+   //     element fespace.
+   VectorGridFunctionCoefficient jCoef(&j_full);
+   LinearForm b(&fespace_nd);
+   b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(jCoef));
+   b.Assemble();
+
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x to zero.
+   GridFunction x(&fespace_nd);
+   x = 0.0;
+
+   // 11. Set up the parallel bilinear form corresponding to the EM
+   //     diffusion operator curl muinv curl + delta I, by adding the
+   //     curl-curl and the mass domain integrators. For standard
+   //     magnetostatics equations choose delta << 1. Larger values of
+   //     delta should make the linear system easier to solve at the
+   //     expense of resembling a diffusive quasistatic magnetic field.
+   //     A reasonable balance must be found whenever the mesh or problem
+   //     setup is altered.
+   ConstantCoefficient muinv(1.0);
+   ConstantCoefficient delta(delta_const);
+   BilinearForm a(&fespace_nd);
+   if (pa_) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.AddDomainIntegrator(new CurlCurlIntegrator(muinv));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(delta));
+
+   // 12. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // 13. Solve the system AX=B
+   if (pa_) // Jacobi preconditioning in partial assembly mode
+   {
+      cout << "\nSolving for magnetic vector potential "
+           << "using CG with a Jacobi preconditioner" << endl;
+
+      OperatorJacobiSmoother M(a, ess_tdof_list);
+      PCG(*A, M, B, X, 1, 1000, 1e-12, 0.0);
+   }
+   else
+   {
+#ifndef MFEM_USE_SUITESPARSE
+      cout << "\nSolving for magnetic vector potential "
+           << "using CG with a Gauss-Seidel preconditioner" << endl;
+
+      // 13a. Define a simple symmetric Gauss-Seidel preconditioner and use
+      //     it to solve the system Ax=b with PCG.
+      GSSmoother M((SparseMatrix&)(*A));
+      PCG(*A, M, B, X, 1, 500, 1e-12, 0.0);
+#else
+      cout << "\nSolving for magnetic vector potential "
+           << "using UMFPack" << endl;
+
+      // 13a. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
+      //     system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(*A);
+      umf_solver.Mult(B, X);
+#endif
+   }
+
+   // 14. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "refined.mesh";
+      sol_name << "sol.gf";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << x
+               << "window_title 'Vector Potential'"
+               << "window_geometry 800 0 400 350" << flush;
+   }
+
+   // 17. Compute the magnetic flux as the curl of the solution
+   DiscreteLinearOperator curl(&fespace_nd, &fespace_rt);
+   curl.AddDomainInterpolator(new CurlInterpolator);
+   curl.Assemble();
+   curl.Finalize();
+
+   GridFunction dx(&fespace_rt);
+   curl.Mult(x, dx);
+
+   // 18. Save the curl of the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g dsol".
+   {
+      ostringstream dsol_name;
+      dsol_name << "dsol.gf";
+
+      ofstream dsol_ofs(dsol_name.str().c_str());
+      dsol_ofs.precision(8);
+      dx.Save(dsol_ofs);
+   }
+
+   // 19. Send the curl of the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << dx
+               << "window_title 'Magnetic Flux'"
+               << "window_geometry 1200 0 400 350" << flush;
+   }
+
+   // 20. Clean exit
    return 0;
 }
 
-void InteriorLFIntegrator::AssembleRHSElementVect(
-   const FiniteElement &el1,
-   const FiniteElement &el2,
-   FaceElementTransformations &trans,
-   Vector &elvect)
+void ComputeCurrentDensityOnSubMesh(int order,
+                                    const Array<int> &phi0_attr,
+                                    const Array<int> &phi1_attr,
+                                    const Array<int> &jn_zero_attr,
+                                    GridFunction &j_cond)
 {
-   int ndof1 = el1.GetDof();
-   int ndof2 = el2.GetDof();
-   int ndof = ndof1 + ndof2;
+   // Exract the finite element space and mesh on which j_cond is defined
+   FiniteElementSpace &fes_cond_rt = *j_cond.FESpace();
+   Mesh &mesh_cond = *fes_cond_rt.GetMesh();
+   int dim  = mesh_cond.Dimension();
 
-#ifdef MFEM_THREAD_SAFE
-   Vector shape1;
-   Vector shape2;
+   // Define a parallel finite element space on the SubMesh. Here we use the
+   // H1 finite elements for the electrostatic potential.
+   H1_FECollection fec_h1(order, dim);
+   FiniteElementSpace fes_cond_h1(&mesh_cond, &fec_h1);
+
+   // Define the conductivity coefficient and the boundaries associated with
+   // the fixed potentials phi0 and phi1 which will drive the current.
+   ConstantCoefficient sigmaCoef(1.0);
+   Array<int> ess_bdr_phi(mesh_cond.bdr_attributes.Max());
+   Array<int> ess_bdr_j(mesh_cond.bdr_attributes.Max());
+   Array<int> ess_bdr_tdof_phi;
+   ess_bdr_phi = 0;
+   ess_bdr_j   = 0;
+   for (int i=0; i<phi0_attr.Size(); i++)
+   {
+      ess_bdr_phi[phi0_attr[i]-1] = 1;
+   }
+   for (int i=0; i<phi1_attr.Size(); i++)
+   {
+      ess_bdr_phi[phi1_attr[i]-1] = 1;
+   }
+   for (int i=0; i<jn_zero_attr.Size(); i++)
+   {
+      ess_bdr_j[jn_zero_attr[i]-1] = 1;
+   }
+   fes_cond_h1.GetEssentialTrueDofs(ess_bdr_phi, ess_bdr_tdof_phi);
+
+   // Setup the bilinear form corresponding to -Div(sigma Grad phi)
+   BilinearForm a_h1(&fes_cond_h1);
+   a_h1.AddDomainIntegrator(new DiffusionIntegrator(sigmaCoef));
+   a_h1.Assemble();
+
+   // Set the r.h.s. to zero
+   LinearForm b_h1(&fes_cond_h1);
+   b_h1 = 0.0;
+
+   // Setup the boundary conditions on phi
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   GridFunction phi_h1(&fes_cond_h1);
+   phi_h1 = 0.0;
+
+   Array<int> bdr0(mesh_cond.bdr_attributes.Max()); bdr0 = 0;
+   for (int i=0; i<phi0_attr.Size(); i++)
+   {
+      bdr0[phi0_attr[i]-1] = 1;
+   }
+   phi_h1.ProjectBdrCoefficient(zero, bdr0);
+
+   Array<int> bdr1(mesh_cond.bdr_attributes.Max()); bdr1 = 0;
+   for (int i=0; i<phi1_attr.Size(); i++)
+   {
+      bdr1[phi1_attr[i]-1] = 1;
+   }
+   phi_h1.ProjectBdrCoefficient(one, bdr1);
+
+   {
+      OperatorPtr A;
+      Vector B, X;
+      a_h1.FormLinearSystem(ess_bdr_tdof_phi, phi_h1, b_h1, A, X, B);
+
+      // Solve the linear system
+      if (!pa_)
+      {
+#ifndef MFEM_USE_SUITESPARSE
+         cout << "\nSolving for electric potential using PCG "
+              << "with a Gauss-Seidel preconditioner" << endl;
+
+         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+         GSSmoother M((SparseMatrix&)(*A));
+         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+         cout << "\nSolving for electric potential using UMFPack" << endl;
+
+         // If MFEM was compiled with SuiteSparse,
+         // use UMFPACK to solve the system.
+         UMFPackSolver umf_solver;
+         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+         umf_solver.SetOperator(*A);
+         umf_solver.Mult(B, X);
 #endif
-   shape1.SetSize(ndof1);
-   shape2.SetSize(ndof2);
-
-   const auto *ir = IntRule;
-   if (ir == NULL)
-   {
-      int order = 2 * max(el1.GetOrder(), el2.GetOrder());
-      ir = &IntRules.Get(trans.GetGeometryType(), order);
-   }
-
-   elvect.SetSize(ndof);
-   Vector elvect1(elvect.GetData(), ndof1);
-   Vector elvect2(elvect.GetData() + ndof1, ndof2);
-   elvect = 0.0;
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const auto &ip = ir->IntPoint(i);
-
-      // Set the integration point in the face and the neighboring element
-      trans.SetAllIntPoints(&ip);
-
-      const double w = ip.weight * trans.Weight();
-
-      // Access the neighboring element's integration point
-      const auto &eip1 = trans.GetElement1IntPoint();
-      const auto &eip2 = trans.GetElement2IntPoint();
-
-      double Q_val = Q.Eval(trans, ip);
-
-      el1.CalcShape(eip1, shape1);
-      el2.CalcShape(eip2, shape2);
-
-      elvect1.Add(Q_val * w, shape1);
-      elvect2.Add(-Q_val * w, shape2);
-   }
-}
-
-Mesh generate_mesh(int ref, int internal_bdr_attr)
-{
-   int nxy = 4 * (ref+1);
-   auto mesh = Mesh::MakeCartesian2D(nxy, nxy, Element::TRIANGLE, true, 2.0, 1.0);
-   // auto mesh = Mesh::MakeCartesian2D(nxy, nxy, Element::QUADRILATERAL, true, 2.0, 1.0);
-
-   // assign element attributes to left and right sides
-   for (int i = 0; i < mesh.GetNE(); ++i)
-   {
-      auto *elem = mesh.GetElement(i);
-
-      Array<int> verts;
-      elem->GetVertices(verts);
-
-      bool left = true;
-      for (int j = 0; j < verts.Size(); ++j)
-      {
-         auto *vtx = mesh.GetVertex(verts[j]);
-         if (vtx[0] <= 1.0)
-         {
-            continue;
-         }
-         else
-         {
-            left = false;
-         }
-      }
-      if (left)
-      {
-         elem->SetAttribute(1);
       }
       else
       {
-         elem->SetAttribute(2);
+         cout << "\nSolving for electric potential using CG" << endl;
+
+         if (UsesTensorBasis(fes_cond_h1))
+         {
+            if (algebraic_ceed_)
+            {
+               ceed::AlgebraicSolver M(a_h1, ess_bdr_tdof_phi);
+               PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+            }
+            else
+            {
+               OperatorJacobiSmoother M(a_h1, ess_bdr_tdof_phi);
+               PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+            }
+         }
+         else
+         {
+            CG(*A, B, X, 1, 400, 1e-12, 0.0);
+         }
       }
+      a_h1.RecoverFEMSolution(X, b_h1, phi_h1);
    }
 
-   // assign boundary element attributes to left and right sides
-   for (int i = 0; i < mesh.GetNBE(); ++i)
    {
-      auto *elem = mesh.GetBdrElement(i);
-
-      Array<int> verts;
-      elem->GetVertices(verts);
-
-      bool left = true;
-      bool right = true;
-      bool top = true;
-      bool bottom = true;
-      for (int j = 0; j < verts.Size(); ++j)
-      {
-         auto *vtx = mesh.GetVertex(verts[j]);
-         left = left && abs(vtx[0] - 0.0) < 1e-12;
-         right = right && abs(vtx[0] - 2.0) < 1e-12;
-         top = top && abs(vtx[1] - 1.0) < 1e-12;
-         bottom = bottom && abs(vtx[1] - 0.0) < 1e-12;
-      }
-      if (left)
-      {
-         elem->SetAttribute(1);
-      }
-      else if (right)
-      {
-         elem->SetAttribute(2);
-      }
-      else if (top)
-      {
-         elem->SetAttribute(3);
-      }
-      else if (bottom)
-      {
-         elem->SetAttribute(4);
-      }
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock.precision(8);
+      port_sock << "solution\n" << mesh_cond << phi_h1
+                << "window_title 'Conductor Potential'"
+                << "window_geometry 0 0 400 350" << flush;
    }
 
-   // add internal boundary elements
-   for (int i = 0; i < mesh.GetNumFaces(); ++i)
-   {
-      int e1, e2;
-      mesh.GetFaceElements(i, &e1, &e2);
-      if (e1 >= 0 && e2 >= 0 && mesh.GetAttribute(e1) != mesh.GetAttribute(e2))
-      {
-         // This is the internal face between attributes.
-         auto *new_elem = mesh.GetFace(i)->Duplicate(&mesh);
-         new_elem->SetAttribute(internal_bdr_attr);
-         mesh.AddBdrElement(new_elem);
-      }
-   }
+   // Solve for the current density J = -sigma Grad phi with boundary
+   // conditions J.n = 0 on the walls of the conductor but not on the
+   // ports where phi=0 and phi=1.
 
-   mesh.FinalizeTopology(); // Finalize to build relevant tables
-   mesh.Finalize();
-   mesh.SetAttributes();
+   // J will be computed in H(div) so we need an RT mass matrix
+   BilinearForm m_rt(&fes_cond_rt);
+   m_rt.AddDomainIntegrator(new VectorFEMassIntegrator);
+   m_rt.Assemble();
 
-   return mesh;
-}
+   // Assemble the (sigma Grad phi) operator
+   MixedBilinearForm d_h1(&fes_cond_h1, &fes_cond_rt);
+   d_h1.AddDomainIntegrator(new MixedVectorGradientIntegrator(sigmaCoef));
+   d_h1.Assemble();
+
+   // Compute the r.h.s, b_rt = sigma E = -sigma Grad phi
+   LinearForm b_rt(&fes_cond_rt);
+   d_h1.Mult(phi_h1, b_rt);
+   b_rt *= -1.0;
+
+   // Apply the necessary boundary conditions and solve for J in H(div)
+   cout << "\nSolving for current density in H(Div) "
+        << "using diagonally scaled CG" << endl;
+   cout << "Size of linear system: "
+        << fes_cond_rt.GetTrueVSize() << endl;
+
+   Array<int> ess_bdr_tdof_rt;
+   OperatorPtr M;
+   Vector B, X;
+
+   fes_cond_rt.GetEssentialTrueDofs(ess_bdr_j, ess_bdr_tdof_rt);
+
+   j_cond = 0.0;
+   m_rt.FormLinearSystem(ess_bdr_tdof_rt, j_cond, b_rt, M, X, B);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*M);
+   cg.Mult(B, X);
+   m_rt.RecoverFEMSolution(X, b_rt, j_cond);
+}

examples/ex34p.cpp

@@ -0,0 +1,649 @@
+//                       MFEM Example 34 - Parallel Version
+//
+// Compile with: make ex34p
+//
+// Sample runs:  mpirun -np 4 ex34p -o 2
+//               mpirun -np 4 ex34p -o 2 -hex -pa
+//
+// Device sample runs:
+//               mpirun -np 4 ex34p -o 2 -hex -pa -d cuda
+//               mpirun -np 4 ex34p -o 2 -no-pa -d cuda
+//
+// Description:  This example code solves a simple magnetostatic problem
+//               curl curl A = J where the current density J is computed on a
+//               subset of the domain as J = -sigma grad phi. We discretize the
+//               vector potential with Nedelec finite elements, the scalar
+//               potential with Lagrange finite elements, and the current
+//               density with Raviart-Thomas finite elements.
+//
+//               The example demonstrates the use of a SubMesh to compute the
+//               scalar potential and its associated current density which is
+//               then transferred to the original mesh and used as a source
+//               function.
+//
+//               Note that this example takes certain liberties with the
+//               current density which is not necessarily divergence free
+//               as it should be. This was done to focus on the use of the
+//               SubMesh to transfer information between a full mesh and a
+//               sub-domain. A more rigorous implementation might employ an
+//               H(div) saddle point solver to obtain a divergence free J on
+//               the SubMesh. It would then also need to ensure that the r.h.s.
+//               of curl curl A = J does in fact lie in the range of the weak
+//               curl operator by performing a divergence cleaning procedure
+//               before the solve. After divergence cleaning the delta
+//               parameter would probably not be needed.
+//
+//               This example is designed to make use of a specific mesh which
+//               has a known configuration of elements and boundary attributes.
+//               Other meshes could be used but extra care would be required to
+//               properly define the SubMesh and the necessary boundaries.
+//
+//               We recommend viewing examples 1 and 3 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void ComputeCurrentDensityOnSubMesh(int order,
+                                    const Array<int> &phi0_attr,
+                                    const Array<int> &phi1_attr,
+                                    const Array<int> &jn_zero_attr,
+                                    ParGridFunction &j_cond);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/fichera-mixed.mesh";
+   Array<int> cond_attr;
+   Array<int> submesh_elems;
+   Array<int> sym_plane_attr;
+   Array<int> phi0_attr;
+   Array<int> phi1_attr;
+   Array<int> jn_zero_attr;
+   int ser_ref_levels = 1;
+   int par_ref_levels = 1;
+   int order = 1;
+   double delta_const = 1e-6;
+   bool mixed = true;
+   bool static_cond = false;
+   bool pa = false;
+   const char *device_config = "cpu";
+   bool visualization = true;
+#ifdef MFEM_USE_AMGX
+   bool useAmgX = false;
+#endif
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_const, "-mc", "--magnetic-cond",
+                  "Magnetic Conductivity");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&mixed, "-mixed", "--mixed-mesh", "-hex",
+                  "--hex-mesh", "Mixed mesh of hexahedral mesh.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+#ifdef MFEM_USE_AMGX
+   args.AddOption(&useAmgX, "-amgx", "--useAmgX", "-no-amgx",
+                  "--no-useAmgX",
+                  "Enable or disable AmgX in MatrixFreeAMS.");
+#endif
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (!mixed || pa)
+   {
+      mesh_file = "../data/fichera.mesh";
+   }
+
+   if (submesh_elems.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0)
+      {
+         submesh_elems.SetSize(5);
+         submesh_elems[0] = 0;
+         submesh_elems[1] = 2;
+         submesh_elems[2] = 3;
+         submesh_elems[3] = 4;
+         submesh_elems[4] = 9;
+      }
+      else if (strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         submesh_elems.SetSize(7);
+         submesh_elems[0] = 10;
+         submesh_elems[1] = 14;
+         submesh_elems[2] = 34;
+         submesh_elems[3] = 36;
+         submesh_elems[4] = 37;
+         submesh_elems[5] = 38;
+         submesh_elems[6] = 39;
+      }
+   }
+   if (sym_plane_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         sym_plane_attr.SetSize(8);
+         sym_plane_attr[0] =  9;
+         sym_plane_attr[1] = 10;
+         sym_plane_attr[2] = 11;
+         sym_plane_attr[3] = 12;
+         sym_plane_attr[4] = 13;
+         sym_plane_attr[5] = 14;
+         sym_plane_attr[6] = 15;
+         sym_plane_attr[7] = 16;
+      }
+   }
+   if (phi0_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         phi0_attr.Append(2);
+      }
+   }
+   if (phi1_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         phi1_attr.Append(23);
+      }
+   }
+   if (jn_zero_attr.Size() == 0)
+   {
+      if (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+          strcmp(mesh_file, "../data/fichera.mesh") == 0)
+      {
+         jn_zero_attr.Append(25);
+      }
+      for (int i=0; i<sym_plane_attr.Size(); i++)
+      {
+         jn_zero_attr.Append(sym_plane_attr[i]);
+      }
+   }
+
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   if (!mixed || pa)
+   {
+      mesh->UniformRefinement();
+
+      if (ser_ref_levels > 0)
+      {
+         ser_ref_levels--;
+      }
+      else
+      {
+         par_ref_levels--;
+      }
+   }
+
+   int submesh_attr = -1;
+   if (cond_attr.Size() == 0 && submesh_elems.Size() > 0)
+   {
+      int max_attr = mesh->attributes.Max();
+      submesh_attr = max_attr + 1;
+
+      for (int i=0; i<submesh_elems.Size(); i++)
+      {
+         mesh->SetAttribute(submesh_elems[i], submesh_attr);
+      }
+      mesh->SetAttributes();
+
+      if (cond_attr.Size() == 0)
+      {
+         cond_attr.Append(submesh_attr);
+      }
+   }
+
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement.
+   {
+      int ref_levels = ser_ref_levels;
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh.UniformRefinement();
+      }
+   }
+
+   // 6b. Extract a submesh covering a portion of the domain
+   ParSubMesh pmesh_cond(ParSubMesh::CreateFromDomain(pmesh, cond_attr));
+
+   // 7. Define a suitable finite element space on the SubMesh and compute
+   //    the current density as an H(div) field.
+   RT_FECollection fec_cond_rt(order - 1, dim);
+   ParFiniteElementSpace fes_cond_rt(&pmesh_cond, &fec_cond_rt);
+   ParGridFunction j_cond(&fes_cond_rt);
+
+   ComputeCurrentDensityOnSubMesh(order, phi0_attr, phi1_attr, jn_zero_attr,
+                                  j_cond);
+
+   // 7a. Save the SubMesh and associated current density in parallel. This
+   //     output can be viewed later using GLVis:
+   //        "glvis -np <np> -m cond_mesh -g cond_j"
+   {
+      ostringstream mesh_name, cond_name;
+      mesh_name << "cond_mesh." << setfill('0') << setw(6) << myid;
+      cond_name << "cond_j." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh_cond.Print(mesh_ofs);
+
+      ofstream cond_ofs(cond_name.str().c_str());
+      cond_ofs.precision(8);
+      j_cond.Save(cond_ofs);
+   }
+   // 7b. Send the current density, computed on the SubMesh, to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock << "parallel " << num_procs << " " << myid << "\n";
+      port_sock.precision(8);
+      port_sock << "solution\n" << pmesh_cond << j_cond
+                << "window_title 'Conductor J'"
+                << "window_geometry 400 0 400 350" << flush;
+   }
+
+   // 8. Define a parallel finite element space on the full mesh. Here we
+   //    use the H(curl) finite elements for the vector potential and H(div)
+   //    for the current density.
+   ND_FECollection fec_nd(order, dim);
+   RT_FECollection fec_rt(order - 1, dim);
+   ParFiniteElementSpace fespace_nd(&pmesh, &fec_nd);
+   ParFiniteElementSpace fespace_rt(&pmesh, &fec_rt);
+
+   ParGridFunction j_full(&fespace_rt);
+   j_full = 0.0;
+   pmesh_cond.Transfer(j_cond, j_full);
+
+   // 8a. Send the transferred current density to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << j_full
+               << "window_title 'J Full'"
+               << "window_geometry 400 430 400 350" << flush;
+   }
+
+   // 9. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes except for those on a symmetry
+   //    plane as essential (Dirichlet) and converting them to a list of
+   //    true dofs.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      for (int i=0; i<sym_plane_attr.Size(); i++)
+      {
+         ess_bdr[sym_plane_attr[i]-1] = 0;
+      }
+      fespace_nd.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system, which in this case is
+   //     (J,W_i) where J is given by the function H(div) field transferred
+   //     from the SubMesh and W_i are the basis functions in the finite
+   //     element fespace.
+   VectorGridFunctionCoefficient jCoef(&j_full);
+   ParLinearForm b(&fespace_nd);
+   b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(jCoef));
+   b.Assemble();
+
+   // 11. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x to zero.
+   ParGridFunction x(&fespace_nd);
+   x = 0.0;
+
+   // 12. Set up the parallel bilinear form corresponding to the EM
+   //     diffusion operator curl muinv curl + delta I, by adding the
+   //     curl-curl and the mass domain integrators. For standard
+   //     magnetostatics equations choose delta << 1. Larger values of
+   //     delta should make the linear system easier to solve at the
+   //     expense of resembling a diffusive quasistatic magnetic field.
+   //     A reasonable balance must be found whenever the mesh or problem
+   //     setup is altered.
+   ConstantCoefficient muinv(1.0);
+   ConstantCoefficient delta(delta_const);
+   ParBilinearForm a(&fespace_nd);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.AddDomainIntegrator(new CurlCurlIntegrator(muinv));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(delta));
+
+   // 13. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "\nSolving for magnetic vector potential "
+           << "using CG with AMS" << endl;
+   }
+
+   // 14. Solve the system AX=B using PCG with an AMS preconditioner.
+   if (pa)
+   {
+#ifdef MFEM_USE_AMGX
+      MatrixFreeAMS ams(a, *A, fespace_nd, &muinv, &delta, NULL, ess_bdr,
+                        useAmgX);
+#else
+      MatrixFreeAMS ams(a, *A, fespace_nd, &muinv, &delta, NULL, ess_bdr);
+#endif
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(1000);
+      cg.SetPrintLevel(1);
+      cg.SetOperator(*A);
+      cg.SetPreconditioner(ams);
+      cg.Mult(B, X);
+   }
+   else
+   {
+      if (myid == 0)
+      {
+         cout << "Size of linear system: "
+              << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
+      }
+
+      ParFiniteElementSpace *prec_fespace =
+         (a.StaticCondensationIsEnabled() ? a.SCParFESpace() : &fespace_nd);
+      HypreAMS ams(*A.As<HypreParMatrix>(), prec_fespace);
+      HyprePCG pcg(*A.As<HypreParMatrix>());
+      pcg.SetTol(1e-12);
+      pcg.SetMaxIter(500);
+      pcg.SetPrintLevel(2);
+      pcg.SetPreconditioner(ams);
+      pcg.Mult(B, X);
+   }
+
+   // 15. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 16. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 17. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x
+               << "window_title 'Vector Potential'"
+               << "window_geometry 800 0 400 350" << flush;
+   }
+
+   // 18. Compute the magnetic flux as the curl of the solution
+   ParDiscreteLinearOperator curl(&fespace_nd, &fespace_rt);
+   curl.AddDomainInterpolator(new CurlInterpolator);
+   curl.Assemble();
+   curl.Finalize();
+
+   ParGridFunction dx(&fespace_rt);
+   curl.Mult(x, dx);
+
+   // 19. Save the curl of the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g dsol".
+   {
+      ostringstream dsol_name;
+      dsol_name << "dsol." << setfill('0') << setw(6) << myid;
+
+      ofstream dsol_ofs(dsol_name.str().c_str());
+      dsol_ofs.precision(8);
+      dx.Save(dsol_ofs);
+   }
+
+   // 20. Send the curl of the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << dx
+               << "window_title 'Magnetic Flux'"
+               << "window_geometry 1200 0 400 350" << flush;
+   }
+
+   // 21. Clean exit
+   return 0;
+}
+
+void ComputeCurrentDensityOnSubMesh(int order,
+                                    const Array<int> &phi0_attr,
+                                    const Array<int> &phi1_attr,
+                                    const Array<int> &jn_zero_attr,
+                                    ParGridFunction &j_cond)
+{
+   // Exract the finite element space and mesh on which j_cond is defined
+   ParFiniteElementSpace &fes_cond_rt = *j_cond.ParFESpace();
+   ParMesh &pmesh_cond = *fes_cond_rt.GetParMesh();
+   int myid = fes_cond_rt.GetMyRank();
+   int dim  = pmesh_cond.Dimension();
+
+   // Define a parallel finite element space on the SubMesh. Here we use the
+   // H1 finite elements for the electrostatic potential.
+   H1_FECollection fec_h1(order, dim);
+   ParFiniteElementSpace fes_cond_h1(&pmesh_cond, &fec_h1);
+
+   // Define the conductivity coefficient and the boundaries associated with
+   // the fixed potentials phi0 and phi1 which will drive the current.
+   ConstantCoefficient sigmaCoef(1.0);
+   Array<int> ess_bdr_phi(pmesh_cond.bdr_attributes.Max());
+   Array<int> ess_bdr_j(pmesh_cond.bdr_attributes.Max());
+   Array<int> ess_bdr_tdof_phi;
+   ess_bdr_phi = 0;
+   ess_bdr_j   = 0;
+   for (int i=0; i<phi0_attr.Size(); i++)
+   {
+      ess_bdr_phi[phi0_attr[i]-1] = 1;
+   }
+   for (int i=0; i<phi1_attr.Size(); i++)
+   {
+      ess_bdr_phi[phi1_attr[i]-1] = 1;
+   }
+   for (int i=0; i<jn_zero_attr.Size(); i++)
+   {
+      ess_bdr_j[jn_zero_attr[i]-1] = 1;
+   }
+   fes_cond_h1.GetEssentialTrueDofs(ess_bdr_phi, ess_bdr_tdof_phi);
+
+   // Setup the bilinear form corresponding to -Div(sigma Grad phi)
+   ParBilinearForm a_h1(&fes_cond_h1);
+   a_h1.AddDomainIntegrator(new DiffusionIntegrator(sigmaCoef));
+   a_h1.Assemble();
+
+   // Set the r.h.s. to zero
+   ParLinearForm b_h1(&fes_cond_h1);
+   b_h1 = 0.0;
+
+   // Setup the boundary conditions on phi
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   ParGridFunction phi_h1(&fes_cond_h1);
+   phi_h1 = 0.0;
+
+   Array<int> bdr0(pmesh_cond.bdr_attributes.Max()); bdr0 = 0;
+   for (int i=0; i<phi0_attr.Size(); i++)
+   {
+      bdr0[phi0_attr[i]-1] = 1;
+   }
+   phi_h1.ProjectBdrCoefficient(zero, bdr0);
+
+   Array<int> bdr1(pmesh_cond.bdr_attributes.Max()); bdr1 = 0;
+   for (int i=0; i<phi1_attr.Size(); i++)
+   {
+      bdr1[phi1_attr[i]-1] = 1;
+   }
+   phi_h1.ProjectBdrCoefficient(one, bdr1);
+
+   // Solve the linear system using algebraic multigrid
+   {
+      if (myid == 0)
+      {
+         cout << "\nSolving for electric potential "
+              << "using CG with AMG" << endl;
+      }
+      OperatorPtr A;
+      Vector B, X;
+      a_h1.FormLinearSystem(ess_bdr_tdof_phi, phi_h1, b_h1, A, X, B);
+
+      HypreBoomerAMG prec;
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      cg.SetPreconditioner(prec);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      a_h1.RecoverFEMSolution(X, b_h1, phi_h1);
+   }
+   {
+      int num_procs = fes_cond_h1.GetNRanks();
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock << "parallel " << num_procs << " " << myid << "\n";
+      port_sock.precision(8);
+      port_sock << "solution\n" << pmesh_cond << phi_h1
+                << "window_title 'Conductor Potential'"
+                << "window_geometry 0 0 400 350" << flush;
+   }
+
+   // Solve for the current density J = -sigma Grad phi with boundary
+   // conditions J.n = 0 on the walls of the conductor but not on the
+   // ports where phi=0 and phi=1.
+
+   // J will be computed in H(div) so we need an RT mass matrix
+   ParBilinearForm m_rt(&fes_cond_rt);
+   m_rt.AddDomainIntegrator(new VectorFEMassIntegrator);
+   m_rt.Assemble();
+
+   // Assemble the (sigma Grad phi) operator
+   ParMixedBilinearForm d_h1(&fes_cond_h1, &fes_cond_rt);
+   d_h1.AddDomainIntegrator(new MixedVectorGradientIntegrator(sigmaCoef));
+   d_h1.Assemble();
+
+   // Compute the r.h.s, b_rt = sigma E = -sigma Grad phi
+   ParLinearForm b_rt(&fes_cond_rt);
+   d_h1.Mult(phi_h1, b_rt);
+   b_rt *= -1.0;
+
+   // Apply the necessary boundary conditions and solve for J in H(div)
+   HYPRE_BigInt glb_size_rt = fes_cond_rt.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "\nSolving for current density in H(Div) "
+           << "using diagonally scaled CG" << endl;
+      cout << "Size of linear system: "
+           << glb_size_rt << endl;
+   }
+   Array<int> ess_bdr_tdof_rt;
+   OperatorPtr M;
+   Vector B, X;
+
+   fes_cond_rt.GetEssentialTrueDofs(ess_bdr_j, ess_bdr_tdof_rt);
+
+   j_cond = 0.0;
+   m_rt.FormLinearSystem(ess_bdr_tdof_rt, j_cond, b_rt, M, X, B);
+
+   HypreDiagScale prec;
+
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(prec);
+   cg.SetOperator(*M);
+   cg.Mult(B, X);
+   m_rt.RecoverFEMSolution(X, b_rt, j_cond);
+}

examples/ex35.cpp

@@ -1,451 +0,0 @@
-//                                MFEM Example 36
-//
-// Compile with: make ex36
-//
-// Sample runs:  ex36
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               discontinuous Galerkin (DG) finite element discretization of
-//               the Laplace problem -Delta u = f with Dirichlet boundary
-//               conditions. Finite element spaces of any order, including zero
-//               on regular grids, are supported. The example highlights the
-//               use of coupling solution domains though custom physics defined
-//               on internal boundaries.
-//
-//               We recommend viewing examples 1, 14, and 34 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-class InteriorMassIntegrator : public BilinearFormIntegrator
-{
-public:
-   InteriorMassIntegrator(Coefficient &Q)
-      : Q(Q)
-   {}
-
-   void AssembleFaceMatrix(const FiniteElement &el1,
-                           const FiniteElement &el2,
-                           FaceElementTransformations &trans,
-                           DenseMatrix &elmat) override;
-
-   using BilinearFormIntegrator::AssembleFaceMatrix;
-
-private:
-   Coefficient &Q;
-
-#ifndef MFEM_THREAD_SAFE
-   Vector shape1;
-   Vector shape2;
-   DenseMatrix elmat11;
-   DenseMatrix elmat12;
-   DenseMatrix elmat21;
-   DenseMatrix elmat22;
-#endif
-};
-
-Mesh generate_mesh(int ref, int internal_bdr_attr = 5);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int ref_levels = 0;
-   int order = 1;
-   int sol_order = 3;
-   double jump = -2;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   double eta = 0.0;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly, -1 for auto.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) >= 0.");
-   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the three DG penalty parameters, typically +1/-1."
-                  " See the documentation of class DGDiffusionIntegrator.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the three DG penalty parameters, should be positive."
-                  " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
-   args.AddOption(&sol_order, "-so", "--solution_order",
-                  "Polynomial order of the exact solution >= 0.");
-   args.AddOption(&jump, "-j", "--jump",
-                  "Value of the discontinuity between the material regions.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   if (kappa < 0)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   if (sol_order < 0)
-   {
-      sol_order = 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Construct the (serial) mesh and refine it if requested.
-   auto mesh = generate_mesh(ref_levels);
-
-   int dim = mesh.Dimension();
-
-   if (mesh.NURBSext)
-   {
-      mesh.SetCurvature(max(order, 1));
-   }
-
-   // 3. Define a finite element space on the mesh. Here we use discontinuous
-   //    finite elements of the specified order >= 0.
-   DG_FECollection fec(order, dim);
-   FiniteElementSpace fespace(&mesh, &fec);
-   cout << "Number of unknowns: " << fespace.GetVSize() << endl;
-
-   // 4. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system.
-   LinearForm b(&fespace);
-
-   Array<int> p1_attr_marker(mesh.attributes.Max());
-   p1_attr_marker = 0;
-   p1_attr_marker[0] = 1;
-
-   FunctionCoefficient p1_source([sol_order](const Vector &p)
-   {
-      const double x = p(0);
-      const double val = -(sol_order - 1)*sol_order*pow(x, sol_order-2);
-      return val;
-   });
-   b.AddDomainIntegrator(new DomainLFIntegrator(p1_source), p1_attr_marker);
-
-   Array<int> p2_attr_marker(mesh.attributes.Max());
-   p2_attr_marker = 0;
-   p2_attr_marker[1] = 1;
-
-   FunctionCoefficient p2_source([sol_order](const Vector &p)
-   {
-      const double x = p(0);
-      double val = -(sol_order - 1)*sol_order*pow(x - 2, sol_order-2);
-      if (sol_order % 2 == 0)
-      {
-         val *= -1.0;
-      }
-      return val;
-   });
-   b.AddDomainIntegrator(new DomainLFIntegrator(p2_source), p2_attr_marker);
-
-   ConstantCoefficient one(1.0);
-
-   Array<int> p1_bdr_attr_marker(mesh.bdr_attributes.Max());
-   p1_bdr_attr_marker = 0;
-   p1_bdr_attr_marker[0] = 1;
-
-   ConstantCoefficient left_bc_val(0.0);
-   b.AddBdrFaceIntegrator(
-      new DGDirichletLFIntegrator(left_bc_val, one, sigma, kappa),
-      p1_bdr_attr_marker);
-
-   Array<int> p2_bdr_attr_marker(mesh.bdr_attributes.Max());
-   p2_bdr_attr_marker = 0;
-   p2_bdr_attr_marker[1] = 1;
-
-   ConstantCoefficient right_bc_val(2.0 + jump);
-   b.AddBdrFaceIntegrator(
-      new DGDirichletLFIntegrator(right_bc_val, one, sigma, kappa),
-      p2_bdr_attr_marker);
-
-   b.Assemble();
-
-   // 5. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 6. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator and the interior and boundary DG face integrators.
-   //    Note that boundary conditions are imposed weakly in the form, so there
-   //    is no need for dof elimination. After assembly and finalizing we
-   //    extract the corresponding sparse matrix A.
-   BilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa),
-                          p1_bdr_attr_marker);
-   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa),
-                          p2_bdr_attr_marker);
-   if (eta > 0)
-   {
-      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-   }
-
-   // 7. Negate the DG interface terms along the internal boundary so that the
-   //    only coupling between domains is from the chosen model (constant flux
-   //    in this case).
-   Array<int> internal_bdr_attr_marker(mesh.bdr_attributes.Max());
-   internal_bdr_attr_marker = 0;
-   internal_bdr_attr_marker[4] = 1;
-
-   ProductCoefficient neg_one(-1.0, one);
-   a.AddInternalBoundaryFaceIntegrator(new DGDiffusionIntegrator(neg_one, sigma,
-                                                                 kappa),
-                                       internal_bdr_attr_marker);
-   if (eta > 0)
-   {
-      a.AddInternalBoundaryFaceIntegrator(new DGDiffusionBR2Integrator(fespace,
-                                                                       neg_one, eta),
-                                          internal_bdr_attr_marker);
-   }
-
-   ConstantCoefficient mass_coeff(sol_order / jump);
-   a.AddInternalBoundaryFaceIntegrator(new InteriorMassIntegrator(mass_coeff),
-                                       internal_bdr_attr_marker);
-
-   a.Assemble();
-   a.Finalize();
-   const SparseMatrix &A = a.SpMat();
-
-#ifndef MFEM_USE_SUITESPARSE
-   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
-   //    non-symmetric one.
-   GSSmoother M(A);
-   if (sigma == -1.0 && !(jump < 0))
-   {
-      PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
-   }
-   else
-   {
-      GMRES(A, M, b, x, 1, 500, 500, 1e-24, 0.0);
-   }
-#else
-   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(b, x);
-#endif
-
-   // 9. Save the refined mesh and the solution. This output can be viewed later
-   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 10. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
-   }
-
-   return 0;
-}
-
-void InteriorMassIntegrator::AssembleFaceMatrix(
-   const FiniteElement &el1,
-   const FiniteElement &el2,
-   FaceElementTransformations &trans,
-   DenseMatrix &elmat)
-{
-   int ndof1 = el1.GetDof();
-   int ndof2 = el2.GetDof();
-   int ndof = ndof1 + ndof2;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector shape1;
-   Vector shape2;
-   DenseMatrix elmat11;
-   DenseMatrix elmat12;
-   DenseMatrix elmat21;
-   DenseMatrix elmat22;
-#endif
-   shape1.SetSize(ndof1);
-   shape2.SetSize(ndof2);
-
-   elmat11.SetSize(ndof1);
-   elmat12.SetSize(ndof1, ndof2);
-   elmat21.SetSize(ndof2, ndof1);
-   elmat22.SetSize(ndof2);
-
-   const auto *ir = IntRule;
-   if (ir == NULL)
-   {
-      int order = 2 * max(el1.GetOrder(), el2.GetOrder());
-      ir = &IntRules.Get(trans.GetGeometryType(), order);
-   }
-
-   elmat.SetSize(ndof);
-   elmat = 0.0;
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const auto &ip = ir->IntPoint(i);
-
-      // Set the integration point in the face and the neighboring element
-      trans.SetAllIntPoints(&ip);
-
-      const double w = ip.weight * trans.Weight();
-
-      // Access the neighboring element's integration point
-      const auto &eip1 = trans.GetElement1IntPoint();
-      const auto &eip2 = trans.GetElement2IntPoint();
-
-      el1.CalcShape(eip1, shape1);
-      el2.CalcShape(eip2, shape2);
-
-      const double Q_val = Q.Eval(trans, ip);
-
-      elmat11 = 0.0;
-      AddMult_a_VVt(Q_val * w, shape1, elmat11);
-
-      elmat12 = 0.0;
-      AddMult_a_VWt(-Q_val * w, shape2, shape1, elmat12);
-
-      elmat21 = 0.0;
-      AddMult_a_VWt(-Q_val * w, shape1, shape2, elmat21);
-
-      elmat22 = 0.0;
-      AddMult_a_VVt(Q_val * w, shape2, elmat22);
-
-      for (int j = 0; j < ndof1; ++j)
-      {
-         for (int k = 0; k < ndof1; ++k)
-         {
-            elmat(j, k) += elmat11(j, k);
-         }
-      }
-
-      for (int j = 0; j < ndof1; ++j)
-      {
-         for (int k = 0; k < ndof2; ++k)
-         {
-            elmat(j, k + ndof1) += elmat12(j, k);
-            elmat(k + ndof1, j) += elmat21(k, j);
-         }
-      }
-
-      for (int j = 0; j < ndof2; ++j)
-      {
-         for (int k = 0; k < ndof2; ++k)
-         {
-            elmat(j + ndof1, k + ndof1) += elmat22(j, k);
-         }
-      }
-   }
-}
-
-Mesh generate_mesh(int ref, int internal_bdr_attr)
-{
-   int nxy = 4 * (ref+1);
-   auto mesh = Mesh::MakeCartesian2D(nxy, nxy, Element::TRIANGLE, true, 2.0, 1.0);
-   // auto mesh = Mesh::MakeCartesian2D(nxy, nxy, Element::QUADRILATERAL, true, 2.0, 1.0);
-
-   // assign element attributes to left and right sides
-   for (int i = 0; i < mesh.GetNE(); ++i)
-   {
-      auto *elem = mesh.GetElement(i);
-
-      Array<int> verts;
-      elem->GetVertices(verts);
-
-      bool left = true;
-      for (int j = 0; j < verts.Size(); ++j)
-      {
-         auto *vtx = mesh.GetVertex(verts[j]);
-         if (vtx[0] <= 1.0)
-         {
-            continue;
-         }
-         else
-         {
-            left = false;
-         }
-      }
-      if (left)
-      {
-         elem->SetAttribute(1);
-      }
-      else
-      {
-         elem->SetAttribute(2);
-      }
-   }
-
-   // assign boundary element attributes to left and right sides
-   for (int i = 0; i < mesh.GetNBE(); ++i)
-   {
-      auto *elem = mesh.GetBdrElement(i);
-
-      Array<int> verts;
-      elem->GetVertices(verts);
-
-      bool left = true;
-      bool right = true;
-      bool top = true;
-      bool bottom = true;
-      for (int j = 0; j < verts.Size(); ++j)
-      {
-         auto *vtx = mesh.GetVertex(verts[j]);
-         left = left && abs(vtx[0] - 0.0) < 1e-12;
-         right = right && abs(vtx[0] - 2.0) < 1e-12;
-         top = top && abs(vtx[1] - 1.0) < 1e-12;
-         bottom = bottom && abs(vtx[1] - 0.0) < 1e-12;
-      }
-      if (left)
-      {
-         elem->SetAttribute(1);
-      }
-      else if (right)
-      {
-         elem->SetAttribute(2);
-      }
-      else if (top)
-      {
-         elem->SetAttribute(3);
-      }
-      else if (bottom)
-      {
-         elem->SetAttribute(4);
-      }
-   }
-
-   // add internal boundary elements
-   for (int i = 0; i < mesh.GetNumFaces(); ++i)
-   {
-      int e1, e2;
-      mesh.GetFaceElements(i, &e1, &e2);
-      if (e1 >= 0 && e2 >= 0 && mesh.GetAttribute(e1) != mesh.GetAttribute(e2))
-      {
-         // This is the internal face between attributes.
-         auto *new_elem = mesh.GetFace(i)->Duplicate(&mesh);
-         new_elem->SetAttribute(internal_bdr_attr);
-         mesh.AddBdrElement(new_elem);
-      }
-   }
-
-   mesh.FinalizeTopology(); // Finalize to build relevant tables
-   mesh.Finalize();
-   mesh.SetAttributes();
-
-   return mesh;
-}

examples/ex35p.cpp

@@ -0,0 +1,818 @@
+//                       MFEM Example 35 - Parallel Version
+//
+// Compile with: make ex35p
+//
+// Sample runs:  mpirun -np 4 ex35p -p 0 -o 2
+//               mpirun -np 4 ex35p -p 0 -o 2 -pbc '22 23 24' -em 0
+//               mpirun -np 4 ex35p -p 1 -o 1 -rp 2
+//               mpirun -np 4 ex35p -p 1 -o 2
+//               mpirun -np 4 ex35p -p 2 -o 1 -rp 2 -c 15
+//
+// Device sample runs:
+//
+// Description:  This example code demonstrates the use of MFEM to define and
+//               solve simple complex-valued linear systems. It implements three
+//               variants of a damped harmonic oscillator:
+//
+//               1) A scalar H1 field
+//                  -Div(a Grad u) - omega^2 b u + i omega c u = 0
+//
+//               2) A vector H(Curl) field
+//                  Curl(a Curl u) - omega^2 b u + i omega c u = 0
+//
+//               3) A vector H(Div) field
+//                  -Grad(a Div u) - omega^2 b u + i omega c u = 0
+//
+//               In each case the field is driven by a forced oscillation, with
+//               angular frequency omega, imposed at the boundary or a portion
+//               of the boundary. The spatial variation of the boundary
+//               condition is computed as an eigenmode of an appropriate
+//               operator defined on a portion of the boundary i.e. a port
+//               boundary condition.
+//
+//               In electromagnetics the coefficients are typically named the
+//               permeability, mu = 1/a, permittivity, epsilon = b, and
+//               conductivity, sigma = c. The user can specify these constants
+//               using either set of names.
+//
+//               This example demonstrates how to transfer fields computed on
+//               a boundary generated SubMesh to the full mesh and apply them
+//               as boundary conditions. The default mesh and corresponding
+//               boundary attriburtes were chosen to verify proper behavior on
+//               both triangular and quadrilateral faces of tetrahedral,
+//               wedge-shaped, and hexahedral elements.
+//
+//               The example also demonstrates how to display a time-varying
+//               solution as a sequence of fields sent to a single GLVis socket.
+//
+//               We recommend viewing examples 11, 13, and 22 before viewing
+//               this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+static double mu_ = 1.0;
+static double epsilon_ = 1.0;
+static double sigma_ = 2.0;
+
+void SetPortBC(int prob, int dim, int mode, ParGridFunction &port_bc);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/fichera-mixed.mesh";
+   int ser_ref_levels = 1;
+   int par_ref_levels = 1;
+   int order = 1;
+   Array<int> port_bc_attr;
+   int prob = 0;
+   int mode = 1;
+   double freq = -1.0;
+   double omega = 2.0 * M_PI;
+   double a_coef = 0.0;
+   bool herm_conv = true;
+   bool slu_solver  = false;
+   bool visualization = 1;
+   bool mixed = true;
+   bool pa = false;
+   const char *device_config = "cpu";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&prob, "-p", "--problem-type",
+                  "Choose between 0: H_1, 1: H(Curl), or 2: H(Div) "
+                  "damped harmonic oscillator.");
+   args.AddOption(&mode, "-em", "--eigenmode",
+                  "Choose the index of the port eigenmode.");
+   args.AddOption(&a_coef, "-a", "--stiffness-coef",
+                  "Stiffness coefficient (spring constant or 1/mu).");
+   args.AddOption(&epsilon_, "-b", "--mass-coef",
+                  "Mass coefficient (or epsilon).");
+   args.AddOption(&sigma_, "-c", "--damping-coef",
+                  "Damping coefficient (or sigma).");
+   args.AddOption(&mu_, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon_, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma_, "-sigma", "--conductivity",
+                  "Conductivity (or damping constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&port_bc_attr, "-pbc", "--port-bc-attr",
+                  "Attributes of port boundary condition");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
+#ifdef MFEM_USE_SUPERLU
+   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
+                  "--no-superlu", "Use the SuperLU Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&mixed, "-mixed", "--mixed-mesh", "-hex",
+                  "--hex-mesh", "Mixed mesh of hexahedral mesh.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+
+   if (!mixed || pa)
+   {
+      mesh_file = "../data/fichera.mesh";
+   }
+
+   if ( a_coef != 0.0 )
+   {
+      mu_ = 1.0 / a_coef;
+   }
+   if ( freq > 0.0 )
+   {
+      omega = 2.0 * M_PI * freq;
+   }
+   if (port_bc_attr.Size() == 0 &&
+       (strcmp(mesh_file, "../data/fichera-mixed.mesh") == 0 ||
+        strcmp(mesh_file, "../data/fichera.mesh") == 0))
+   {
+      port_bc_attr.SetSize(4);
+      port_bc_attr[0] =  7;
+      port_bc_attr[1] =  8;
+      port_bc_attr[2] = 11;
+      port_bc_attr[3] = 12;
+   }
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   MFEM_VERIFY(prob >= 0 && prob <=2,
+               "Unrecognized problem type: " << prob);
+
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution.
+   for (int l = 0; l < ser_ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 6a. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int l = 0; l < par_ref_levels; l++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   // 6b. Extract a submesh covering a portion of the boundary
+   ParSubMesh pmesh_port(ParSubMesh::CreateFromBoundary(pmesh, port_bc_attr));
+
+   // 7a. Define a parallel finite element space on the parallel mesh. Here we
+   //     use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements
+   //     of the specified order.
+   if (dim == 1 && prob != 0 )
+   {
+      if (myid == 0)
+      {
+         cout << "Switching to problem type 0, H1 basis functions, "
+              << "for 1 dimensional mesh." << endl;
+      }
+      prob = 0;
+   }
+
+   FiniteElementCollection *fec = NULL;
+   switch (prob)
+   {
+      case 0:  fec = new H1_FECollection(order, dim);      break;
+      case 1:  fec = new ND_FECollection(order, dim);      break;
+      case 2:  fec = new RT_FECollection(order - 1, dim);  break;
+      default: break; // This should be unreachable
+   }
+   ParFiniteElementSpace fespace(&pmesh, fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7b. Define a parallel finite element space on the sub-mesh. Here we
+   //    use continuous Lagrange, Nedelec, or L2 finite elements of
+   //    the specified order.
+   FiniteElementCollection *fec_port = NULL;
+   switch (prob)
+   {
+      case 0:  fec_port = new H1_FECollection(order, dim-1);      break;
+      case 1:
+         if (dim == 3)
+         {
+            fec_port = new ND_FECollection(order, dim-1);
+         }
+         else
+         {
+            fec_port = new L2_FECollection(order - 1, dim-1,
+                                           BasisType::GaussLegendre,
+                                           FiniteElement::INTEGRAL);
+         }
+         break;
+      case 2:  fec_port = new L2_FECollection(order - 1, dim-1,
+                                                 BasisType::GaussLegendre,
+                                                 FiniteElement::INTEGRAL); break;
+      default: break; // This should be unreachable
+   }
+   ParFiniteElementSpace fespace_port(&pmesh_port, fec_port);
+   HYPRE_BigInt size_port = fespace_port.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element port BC unknowns: " << size_port
+           << endl;
+   }
+
+   // 8a. Define a parallel grid function on the SubMesh which will contain
+   //     the field to be applied as a port boundary condition.
+   ParGridFunction port_bc(&fespace_port);
+   port_bc = 0.0;
+
+   SetPortBC(prob, dim, mode, port_bc);
+
+   // 8b. Save the SubMesh and associated port boundary condition in parallel.
+   //     This output can be viewed later using GLVis:
+   //        "glvis -np <np> -m port_mesh -g port_mode"
+   {
+      ostringstream mesh_name, port_name;
+      mesh_name << "port_mesh." << setfill('0') << setw(6) << myid;
+      port_name << "port_mode." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh_port.Print(mesh_ofs);
+
+      ofstream port_ofs(port_name.str().c_str());
+      port_ofs.precision(8);
+      port_bc.Save(port_ofs);
+   }
+   // 8c. Send the port bc, computed on the SubMesh, to a GLVis server.
+   if (visualization && dim == 3)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream port_sock(vishost, visport);
+      port_sock << "parallel " << num_procs << " " << myid << "\n";
+      port_sock.precision(8);
+      port_sock << "solution\n" << pmesh_port << port_bc
+                << "window_title 'Port BC'"
+                << "window_geometry 0 0 400 350" << flush;
+   }
+
+   // 9. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    using an eigenmode of the appropriate type computed on the SubMesh.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system.
+   ParComplexLinearForm b(&fespace, conv);
+   b.Vector::operator=(0.0);
+
+   // 11a. Define the solution vector u as a parallel complex finite element
+   //      grid function corresponding to fespace. Initialize u to equal zero.
+   ParComplexGridFunction u(&fespace);
+   u = 0.0;
+   pmesh_port.Transfer(port_bc, u.real());
+
+   // 11b. Send the transferred port bc field to a GLVis server.
+   {
+      ParGridFunction full_bc(&fespace);
+      ParTransferMap port_to_full(port_bc, full_bc);
+
+      full_bc = 0.0;
+      port_to_full.Transfer(port_bc, full_bc);
+
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream full_sock(vishost, visport);
+         full_sock << "parallel " << num_procs << " " << myid << "\n";
+         full_sock.precision(8);
+         full_sock << "solution\n" << pmesh << full_bc
+                   << "window_title 'Transferred BC'"
+                   << "window_geometry 400 0 400 350"<< flush;
+      }
+   }
+
+   // 12. Set up the parallel sesquilinear form a(.,.) on the finite element
+   //     space corresponding to the damped harmonic oscillator operator of the
+   //     appropriate type:
+   //
+   //     0) A scalar H1 field
+   //        -Div(a Grad) - omega^2 b + i omega c
+   //
+   //     1) A vector H(Curl) field
+   //        Curl(a Curl) - omega^2 b + i omega c
+   //
+   //     2) A vector H(Div) field
+   //        -Grad(a Div) - omega^2 b + i omega c
+   //
+   ConstantCoefficient stiffnessCoef(1.0/mu_);
+   ConstantCoefficient massCoef(-omega * omega * epsilon_);
+   ConstantCoefficient lossCoef(omega * sigma_);
+   ConstantCoefficient negMassCoef(omega * omega * epsilon_);
+
+   ParSesquilinearForm a(&fespace, conv);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   switch (prob)
+   {
+      case 0:
+         a.AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef),
+                               NULL);
+         a.AddDomainIntegrator(new MassIntegrator(massCoef),
+                               new MassIntegrator(lossCoef));
+         break;
+      case 1:
+         a.AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
+                               NULL);
+         a.AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                               new VectorFEMassIntegrator(lossCoef));
+         break;
+      case 2:
+         a.AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef),
+                               NULL);
+         a.AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                               new VectorFEMassIntegrator(lossCoef));
+         break;
+      default: break; // This should be unreachable
+   }
+
+   // 13. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, etc.
+   a.Assemble();
+
+   OperatorHandle A;
+   Vector B, U;
+
+   a.FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: "
+           << 2 * size << endl << endl;
+   }
+
+   if (!slu_solver)
+   {
+      // 14a. Set up the parallel bilinear form for the preconditioner
+      //      corresponding to the appropriate operator
+      //
+      //      0) A scalar H1 field
+      //         -Div(a Grad) - omega^2 b + i omega c
+      //
+      //      1) A vector H(Curl) field
+      //         Curl(a Curl) + omega^2 b + i omega c
+      //
+      //      2) A vector H(Div) field
+      //         -Grad(a Div) - omega^2 b + i omega c
+      //
+      ParBilinearForm pcOp(&fespace);
+      if (pa) { pcOp.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+      switch (prob)
+      {
+         case 0:
+            pcOp.AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef));
+            pcOp.AddDomainIntegrator(new MassIntegrator(massCoef));
+            pcOp.AddDomainIntegrator(new MassIntegrator(lossCoef));
+            break;
+         case 1:
+            pcOp.AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef));
+            pcOp.AddDomainIntegrator(new VectorFEMassIntegrator(negMassCoef));
+            pcOp.AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
+            break;
+         case 2:
+            pcOp.AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef));
+            pcOp.AddDomainIntegrator(new VectorFEMassIntegrator(massCoef));
+            pcOp.AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
+            break;
+         default: break; // This should be unreachable
+      }
+      pcOp.Assemble();
+
+      // 14b. Define and apply a parallel FGMRES solver for AU=B with a block
+      //     diagonal preconditioner based on the appropriate multigrid
+      //     preconditioner from hypre.
+      Array<int> blockTrueOffsets;
+      blockTrueOffsets.SetSize(3);
+      blockTrueOffsets[0] = 0;
+      blockTrueOffsets[1] = A->Height() / 2;
+      blockTrueOffsets[2] = A->Height() / 2;
+      blockTrueOffsets.PartialSum();
+
+      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
+
+      Operator * pc_r = NULL;
+      Operator * pc_i = NULL;
+
+      if (pa)
+      {
+         pc_r = new OperatorJacobiSmoother(pcOp, ess_tdof_list);
+      }
+      else
+      {
+         OperatorHandle PCOp;
+         pcOp.FormSystemMatrix(ess_tdof_list, PCOp);
+
+         switch (prob)
+         {
+            case 0:
+               pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
+               break;
+            case 1:
+               pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), &fespace);
+               break;
+            case 2:
+               if (dim == 2 )
+               {
+                  pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), &fespace);
+               }
+               else
+               {
+                  pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), &fespace);
+               }
+               break;
+            default: break; // This should be unreachable
+         }
+      }
+      pc_i = new ScaledOperator(pc_r,
+                                (conv == ComplexOperator::HERMITIAN) ?
+                                -1.0:1.0);
+
+      BDP.SetDiagonalBlock(0, pc_r);
+      BDP.SetDiagonalBlock(1, pc_i);
+      BDP.owns_blocks = 1;
+
+      FGMRESSolver fgmres(MPI_COMM_WORLD);
+      fgmres.SetPreconditioner(BDP);
+      fgmres.SetOperator(*A.Ptr());
+      fgmres.SetRelTol(1e-6);
+      fgmres.SetMaxIter(1000);
+      fgmres.SetPrintLevel(1);
+      fgmres.Mult(B, U);
+   }
+#ifdef MFEM_USE_SUPERLU
+   else
+   {
+      // 14. Solve using a direct solver
+      // Transform to monolithic HypreParMatrix
+      HypreParMatrix *A_hyp = A.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      SuperLURowLocMatrix SA(*A_hyp);
+      SuperLUSolver superlu(MPI_COMM_WORLD);
+      superlu.SetPrintStatistics(true);
+      superlu.SetSymmetricPattern(false);
+      superlu.SetColumnPermutation(superlu::PARMETIS);
+      superlu.SetOperator(SA);
+      superlu.Mult(B, U);
+      delete A_hyp;
+   }
+#endif
+
+   // 15. Recover the parallel grid function corresponding to U. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(U, b, u);
+
+   // 16. Save the refined mesh and the solution in parallel. This output can be
+   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol_r" or
+   //     "glvis -np <np> -m mesh -g sol_i".
+   {
+      ostringstream mesh_name, sol_r_name, sol_i_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_r_name << "sol_r." << setfill('0') << setw(6) << myid;
+      sol_i_name << "sol_i." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_r_ofs(sol_r_name.str().c_str());
+      ofstream sol_i_ofs(sol_i_name.str().c_str());
+      sol_r_ofs.precision(8);
+      sol_i_ofs.precision(8);
+      u.real().Save(sol_r_ofs);
+      u.imag().Save(sol_i_ofs);
+   }
+
+   // 17. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_r.precision(8);
+      sol_sock_r << "solution\n" << pmesh << u.real()
+                 << "window_title 'Solution: Real Part'"
+                 << "window_geometry 800 0 400 350" << flush;
+
+      MPI_Barrier(MPI_COMM_WORLD);
+
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i.precision(8);
+      sol_sock_i << "solution\n" << pmesh << u.imag()
+                 << "window_title 'Solution: Imaginary Part'"
+                 << "window_geometry 1200 0 400 350" << flush;
+   }
+   if (visualization)
+   {
+      ParGridFunction u_t(&fespace);
+      u_t = u.real();
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << u_t
+               << "window_title 'Harmonic Solution (t = 0.0 T)'"
+               << "window_geometry 0 432 600 450"
+               << "pause\n" << flush;
+      if (myid == 0)
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 32;
+      int i = 0;
+      while (sol_sock)
+      {
+         double t = (double)(i % num_frames) / num_frames;
+         ostringstream oss;
+         oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos( 2.0 * M_PI * t), u.real(),
+             sin(-2.0 * M_PI * t), u.imag(), u_t);
+         sol_sock << "parallel " << num_procs << " " << myid << "\n";
+         sol_sock << "solution\n" << pmesh << u_t
+                  << "window_title '" << oss.str() << "'" << flush;
+         i++;
+      }
+   }
+
+   // 18. Free the used memory.
+   delete fec_port;
+   delete fec;
+
+   return 0;
+}
+
+/**
+   Solves the eigenvalue problem -Div(Grad x) = lambda x with
+   homogeneous Dirichlet boundary conditions on the boundary of the
+   domain. Returns mode number "mode" (counting from zero) in the
+   ParGridFunction "x".
+*/
+void ScalarWaveGuide(int mode, ParGridFunction &x)
+{
+   int nev = std::max(mode + 2, 5);
+   int seed = 75;
+
+   ParFiniteElementSpace &fespace = *x.ParFESpace();
+   ParMesh &pmesh = *fespace.GetParMesh();
+
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator);
+   a.Assemble();
+   a.EliminateEssentialBCDiag(ess_bdr, 1.0);
+   a.Finalize();
+
+   ParBilinearForm m(&fespace);
+   m.AddDomainIntegrator(new MassIntegrator);
+   m.Assemble();
+   // shift the eigenvalue corresponding to eliminated dofs to a large value
+   m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m.Finalize();
+
+   HypreParMatrix *A = a.ParallelAssemble();
+   HypreParMatrix *M = m.ParallelAssemble();
+
+   HypreBoomerAMG amg(*A);
+   amg.SetPrintLevel(0);
+
+   HypreLOBPCG lobpcg(MPI_COMM_WORLD);
+   lobpcg.SetNumModes(nev);
+   lobpcg.SetRandomSeed(seed);
+   lobpcg.SetPreconditioner(amg);
+   lobpcg.SetMaxIter(200);
+   lobpcg.SetTol(1e-8);
+   lobpcg.SetPrecondUsageMode(1);
+   lobpcg.SetPrintLevel(1);
+   lobpcg.SetMassMatrix(*M);
+   lobpcg.SetOperator(*A);
+   lobpcg.Solve();
+
+   x = lobpcg.GetEigenvector(mode);
+
+   delete A;
+   delete M;
+}
+
+/**
+   Solves the eigenvalue problem -Curl(Curl x) = lambda x with
+   homogeneous Dirichlet boundary conditions, on the tangential
+   component of x, on the boundary of the domain. Returns mode number
+   "mode" (counting from zero) in the ParGridFunction "x".
+*/
+void VectorWaveGuide(int mode, ParGridFunction &x)
+{
+   int nev = std::max(mode + 2, 5);
+
+   ParFiniteElementSpace &fespace = *x.ParFESpace();
+   ParMesh &pmesh = *fespace.GetParMesh();
+
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new CurlCurlIntegrator);
+   a.Assemble();
+   a.EliminateEssentialBCDiag(ess_bdr, 1.0);
+   a.Finalize();
+
+   ParBilinearForm m(&fespace);
+   m.AddDomainIntegrator(new VectorFEMassIntegrator);
+   m.Assemble();
+   // shift the eigenvalue corresponding to eliminated dofs to a large value
+   m.EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m.Finalize();
+
+   HypreParMatrix *A = a.ParallelAssemble();
+   HypreParMatrix *M = m.ParallelAssemble();
+
+   HypreAMS ams(*A,&fespace);
+   ams.SetPrintLevel(0);
+   ams.SetSingularProblem();
+
+   HypreAME ame(MPI_COMM_WORLD);
+   ame.SetNumModes(nev);
+   ame.SetPreconditioner(ams);
+   ame.SetMaxIter(100);
+   ame.SetTol(1e-8);
+   ame.SetPrintLevel(1);
+   ame.SetMassMatrix(*M);
+   ame.SetOperator(*A);
+   ame.Solve();
+
+   x = ame.GetEigenvector(mode);
+
+   delete A;
+   delete M;
+}
+
+/**
+   Solves the eigenvalue problem -Div(Grad x) = lambda x with
+   homogeneous Neumann boundary conditions on the boundary of the
+   domain. Returns mode number "mode" (counting from zero) in the
+   ParGridFunction "x_l2". Note that mode 0 is a constant field so
+   higher mode numbers are often more interesting. The eigenmode is
+   solved using continuous H1 basis of the appropriate order and then
+   projected onto the L2 basis and returned.
+*/
+void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
+{
+   int nev = std::max(mode + 2, 5);
+   int seed = 75;
+
+   ParFiniteElementSpace &fespace_l2 = *x_l2.ParFESpace();
+   ParMesh &pmesh = *fespace_l2.GetParMesh();
+   int order_l2 = fespace_l2.FEColl()->GetOrder();
+
+   H1_FECollection fec(order_l2+1, pmesh.Dimension());
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   GridFunctionCoefficient xCoef(&x);
+
+   if (mode == 0)
+   {
+      x = 1.0;
+      x_l2.ProjectCoefficient(xCoef);
+      return;
+   }
+
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator);
+   a.AddDomainIntegrator(new MassIntegrator); // Shift eigenvalues by 1
+   a.Assemble();
+   a.Finalize();
+
+   ParBilinearForm m(&fespace);
+   m.AddDomainIntegrator(new MassIntegrator);
+   m.Assemble();
+   m.Finalize();
+
+   HypreParMatrix *A = a.ParallelAssemble();
+   HypreParMatrix *M = m.ParallelAssemble();
+
+   HypreBoomerAMG amg(*A);
+   amg.SetPrintLevel(0);
+
+   HypreLOBPCG lobpcg(MPI_COMM_WORLD);
+   lobpcg.SetNumModes(nev);
+   lobpcg.SetRandomSeed(seed);
+   lobpcg.SetPreconditioner(amg);
+   lobpcg.SetMaxIter(200);
+   lobpcg.SetTol(1e-8);
+   lobpcg.SetPrecondUsageMode(1);
+   lobpcg.SetPrintLevel(1);
+   lobpcg.SetMassMatrix(*M);
+   lobpcg.SetOperator(*A);
+   lobpcg.Solve();
+
+   x = lobpcg.GetEigenvector(mode);
+
+   x_l2.ProjectCoefficient(xCoef);
+
+   delete A;
+   delete M;
+}
+
+// Compute eigenmode "mode" of either a Dirichlet or Neumann Laplacian
+// or of a Dirichlet curl curl operator based on the problem type and
+// dimension of the domain.
+void SetPortBC(int prob, int dim, int mode, ParGridFunction &port_bc)
+{
+   switch (prob)
+   {
+      case 0:
+         ScalarWaveGuide(mode, port_bc);
+         break;
+      case 1:
+         if (dim == 3)
+         {
+            VectorWaveGuide(mode, port_bc);
+         }
+         else
+         {
+            PseudoScalarWaveGuide(mode, port_bc);
+         }
+         break;
+      case 2:
+         PseudoScalarWaveGuide(mode, port_bc);
+         break;
+   }
+}

examples/ex36.cpp

@@ -0,0 +1,463 @@
+//                                MFEM Example 36
+//
+//
+// Compile with: make ex36
+//
+// Sample runs: ex36 -o 2
+//              ex36 -o 2 -r 4
+//
+//
+// Description: This example code demonstrates the use of MFEM to solve the
+//              bound-constrained energy minimization problem
+//
+//                      minimize ||∇u||² subject to u ≥ ϕ in H¹₀.
+//
+//              This is known as the obstacle problem, and it is a simple
+//              mathematical model for contact mechanics.
+//
+//              In this example, the obstacle ϕ is a half-sphere centered
+//              at the origin of a circular domain Ω. After solving to a
+//              specified tolerance, the numerical solution is compared to
+//              a closed-form exact solution to assess accuracy.
+//
+//              The problem is discretized and solved using the proximal
+//              Galerkin finite element method, introduced by Keith and
+//              Surowiec [1].
+//
+//              This example highlights the ability of MFEM to deliver high-
+//              order solutions to variation inequality problems and
+//              showcases how to set up and solve nonlinear mixed methods.
+//
+//
+// [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//     preserving finite element method for pointwise bound constraints.
+//     arXiv:2307.12444 [math.NA]
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double spherical_obstacle(const Vector &pt);
+double exact_solution_obstacle(const Vector &pt);
+void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad);
+
+class LogarithmGridFunctionCoefficient : public Coefficient
+{
+protected:
+   GridFunction *u; // grid function
+   Coefficient *obstacle;
+   double min_val;
+
+public:
+   LogarithmGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
+                                    double min_val_=-36)
+      : u(&u_), obstacle(&obst_), min_val(min_val_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+class ExponentialGridFunctionCoefficient : public Coefficient
+{
+protected:
+   GridFunction *u; // grid function
+   Coefficient *obstacle;
+   double min_val;
+   double max_val;
+
+public:
+   ExponentialGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
+                                      double min_val_=0.0, double max_val_=1e6)
+      : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int order = 1;
+   int max_it = 10;
+   int ref_levels = 3;
+   double alpha = 1.0;
+   double tol = 1e-5;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the mesh file.
+   const char *mesh_file = "../data/disc-nurbs.mesh";
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 3C. Rescale the domain to a unit circle (radius = 1).
+   GridFunction *nodes = mesh.GetNodes();
+   double scale = 2*sqrt(2);
+   *nodes /= scale;
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   H1_FECollection H1fec(order+1, dim);
+   FiniteElementSpace H1fes(&mesh, &H1fec);
+
+   L2_FECollection L2fec(order-1, dim);
+   FiniteElementSpace L2fes(&mesh, &L2fec);
+
+   cout << "Number of H1 finite element unknowns: "
+        << H1fes.GetTrueVSize() << endl;
+   cout << "Number of L2 finite element unknowns: "
+        << L2fes.GetTrueVSize() << endl;
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = H1fes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   // 6. Define an initial guess for the solution.
+   auto IC_func = [](const Vector &x)
+   {
+      double r0 = 1.0;
+      double rr = 0.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         rr += x(i)*x(i);
+      }
+      return r0*r0 - rr;
+   };
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+
+   // 7. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   GridFunction u_gf, delta_psi_gf;
+
+   u_gf.MakeRef(&H1fes,x,offsets[0]);
+   delta_psi_gf.MakeRef(&L2fes,x,offsets[1]);
+   delta_psi_gf = 0.0;
+
+   GridFunction u_old_gf(&H1fes);
+   GridFunction psi_old_gf(&L2fes);
+   GridFunction psi_gf(&L2fes);
+   u_old_gf = 0.0;
+   psi_old_gf = 0.0;
+
+   // 8. Define the function coefficients for the solution and use them to
+   //    initialize the initial guess
+   FunctionCoefficient exact_coef(exact_solution_obstacle);
+   VectorFunctionCoefficient exact_grad_coef(dim,exact_solution_gradient_obstacle);
+   FunctionCoefficient IC_coef(IC_func);
+   ConstantCoefficient f(0.0);
+   FunctionCoefficient obstacle(spherical_obstacle);
+   u_gf.ProjectCoefficient(IC_coef);
+   u_old_gf = u_gf;
+
+   // 9. Initialize the slack variable ψₕ = exp(uₕ)
+   LogarithmGridFunctionCoefficient ln_u(u_gf, obstacle);
+   psi_gf.ProjectCoefficient(ln_u);
+   psi_old_gf = psi_gf;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 10. Iterate
+   int k;
+   int total_iterations = 0;
+   double increment_u = 0.1;
+   for (k = 0; k < max_it; k++)
+   {
+      GridFunction u_tmp(&H1fes);
+      u_tmp = u_old_gf;
+
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+
+      int j;
+      for ( j = 0; j < 10; j++)
+      {
+         total_iterations++;
+
+         ConstantCoefficient alpha_cf(alpha);
+
+         LinearForm b0,b1;
+         b0.Update(&H1fes,rhs.GetBlock(0),0);
+         b1.Update(&L2fes,rhs.GetBlock(1),0);
+
+         ExponentialGridFunctionCoefficient exp_psi(psi_gf, zero);
+         ProductCoefficient neg_exp_psi(-1.0,exp_psi);
+         GradientGridFunctionCoefficient grad_u_old(&u_old_gf);
+         ProductCoefficient alpha_f(alpha, f);
+         GridFunctionCoefficient psi_cf(&psi_gf);
+         GridFunctionCoefficient psi_old_cf(&psi_old_gf);
+         SumCoefficient psi_old_minus_psi(psi_old_cf, psi_cf, 1.0, -1.0);
+
+         b0.AddDomainIntegrator(new DomainLFIntegrator(alpha_f));
+         b0.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+         b0.Assemble();
+
+         b1.AddDomainIntegrator(new DomainLFIntegrator(exp_psi));
+         b1.AddDomainIntegrator(new DomainLFIntegrator(obstacle));
+         b1.Assemble();
+
+         BilinearForm a00(&H1fes);
+         a00.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+         a00.AddDomainIntegrator(new DiffusionIntegrator(alpha_cf));
+         a00.Assemble();
+         a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),
+                                  mfem::Operator::DIAG_ONE);
+         a00.Finalize();
+         SparseMatrix &A00 = a00.SpMat();
+
+         MixedBilinearForm a10(&H1fes,&L2fes);
+         a10.AddDomainIntegrator(new MixedScalarMassIntegrator());
+         a10.Assemble();
+         a10.EliminateTrialDofs(ess_bdr, x.GetBlock(0), rhs.GetBlock(1));
+         a10.Finalize();
+         SparseMatrix &A10 = a10.SpMat();
+
+         SparseMatrix *A01 = Transpose(A10);
+
+         BilinearForm a11(&L2fes);
+         a11.AddDomainIntegrator(new MassIntegrator(neg_exp_psi));
+         // NOTE: Shift the spectrum of the Hessian matrix for additional
+         //       stability (Quasi-Newton).
+         ConstantCoefficient eps_cf(-1e-6);
+         if (order == 1)
+         {
+            // NOTE: ∇ₕuₕ = 0 for constant functions.
+            //       Therefore, we use the mass matrix to shift the spectrum
+            a11.AddDomainIntegrator(new MassIntegrator(eps_cf));
+         }
+         else
+         {
+            a11.AddDomainIntegrator(new DiffusionIntegrator(eps_cf));
+         }
+         a11.Assemble();
+         a11.Finalize();
+         SparseMatrix &A11 = a11.SpMat();
+
+         BlockOperator A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+
+         BlockDiagonalPreconditioner prec(offsets);
+         prec.SetDiagonalBlock(0,new GSSmoother(A00));
+         prec.SetDiagonalBlock(1,new GSSmoother(A11));
+         prec.owns_blocks = 1;
+
+         GMRES(A,prec,rhs,x,0,10000,500,1e-12,0.0);
+
+         u_gf.MakeRef(&H1fes, x.GetBlock(0), 0);
+         delta_psi_gf.MakeRef(&L2fes, x.GetBlock(1), 0);
+
+         u_tmp -= u_gf;
+         double Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         double gamma = 1.0;
+         delta_psi_gf *= gamma;
+         psi_gf += delta_psi_gf;
+
+         if (visualization)
+         {
+            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         delete A01;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+      mfem::out << "H1-error  (|| u - uₕᵏ||)       = " << H1_error << endl;
+
+   }
+
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << H1fes.GetTrueVSize() + L2fes.GetTrueVSize()
+             << endl;
+
+   // 11. Exact solution.
+   if (visualization)
+   {
+      socketstream err_sock(vishost, visport);
+      err_sock.precision(8);
+
+      GridFunction error_gf(&H1fes);
+      error_gf.ProjectCoefficient(exact_coef);
+      error_gf -= u_gf;
+
+      err_sock << "solution\n" << mesh << error_gf << "window_title 'Error'"  <<
+               flush;
+   }
+
+   {
+      double L2_error = u_gf.ComputeL2Error(exact_coef);
+      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+
+      ExponentialGridFunctionCoefficient u_alt_cf(psi_gf,obstacle);
+      GridFunction u_alt_gf(&L2fes);
+      u_alt_gf.ProjectCoefficient(u_alt_cf);
+      double L2_error_alt = u_alt_gf.ComputeL2Error(exact_coef);
+
+      mfem::out << "\n Final L2-error (|| u - uₕ||)          = " << L2_error <<
+                endl;
+      mfem::out << " Final H1-error (|| u - uₕ||)          = " << H1_error << endl;
+      mfem::out << " Final L2-error (|| u - ϕ - exp(ψₕ)||) = " << L2_error_alt <<
+                endl;
+   }
+
+   return 0;
+}
+
+double LogarithmGridFunctionCoefficient::Eval(ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(u != NULL, "grid function is not set");
+
+   double val = u->GetValue(T, ip) - obstacle->Eval(T, ip);
+   return max(min_val, log(val));
+}
+
+double ExponentialGridFunctionCoefficient::Eval(ElementTransformation &T,
+                                                const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(u != NULL, "grid function is not set");
+
+   double val = u->GetValue(T, ip);
+   return min(max_val, max(min_val, exp(val) + obstacle->Eval(T, ip)));
+}
+
+double spherical_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double beta = 0.9;
+
+   double b = r0*beta;
+   double tmp = sqrt(r0*r0 - b*b);
+   double B = tmp + b*b/tmp;
+   double C = -b/tmp;
+
+   if (r > b)
+   {
+      return B + r * C;
+   }
+   else
+   {
+      return sqrt(r0*r0 - r*r);
+   }
+}
+
+double exact_solution_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      return A * log(r);
+   }
+   else
+   {
+      return sqrt(r0*r0-r*r);
+   }
+}
+
+void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      grad(0) =  A * x / (r*r);
+      grad(1) =  A * y / (r*r);
+   }
+   else
+   {
+      grad(0) = - x / sqrt( r0*r0 - r*r );
+      grad(1) = - y / sqrt( r0*r0 - r*r );
+   }
+}

examples/ex36p.cpp

@@ -0,0 +1,528 @@
+//                         MFEM Example 36 - Parallel Version
+//
+//
+// Compile with: make ex36p
+//
+// Sample runs: mpirun -np 4 ex36p -o 2
+//              mpirun -np 4 ex36p -o 2 -r 4
+//
+//
+// Description: This example code demonstrates the use of MFEM to solve the
+//              bound-constrained energy minimization problem
+//
+//                      minimize ||∇u||² subject to u ≥ ϕ in H¹₀.
+//
+//              This is known as the obstacle problem, and it is a simple
+//              mathematical model for contact mechanics.
+//
+//              In this example, the obstacle ϕ is a half-sphere centered
+//              at the origin of a circular domain Ω. After solving to a
+//              specified tolerance, the numerical solution is compared to
+//              a closed-form exact solution to assess accuracy.
+//
+//              The problem is discretized and solved using the proximal
+//              Galerkin finite element method, introduced by Keith and
+//              Surowiec [1].
+//
+//              This example highlights the ability of MFEM to deliver high-
+//              order solutions to variation inequality problems and
+//              showcases how to set up and solve nonlinear mixed methods.
+//
+//
+// [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//     preserving finite element method for pointwise bound constraints.
+//     arXiv:2307.12444 [math.NA]
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double spherical_obstacle(const Vector &pt);
+double exact_solution_obstacle(const Vector &pt);
+void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad);
+
+class LogarithmGridFunctionCoefficient : public Coefficient
+{
+protected:
+   GridFunction *u; // grid function
+   Coefficient *obstacle;
+   double min_val;
+
+public:
+   LogarithmGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
+                                    double min_val_=-36)
+      : u(&u_), obstacle(&obst_), min_val(min_val_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+class ExponentialGridFunctionCoefficient : public Coefficient
+{
+protected:
+   GridFunction *u; // grid function
+   Coefficient *obstacle;
+   double min_val;
+   double max_val;
+
+public:
+   ExponentialGridFunctionCoefficient(GridFunction &u_, Coefficient &obst_,
+                                      double min_val_=0.0, double max_val_=1e6)
+      : u(&u_), obstacle(&obst_), min_val(min_val_), max_val(max_val_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   int order = 1;
+   int max_it = 10;
+   int ref_levels = 3;
+   double alpha = 1.0;
+   double tol = 1e-5;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Read the mesh from the mesh file.
+   const char *mesh_file = "../data/disc-nurbs.mesh";
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 3C. Rescale the domain to a unit circle (radius = 1).
+   GridFunction *nodes = mesh.GetNodes();
+   double scale = 2*sqrt(2);
+   *nodes /= scale;
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   H1_FECollection H1fec(order+1, dim);
+   ParFiniteElementSpace H1fes(&pmesh, &H1fec);
+
+   L2_FECollection L2fec(order-1, dim);
+   ParFiniteElementSpace L2fes(&pmesh, &L2fec);
+
+   int num_dofs_H1 = H1fes.GetTrueVSize();
+   MPI_Allreduce(MPI_IN_PLACE, &num_dofs_H1, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+   int num_dofs_L2 = L2fes.GetTrueVSize();
+   MPI_Allreduce(MPI_IN_PLACE, &num_dofs_L2, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+   if (myid == 0)
+   {
+      cout << "Number of H1 finite element unknowns: "
+           << num_dofs_H1 << endl;
+      cout << "Number of L2 finite element unknowns: "
+           << num_dofs_L2 << endl;
+   }
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = H1fes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   Array<int> toffsets(3);
+   toffsets[0] = 0;
+   toffsets[1] = H1fes.GetTrueVSize();
+   toffsets[2] = L2fes.GetTrueVSize();
+   toffsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   BlockVector tx(toffsets), trhs(toffsets);
+   tx = 0.0; trhs = 0.0;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   Array<int> empty;
+   Array<int> ess_tdof_list;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 6. Define an initial guess for the solution.
+   auto IC_func = [](const Vector &x)
+   {
+      double r0 = 1.0;
+      double rr = 0.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         rr += x(i)*x(i);
+      }
+      return r0*r0 - rr;
+   };
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+
+   // 7. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   ParGridFunction u_gf, delta_psi_gf;
+   u_gf.MakeRef(&H1fes,x,offsets[0]);
+   delta_psi_gf.MakeRef(&L2fes,x,offsets[1]);
+   delta_psi_gf = 0.0;
+
+   ParGridFunction u_old_gf(&H1fes);
+   ParGridFunction psi_old_gf(&L2fes);
+   ParGridFunction psi_gf(&L2fes);
+   u_old_gf = 0.0;
+   psi_old_gf = 0.0;
+
+
+   // 8. Define the function coefficients for the solution and use them to
+   //    initialize the initial guess
+   FunctionCoefficient exact_coef(exact_solution_obstacle);
+   VectorFunctionCoefficient exact_grad_coef(dim,exact_solution_gradient_obstacle);
+   FunctionCoefficient IC_coef(IC_func);
+   ConstantCoefficient f(0.0);
+   FunctionCoefficient obstacle(spherical_obstacle);
+   u_gf.ProjectCoefficient(IC_coef);
+   u_old_gf = u_gf;
+
+   // 9. Initialize the slack variable ψₕ = exp(uₕ)
+   LogarithmGridFunctionCoefficient ln_u(u_gf, obstacle);
+   psi_gf.ProjectCoefficient(ln_u);
+   psi_old_gf = psi_gf;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 10. Iterate
+   int k;
+   int total_iterations = 0;
+   double increment_u = 0.1;
+   for (k = 0; k < max_it; k++)
+   {
+      ParGridFunction u_tmp(&H1fes);
+      u_tmp = u_old_gf;
+
+      if (myid == 0)
+      {
+         mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+      }
+
+      int j;
+      for ( j = 0; j < 10; j++)
+      {
+         total_iterations++;
+
+         ConstantCoefficient alpha_cf(alpha);
+
+         ParLinearForm b0,b1;
+         b0.Update(&H1fes,rhs.GetBlock(0),0);
+         b1.Update(&L2fes,rhs.GetBlock(1),0);
+
+         ExponentialGridFunctionCoefficient exp_psi(psi_gf, zero);
+         ProductCoefficient neg_exp_psi(-1.0,exp_psi);
+         GradientGridFunctionCoefficient grad_u_old(&u_old_gf);
+         ProductCoefficient alpha_f(alpha, f);
+         GridFunctionCoefficient psi_cf(&psi_gf);
+         GridFunctionCoefficient psi_old_cf(&psi_old_gf);
+         SumCoefficient psi_old_minus_psi(psi_old_cf, psi_cf, 1.0, -1.0);
+
+         b0.AddDomainIntegrator(new DomainLFIntegrator(alpha_f));
+         b0.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+         b0.Assemble();
+
+         b1.AddDomainIntegrator(new DomainLFIntegrator(exp_psi));
+         b1.AddDomainIntegrator(new DomainLFIntegrator(obstacle));
+         b1.Assemble();
+
+         ParBilinearForm a00(&H1fes);
+         a00.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+         a00.AddDomainIntegrator(new DiffusionIntegrator(alpha_cf));
+         a00.Assemble();
+         HypreParMatrix A00;
+         a00.FormLinearSystem(ess_tdof_list, x.GetBlock(0), rhs.GetBlock(0),
+                              A00, tx.GetBlock(0), trhs.GetBlock(0));
+
+
+         ParMixedBilinearForm a10(&H1fes,&L2fes);
+         a10.AddDomainIntegrator(new MixedScalarMassIntegrator());
+         a10.Assemble();
+         HypreParMatrix A10;
+         a10.FormRectangularLinearSystem(ess_tdof_list, empty, x.GetBlock(0),
+                                         rhs.GetBlock(1),
+                                         A10, tx.GetBlock(0), trhs.GetBlock(1));
+
+         HypreParMatrix *A01 = A10.Transpose();
+
+         ParBilinearForm a11(&L2fes);
+         a11.AddDomainIntegrator(new MassIntegrator(neg_exp_psi));
+         // NOTE: Shift the spectrum of the Hessian matrix for additional
+         //       stability (Quasi-Newton).
+         ConstantCoefficient eps_cf(-1e-6);
+         if (order == 1)
+         {
+            // NOTE: ∇ₕuₕ = 0 for constant functions.
+            //       Therefore, we use the mass matrix to shift the spectrum
+            a11.AddDomainIntegrator(new MassIntegrator(eps_cf));
+         }
+         else
+         {
+            a11.AddDomainIntegrator(new DiffusionIntegrator(eps_cf));
+         }
+         a11.Assemble();
+         a11.Finalize();
+         HypreParMatrix A11;
+         a11.FormSystemMatrix(empty, A11);
+
+         BlockOperator A(toffsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+
+         BlockDiagonalPreconditioner prec(toffsets);
+         HypreBoomerAMG P00(A00);
+         P00.SetPrintLevel(0);
+         HypreSmoother P11(A11);
+         prec.SetDiagonalBlock(0,&P00);
+         prec.SetDiagonalBlock(1,&P11);
+
+         GMRESSolver gmres(MPI_COMM_WORLD);
+         gmres.SetPrintLevel(-1);
+         gmres.SetRelTol(1e-8);
+         gmres.SetMaxIter(20000);
+         gmres.SetKDim(500);
+         gmres.SetOperator(A);
+         gmres.SetPreconditioner(prec);
+         gmres.Mult(trhs,tx);
+
+         u_gf.SetFromTrueDofs(tx.GetBlock(0));
+         delta_psi_gf.SetFromTrueDofs(tx.GetBlock(1));
+
+         u_tmp -= u_gf;
+         double Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         double gamma = 1.0;
+         delta_psi_gf *= gamma;
+         psi_gf += delta_psi_gf;
+
+         if (visualization)
+         {
+            sol_sock << "parallel " << num_procs << " " << myid << "\n";
+            sol_sock << "solution\n" << pmesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         if (myid == 0)
+         {
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         delete A01;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      if (myid == 0)
+      {
+         mfem::out << "Number of Newton iterations = " << j+1 << endl;
+         mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+      }
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+      if (myid == 0)
+      {
+         mfem::out << "H1-error  (|| u - uₕᵏ||)       = " << H1_error << endl;
+      }
+
+   }
+
+   if (myid == 0)
+   {
+      mfem::out << "\n Outer iterations: " << k+1
+                << "\n Total iterations: " << total_iterations
+                << "\n Total dofs:       " << num_dofs_H1 + num_dofs_L2
+                << endl;
+   }
+
+   // 11. Exact solution.
+   if (visualization)
+   {
+      socketstream err_sock(vishost, visport);
+      err_sock.precision(8);
+
+      ParGridFunction error_gf(&H1fes);
+      error_gf.ProjectCoefficient(exact_coef);
+      error_gf -= u_gf;
+
+      err_sock << "parallel " << num_procs << " " << myid << "\n";
+      err_sock << "solution\n" << pmesh << error_gf << "window_title 'Error'"  <<
+               flush;
+   }
+
+   {
+      double L2_error = u_gf.ComputeL2Error(exact_coef);
+      double H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
+
+      ExponentialGridFunctionCoefficient u_alt_cf(psi_gf,obstacle);
+      ParGridFunction u_alt_gf(&L2fes);
+      u_alt_gf.ProjectCoefficient(u_alt_cf);
+      double L2_error_alt = u_alt_gf.ComputeL2Error(exact_coef);
+
+      if (myid == 0)
+      {
+         mfem::out << "\n Final L2-error (|| u - uₕ||)          = " << L2_error <<
+                   endl;
+         mfem::out << " Final H1-error (|| u - uₕ||)          = " << H1_error << endl;
+         mfem::out << " Final L2-error (|| u - ϕ - exp(ψₕ)||) = " << L2_error_alt <<
+                   endl;
+      }
+   }
+
+   return 0;
+}
+
+double LogarithmGridFunctionCoefficient::Eval(ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(u != NULL, "grid function is not set");
+
+   double val = u->GetValue(T, ip) - obstacle->Eval(T, ip);
+   return max(min_val, log(val));
+}
+
+double ExponentialGridFunctionCoefficient::Eval(ElementTransformation &T,
+                                                const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(u != NULL, "grid function is not set");
+
+   double val = u->GetValue(T, ip);
+   return min(max_val, max(min_val, exp(val) + obstacle->Eval(T, ip)));
+}
+
+double spherical_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double beta = 0.9;
+
+   double b = r0*beta;
+   double tmp = sqrt(r0*r0 - b*b);
+   double B = tmp + b*b/tmp;
+   double C = -b/tmp;
+
+   if (r > b)
+   {
+      return B + r * C;
+   }
+   else
+   {
+      return sqrt(r0*r0 - r*r);
+   }
+}
+
+double exact_solution_obstacle(const Vector &pt)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      return A * log(r);
+   }
+   else
+   {
+      return sqrt(r0*r0-r*r);
+   }
+}
+
+void exact_solution_gradient_obstacle(const Vector &pt, Vector &grad)
+{
+   double x = pt(0), y = pt(1);
+   double r = sqrt(x*x + y*y);
+   double r0 = 0.5;
+   double a =  0.348982574111686;
+   double A = -0.340129705945858;
+
+   if (r > a)
+   {
+      grad(0) =  A * x / (r*r);
+      grad(1) =  A * y / (r*r);
+   }
+   else
+   {
+      grad(0) = - x / sqrt( r0*r0 - r*r );
+      grad(1) = - y / sqrt( r0*r0 - r*r );
+   }
+}

examples/ex9p.cpp

@@ -536,8 +536,10 @@ int main(int argc, char *argv[])
       if (!sout)
       {
          if (Mpi::Root())
+         {
             cout << "Unable to connect to GLVis server at "
                  << vishost << ':' << visport << endl;
+         }
          visualization = false;
          if (Mpi::Root())
          {
@@ -552,8 +554,10 @@ int main(int argc, char *argv[])
          sout << "pause\n";
          sout << flush;
          if (Mpi::Root())
+         {
             cout << "GLVis visualization paused."
                  << " Press space (in the GLVis window) to resume it.\n";
+         }
       }
    }
 

examples/makefile

@@ -23,13 +23,13 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-  ex31 ex33 ex34 ex35
+  ex31 ex33 ex34 ex36
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-  ex24p ex25p ex26p
+  ex24p ex25p ex26p ex34p ex35p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -183,3 +183,4 @@ clean-exec:
 	@rm -f ex23.mesh ex23-*.gf
 	@rm -f ex25.mesh ex25-*.gf ex25p-*.*
 	@rm -rf ex28_* ex28p_*
+	@rm -rf cond.* cond_mesh.* cond_j.* dsol.* port_mesh.* port_mode.*

examples/sundials/CMakeLists.txt

@@ -68,11 +68,43 @@ if (MFEM_ENABLE_TESTING)
       add_test(NAME ${TEST_NAME}_ser
         COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
     else()
-      add_test(NAME ${TEST_NAME}_np=4
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
         COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
         ${MPIEXEC_PREFLAGS}
         $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
         ${MPIEXEC_POSTFLAGS})
     endif()
   endforeach()
-endif()
+
+  # Add CUDA/HIP tests.
+  set(DEVICE_EXAMPLES
+      # serial examples with device support:
+      ex9
+      # parallel examples with device support:
+      ex9p)
+  set(MFEM_TEST_DEVICE)
+  if (MFEM_USE_CUDA)
+    set(MFEM_TEST_DEVICE "cuda")
+  elseif (MFEM_USE_HIP)
+    set(MFEM_TEST_DEVICE "hip")
+  endif()
+  if (MFEM_TEST_DEVICE)
+    foreach(TEST_NAME ${DEVICE_EXAMPLES})
+      string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+
+      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+
+      if (NOT (${TEST_NAME} MATCHES ".*p$"))
+        add_test(NAME ${PFX}${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+          COMMAND ${PFX}${TEST_NAME} ${THIS_TEST_OPTIONS})
+      else()
+        add_test(NAME ${PFX}${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+          COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+          ${MPIEXEC_PREFLAGS}
+          $<TARGET_FILE:${PFX}${TEST_NAME}> ${THIS_TEST_OPTIONS}
+          ${MPIEXEC_POSTFLAGS})
+      endif()
+    endforeach()
+  endif(MFEM_TEST_DEVICE)
+endif(MFEM_ENABLE_TESTING)

examples/sundials/README

@@ -12,8 +12,7 @@ use of MFEM features based on the SUNDIALS suite of time integration and
 non-linear solvers.
 
 To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_SUNDIALS = YES", see the top-level INSTALL file for details (version
-2.7 or higher of SUNDIALS is required).
+"MFEM_USE_SUNDIALS = YES", see the top-level INSTALL file for details.
 
 We recommend comparing the original example codes with the corresponding files
 in the current directory.

examples/sundials/ex9.cpp

@@ -280,15 +280,16 @@ int main(int argc, char *argv[])
       k.SetAssemblyLevel(AssemblyLevel::FULL);
    }
    m.AddDomainIntegrator(new MassIntegrator);
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   constexpr double alpha = -1.0;
+   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
    k.AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
    k.AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
 
    LinearForm b(&fes);
    b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+      new BoundaryFlowIntegrator(inflow, velocity, alpha));
 
    m.Assemble();
    int skip_zeros = 0;

examples/sundials/ex9p.cpp

@@ -63,6 +63,66 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
+// Type of preconditioner for implicit time integrator
+enum class PrecType : int
+{
+   ILU = 0,
+   AIR = 1
+};
+
+#if MFEM_HYPRE_VERSION >= 21800
+// Algebraic multigrid preconditioner for advective problems based on
+// approximate ideal restriction (AIR). Most effective when matrix is
+// first scaled by DG block inverse, and AIR applied to scaled matrix.
+// See https://doi.org/10.1137/17M1144350.
+class AIR_prec : public Solver
+{
+private:
+   const HypreParMatrix *A;
+   // Copy of A scaled by block-diagonal inverse
+   HypreParMatrix A_s;
+
+   HypreBoomerAMG *AIR_solver;
+   int blocksize;
+
+public:
+   AIR_prec(int blocksize_) : AIR_solver(NULL), blocksize(blocksize_) { }
+
+   void SetOperator(const Operator &op)
+   {
+      width = op.Width();
+      height = op.Height();
+
+      A = dynamic_cast<const HypreParMatrix *>(&op);
+      MFEM_VERIFY(A != NULL, "AIR_prec requires a HypreParMatrix.")
+
+      // Scale A by block-diagonal inverse
+      BlockInverseScale(A, &A_s, NULL, NULL, blocksize,
+                        BlockInverseScaleJob::MATRIX_ONLY);
+      delete AIR_solver;
+      AIR_solver = new HypreBoomerAMG(A_s);
+      AIR_solver->SetAdvectiveOptions(1, "", "FA");
+      AIR_solver->SetPrintLevel(0);
+      AIR_solver->SetMaxLevels(50);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      // Scale the rhs by block inverse and solve system
+      HypreParVector z_s;
+      BlockInverseScale(A, NULL, &x, &z_s, blocksize,
+                        BlockInverseScaleJob::RHS_ONLY);
+      AIR_solver->Mult(z_s, y);
+   }
+
+   ~AIR_prec()
+   {
+      delete AIR_solver;
+   }
+};
+#endif
+
+
 class DG_Solver : public Solver
 {
 private:
@@ -70,24 +130,37 @@ private:
    SparseMatrix M_diag;
    HypreParMatrix *A;
    GMRESSolver linear_solver;
-   BlockILU prec;
+   Solver *prec;
    double dt;
 public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
+   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes,
+             PrecType prec_type)
       : M(M_),
         K(K_),
         A(NULL),
         linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
         dt(-1.0)
    {
+      int block_size = fes.GetFE(0)->GetDof();
+      if (prec_type == PrecType::ILU)
+      {
+         prec = new BlockILU(block_size,
+                             BlockILU::Reordering::MINIMUM_DISCARDED_FILL);
+      }
+      else if (prec_type == PrecType::AIR)
+      {
+#if MFEM_HYPRE_VERSION >= 21800
+         prec = new AIR_prec(block_size);
+#else
+         MFEM_ABORT("Must have MFEM_HYPRE_VERSION >= 21800 to use AIR.\n");
+#endif
+      }
       linear_solver.iterative_mode = false;
       linear_solver.SetRelTol(1e-9);
       linear_solver.SetAbsTol(0.0);
       linear_solver.SetMaxIter(100);
       linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
+      linear_solver.SetPreconditioner(*prec);
 
       M.GetDiag(M_diag);
    }
@@ -120,10 +193,12 @@ public:
 
    ~DG_Solver()
    {
+      delete prec;
       delete A;
    }
 };
 
+
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
     and advection matrices, and b describes the flow on the boundary. This can
@@ -141,7 +216,8 @@ private:
    mutable Vector z;
 
 public:
-   FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_);
+   FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_,
+                PrecType prec_type);
 
    virtual void Mult(const Vector &x, Vector &y) const;
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
@@ -178,6 +254,11 @@ int main(int argc, char *argv[])
    bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
+#if MFEM_HYPRE_VERSION >= 21800
+   PrecType prec_type = PrecType::AIR;
+#else
+   PrecType prec_type = PrecType::ILU;
+#endif
 
    // Relative and absolute tolerances for CVODE and ARKODE.
    const double reltol = 1e-2, abstol = 1e-2;
@@ -218,6 +299,8 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
+   args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
+                  "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -238,13 +321,13 @@ int main(int argc, char *argv[])
    args.Parse();
    if (!args.Good())
    {
-      if (myid == 0)
+      if (Mpi::Root())
       {
          args.PrintUsage(cout);
       }
       return 1;
    }
-   if (myid == 0)
+   if (Mpi::Root())
    {
       args.PrintOptions(cout);
    }
@@ -252,7 +335,7 @@ int main(int argc, char *argv[])
    // check for valid ODE solver option
    if (ode_solver_type < 1 || ode_solver_type > 9)
    {
-      if (myid == 0)
+      if (Mpi::Root())
       {
          cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
       }
@@ -260,7 +343,7 @@ int main(int argc, char *argv[])
    }
 
    Device device(device_config);
-   if (myid == 0) { device.Print(); }
+   if (Mpi::Root()) { device.Print(); }
 
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle geometrically periodic meshes in this code.
@@ -297,7 +380,7 @@ int main(int argc, char *argv[])
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
 
    HYPRE_BigInt global_vSize = fes->GlobalTrueVSize();
-   if (myid == 0)
+   if (Mpi::Root())
    {
       cout << "Number of unknowns: " << global_vSize << endl;
    }
@@ -328,15 +411,16 @@ int main(int argc, char *argv[])
    }
 
    m->AddDomainIntegrator(new MassIntegrator);
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   constexpr double alpha = -1.0;
+   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
    k->AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
    k->AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
 
    ParLinearForm *b = new ParLinearForm(fes);
    b->AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+      new BoundaryFlowIntegrator(inflow, velocity, alpha));
 
    int skip_zeros = 0;
    m->Assemble();
@@ -435,11 +519,13 @@ int main(int argc, char *argv[])
       sout.open(vishost, visport);
       if (!sout)
       {
-         if (myid == 0)
+         if (Mpi::Root())
+         {
             cout << "Unable to connect to GLVis server at "
                  << vishost << ':' << visport << endl;
+         }
          visualization = false;
-         if (myid == 0)
+         if (Mpi::Root())
          {
             cout << "GLVis visualization disabled.\n";
          }
@@ -451,15 +537,17 @@ int main(int argc, char *argv[])
          sout << "solution\n" << *pmesh << *u;
          sout << "pause\n";
          sout << flush;
-         if (myid == 0)
+         if (Mpi::Root())
+         {
             cout << "GLVis visualization paused."
                  << " Press space (in the GLVis window) to resume it.\n";
+         }
       }
    }
 
    // 9. Define the time-dependent evolution operator describing the ODE
    //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(*m, *k, *B);
+   FE_Evolution adv(*m, *k, *B, prec_type);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -511,7 +599,7 @@ int main(int argc, char *argv[])
 
       if (done || ti % vis_steps == 0)
       {
-         if (myid == 0)
+         if (Mpi::Root())
          {
             cout << "time step: " << ti << ", time: " << t << endl;
             if (cvode) { cvode->PrintInfo(); }
@@ -590,7 +678,7 @@ int main(int argc, char *argv[])
 
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
-                           const Vector &b_)
+                           const Vector &b_, PrecType prec_type)
    : TimeDependentOperator(M_.Height()),
      b(b_),
      M_solver(M_.ParFESpace()->GetComm()),
@@ -617,7 +705,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
       HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
       M_prec = hypre_prec;
 
-      dg_solver = new DG_Solver(M_mat, K_mat, *M_.FESpace());
+      dg_solver = new DG_Solver(M_mat, K_mat, *M_.FESpace(), prec_type);
    }
    else
    {
@@ -633,6 +721,10 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
    M_solver.SetPrintLevel(0);
 }
 
+// Solve the equation:
+//    u_t = M^{-1}(Ku + b),
+// by solving associated linear system
+//    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
 {
    K->Mult(x, z);

examples/sundials/makefile

@@ -23,6 +23,8 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES = ex9 ex10 ex16
 PAR_EXAMPLES = ex9p ex10p ex16p
+SEQ_DEVICE_EXAMPLES = ex9
+PAR_DEVICE_EXAMPLES = ex9p
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -54,10 +56,22 @@ include $(MFEM_TEST_MK)
 RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 SERIAL_NAME := Serial SUNDIALS example
 PARALLEL_NAME := Parallel SUNDIALS example
+SERIAL_CUDA_NAME := Serial SUNDIALS CUDA example
+PARALLEL_CUDA_NAME := Parallel SUNDIALS CUDA example
+SERIAL_HIP_NAME := Serial SUNDIALS HIP example
+PARALLEL_HIP_NAME := Parallel SUNDIALS HIP example
 %-test-par: %
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
 %-test-seq: %
 	@$(call mfem-test,$<,, $(SERIAL_NAME))
+%-test-par-cuda: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_CUDA_NAME),-d cuda)
+%-test-seq-cuda: %
+	@$(call mfem-test,$<,, $(SERIAL_CUDA_NAME),-d cuda)
+%-test-par-hip: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_HIP_NAME),-d hip)
+%-test-seq-hip: %
+	@$(call mfem-test,$<,, $(SERIAL_HIP_NAME),-d hip)
 
 # Testing: Specific execution options:
 # Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
@@ -68,6 +82,16 @@ ex9-test-seq: ex9
 	@$(call mfem-test,$<,, $(SERIAL_NAME),$(EX9_ARGS))
 ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX9P_ARGS))
+ex9-test-seq-cuda: ex9
+	@$(call mfem-test,$<,, $(SERIAL_CUDA_NAME),-d cuda $(EX9_ARGS))
+ex9p-test-par-cuda: ex9p
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_CUDA_NAME),-d cuda \
+	   $(EX9P_ARGS))
+ex9-test-seq-hip: ex9
+	@$(call mfem-test,$<,, $(SERIAL_HIP_NAME),-d hip $(EX9_ARGS))
+ex9p-test-par-hip: ex9p
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_HIP_NAME),-d hip \
+	   $(EX9P_ARGS))
 # Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
 EX10_COMMON_ARGS := -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10
 EX10_ARGS  := $(EX10_COMMON_ARGS) -r  2

fem/bilinearform.cpp

@@ -14,7 +14,6 @@
 #include "fem.hpp"
 #include "../general/device.hpp"
 #include <cmath>
-#include <cstddef>
 
 namespace mfem
 {
@@ -110,9 +109,6 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
    boundary_face_integs = bf->boundary_face_integs;
    boundary_face_integs_marker = bf->boundary_face_integs_marker;
 
-   internal_boundary_face_integs = bf->internal_boundary_face_integs;
-   internal_boundary_face_integs_marker = bf->internal_boundary_face_integs_marker;
-
    AllocMat();
 }
 
@@ -282,22 +278,6 @@ void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
    boundary_face_integs_marker.Append(&bdr_marker);
 }
 
-void BilinearForm::AddInternalBoundaryFaceIntegrator(BilinearFormIntegrator
-                                                     *bfi)
-{
-   internal_boundary_face_integs.Append(bfi);
-   // nullptr -> all attributes are active
-   internal_boundary_face_integs_marker.Append(nullptr);
-}
-
-void BilinearForm::AddInternalBoundaryFaceIntegrator(BilinearFormIntegrator
-                                                     *bfi,
-                                                     Array<int> &internal_bdr_attr_marker)
-{
-   internal_boundary_face_integs.Append(bfi);
-   internal_boundary_face_integs_marker.Append(&internal_bdr_attr_marker);
-}
-
 void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 {
    if (element_matrices)
@@ -650,59 +630,6 @@ void BilinearForm::Assemble(int skip_zeros)
       }
    }
 
-   if (internal_boundary_face_integs.Size())
-   {
-      // Which internal boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < internal_boundary_face_integs.Size(); k++)
-      {
-         if (internal_boundary_face_integs_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         auto &bdr_marker = *internal_boundary_face_integs_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for internal boundary face "
-                     "integrator #" << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      Array<int> vdofs2;
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         auto *tr = mesh->GetInternalBdrFaceTransformations(i);
-         if (tr != nullptr)
-         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            fes->GetElementVDofs(tr->Elem2No, vdofs2);
-            vdofs.Append(vdofs2);
-            const auto *fe1 = fes->GetFE(tr->Elem1No);
-            const auto *fe2 = fes->GetFE(tr->Elem2No);
-            for (int k = 0; k < internal_boundary_face_integs.Size(); k++)
-            {
-               if (internal_boundary_face_integs_marker[k] &&
-                   (*internal_boundary_face_integs_marker[k])[bdr_attr - 1] == 0)
-               {
-                  continue;
-               }
-
-               internal_boundary_face_integs[k]->AssembleFaceMatrix(
-                  *fe1, *fe2, *tr, elemmat);
-               mat->AddSubMatrix(vdofs, vdofs, elemmat, skip_zeros);
-            }
-         }
-      }
-   }
-
 #ifdef MFEM_USE_LEGACY_OPENMP
    if (free_element_matrices)
    {
@@ -1216,10 +1143,6 @@ BilinearForm::~BilinearForm()
       { delete interior_face_integs[k]; }
       for (k=0; k < boundary_face_integs.Size(); k++)
       { delete boundary_face_integs[k]; }
-      for (int i = 0; i < internal_boundary_face_integs.Size(); i++)
-      {
-         delete internal_boundary_face_integs[i];
-      }
    }
 
    delete ext;

fem/bilinearform.hpp

@@ -113,10 +113,6 @@ protected:
    Array<BilinearFormIntegrator*> boundary_face_integs;
    Array<Array<int>*> boundary_face_integs_marker; ///< Entries are not owned.
 
-   /// Set of internal boundary face integrators to be applied.
-   Array<BilinearFormIntegrator*> internal_boundary_face_integs;
-   Array<Array<int>*> internal_boundary_face_integs_marker; ///< Entries not owned.
-
    DenseMatrix elemmat;
    Array<int>  vdofs;
 
@@ -420,18 +416,6 @@ public:
    void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                              Array<int> &bdr_marker);
 
-   /// @brief Add new internal boundary face integrator. Assumes ownership of
-   /// @a bfi.
-   void AddInternalBoundaryFaceIntegrator(BilinearFormIntegrator *bfi);
-
-   /** @brief Add new internal boundary face integrator, restricted to the given
-       boundary attributes.
-
-       Assumes ownership of @a bfi. The array @a internal_bdr_attr_marker is
-       stored internally as a pointer to the given Array<int> object. */
-   void AddInternalBoundaryFaceIntegrator(BilinearFormIntegrator *bfi,
-                                          Array<int> &internal_bdr_attr_marker);
-
    /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
    void operator=(const double a)
    {

fem/ceed/interface/util.hpp

@@ -13,13 +13,13 @@
 #define MFEM_LIBCEED_UTIL
 
 #include "../../../config/config.hpp"
+#include <functional>
+#include <string>
 #include <tuple>
 #include <unordered_map>
-#include <string>
 
 #include "ceed.hpp"
 #ifdef MFEM_USE_CEED
-#include <ceed/hash.h>
 #include <ceed/backend.h>  // for CeedOperatorField
 #endif
 
@@ -105,6 +105,21 @@ const IntegrationRule & GetRule(
 /// Return the path to the libCEED q-function headers.
 const std::string &GetCeedPath();
 
+/// Wrapper for std::hash.
+template <typename T>
+inline std::size_t CeedHash(const T key)
+{
+   return std::hash<T> {}(key);
+}
+
+/// Effective way to combine hashes (from libCEED).
+inline std::size_t CeedHashCombine(std::size_t seed, std::size_t hash)
+{
+   // See https://doi.org/10.1002/asi.10170, or
+   //     https://dl.acm.org/citation.cfm?id=759509.
+   return seed ^ (hash + (seed << 6) + (seed >> 2));
+}
+
 // Hash table for CeedBasis
 using BasisKey = std::tuple<const mfem::FiniteElementSpace*,
       const mfem::IntegrationRule*,
@@ -115,12 +130,12 @@ struct BasisHash
    {
       return CeedHashCombine(
                 CeedHashCombine(
-                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<1>(k)))),
+                   CeedHash(std::get<0>(k)),
+                   CeedHash(std::get<1>(k))),
                 CeedHashCombine(
-                   CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                   CeedHashInt(std::get<3>(k))),
-                   CeedHashInt(std::get<4>(k))));
+                   CeedHashCombine(CeedHash(std::get<2>(k)),
+                                   CeedHash(std::get<3>(k))),
+                   CeedHash(std::get<4>(k))));
    }
 };
 using BasisMap = std::unordered_map<const BasisKey, CeedBasis, BasisHash>;
@@ -137,11 +152,11 @@ struct RestrHash
       return CeedHashCombine(
                 CeedHashCombine(
                    CeedHashCombine(
-                      CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                      CeedHashInt(std::get<1>(k))),
-                   CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                   CeedHashInt(std::get<3>(k)))),
-                CeedHashInt(std::get<4>(k)));
+                      CeedHash(std::get<0>(k)),
+                      CeedHash(std::get<1>(k))),
+                   CeedHashCombine(CeedHash(std::get<2>(k)),
+                                   CeedHash(std::get<3>(k)))),
+                CeedHash(std::get<4>(k)));
    }
 };
 using RestrMap =

fem/doftrans.cpp

@@ -14,54 +14,6 @@
 namespace mfem
 {
 
-void DofTransformation::TransformPrimal(Vector &v) const
-{
-   TransformPrimal(v.GetData());
-}
-
-void DofTransformation::TransformPrimalCols(DenseMatrix &V) const
-{
-   for (int c=0; c<V.Width(); c++)
-   {
-      TransformPrimal(V.GetColumn(c));
-   }
-}
-
-void DofTransformation::TransformDual(Vector &v) const
-{
-   TransformDual(v.GetData());
-}
-
-void DofTransformation::TransformDual(DenseMatrix &V) const
-{
-   TransformDualCols(V);
-   TransformDualRows(V);
-}
-
-void DofTransformation::TransformDualRows(DenseMatrix &V) const
-{
-   Vector row;
-   for (int r=0; r<V.Height(); r++)
-   {
-      V.GetRow(r, row);
-      TransformDual(row);
-      V.SetRow(r, row);
-   }
-}
-
-void DofTransformation::TransformDualCols(DenseMatrix &V) const
-{
-   for (int c=0; c<V.Width(); c++)
-   {
-      TransformDual(V.GetColumn(c));
-   }
-}
-
-void DofTransformation::InvTransformPrimal(Vector &v) const
-{
-   InvTransformPrimal(v.GetData());
-}
-
 void TransformPrimal(const DofTransformation *ran_dof_trans,
                      const DofTransformation *dom_dof_trans,
                      DenseMatrix &elmat)
@@ -85,11 +37,6 @@ void TransformPrimal(const DofTransformation *ran_dof_trans,
    }
 }
 
-void DofTransformation::InvTransformDual(Vector &v) const
-{
-   InvTransformDual(v.GetData());
-}
-
 void TransformDual(const DofTransformation *ran_dof_trans,
                    const DofTransformation *dom_dof_trans,
                    DenseMatrix &elmat)
@@ -113,15 +60,16 @@ void TransformDual(const DofTransformation *ran_dof_trans,
    }
 }
 
-void VDofTransformation::TransformPrimal(double *v) const
+void StatelessVDofTransformation::TransformPrimal(const Array<int> & face_ori,
+                                                  double *v) const
 {
-   int size = doftrans_->Size();
+   int size = sdoftrans_->Size();
 
    if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
    {
       for (int i=0; i<vdim_; i++)
       {
-         doftrans_->TransformPrimal(&v[i*size]);
+         sdoftrans_->TransformPrimal(face_ori, &v[i*size]);
       }
    }
    else
@@ -133,7 +81,7 @@ void VDofTransformation::TransformPrimal(double *v) const
          {
             vec(j) = v[j*vdim_+i];
          }
-         doftrans_->TransformPrimal(vec);
+         sdoftrans_->TransformPrimal(face_ori, vec);
          for (int j=0; j<size; j++)
          {
             v[j*vdim_+i] = vec(j);
@@ -142,15 +90,17 @@ void VDofTransformation::TransformPrimal(double *v) const
    }
 }
 
-void VDofTransformation::InvTransformPrimal(double *v) const
+void StatelessVDofTransformation::InvTransformPrimal(
+   const Array<int> & face_ori,
+   double *v) const
 {
-   int size = doftrans_->Height();
+   int size = sdoftrans_->Height();
 
    if ((Ordering::Type)ordering_ == Ordering::byNODES)
    {
       for (int i=0; i<vdim_; i++)
       {
-         doftrans_->InvTransformPrimal(&v[i*size]);
+         sdoftrans_->InvTransformPrimal(face_ori, &v[i*size]);
       }
    }
    else
@@ -162,7 +112,7 @@ void VDofTransformation::InvTransformPrimal(double *v) const
          {
             vec(j) = v[j*vdim_+i];
          }
-         doftrans_->InvTransformPrimal(vec);
+         sdoftrans_->InvTransformPrimal(face_ori, vec);
          for (int j=0; j<size; j++)
          {
             v[j*vdim_+i] = vec(j);
@@ -171,15 +121,16 @@ void VDofTransformation::InvTransformPrimal(double *v) const
    }
 }
 
-void VDofTransformation::TransformDual(double *v) const
+void StatelessVDofTransformation::TransformDual(const Array<int> & face_ori,
+                                                double *v) const
 {
-   int size = doftrans_->Size();
+   int size = sdoftrans_->Size();
 
    if ((Ordering::Type)ordering_ == Ordering::byNODES)
    {
       for (int i=0; i<vdim_; i++)
       {
-         doftrans_->TransformDual(&v[i*size]);
+         sdoftrans_->TransformDual(face_ori, &v[i*size]);
       }
    }
    else
@@ -191,7 +142,7 @@ void VDofTransformation::TransformDual(double *v) const
          {
             vec(j) = v[j*vdim_+i];
          }
-         doftrans_->TransformDual(vec);
+         sdoftrans_->TransformDual(face_ori, vec);
          for (int j=0; j<size; j++)
          {
             v[j*vdim_+i] = vec(j);
@@ -200,15 +151,16 @@ void VDofTransformation::TransformDual(double *v) const
    }
 }
 
-void VDofTransformation::InvTransformDual(double *v) const
+void StatelessVDofTransformation::InvTransformDual(const Array<int> & face_ori,
+                                                   double *v) const
 {
-   int size = doftrans_->Size();
+   int size = sdoftrans_->Size();
 
    if ((Ordering::Type)ordering_ == Ordering::byNODES)
    {
       for (int i=0; i<vdim_; i++)
       {
-         doftrans_->InvTransformDual(&v[i*size]);
+         sdoftrans_->InvTransformDual(face_ori, &v[i*size]);
       }
    }
    else
@@ -220,7 +172,7 @@ void VDofTransformation::InvTransformDual(double *v) const
          {
             vec(j) = v[j*vdim_+i];
          }
-         doftrans_->InvTransformDual(vec);
+         sdoftrans_->InvTransformDual(face_ori, vec);
          for (int j=0; j<size; j++)
          {
             v[j*vdim_+i] = vec(j);
@@ -229,7 +181,8 @@ void VDofTransformation::InvTransformDual(double *v) const
    }
 }
 
-const double ND_DofTransformation::T_data[24] =
+// ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
+const double ND_StatelessDofTransformation::T_data[24] =
 {
    1.0,  0.0,  0.0,  1.0,
    -1.0, -1.0,  0.0,  1.0,
@@ -239,10 +192,11 @@ const double ND_DofTransformation::T_data[24] =
    0.0,  1.0,  1.0,  0.0
 };
 
-const DenseTensor ND_DofTransformation
-::T(const_cast<double*>(ND_DofTransformation::T_data), 2, 2, 6);
+const DenseTensor ND_StatelessDofTransformation
+::T(const_cast<double*>(ND_StatelessDofTransformation::T_data), 2, 2, 6);
 
-const double ND_DofTransformation::TInv_data[24] =
+// ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
+const double ND_StatelessDofTransformation::TInv_data[24] =
 {
    1.0,  0.0,  0.0,  1.0,
    -1.0, -1.0,  0.0,  1.0,
@@ -252,301 +206,113 @@ const double ND_DofTransformation::TInv_data[24] =
    0.0,  1.0,  1.0,  0.0
 };
 
-const DenseTensor ND_DofTransformation
+const DenseTensor ND_StatelessDofTransformation
 ::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
 
-ND_DofTransformation::ND_DofTransformation(int size, int p)
-   : DofTransformation(size)
+ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
+                                                             int num_edges,
+                                                             int num_tri_faces)
+   : StatelessDofTransformation(size)
    , order(p)
    , nedofs(p)
    , nfdofs(p*(p-1))
+   , nedges(num_edges)
+   , nfaces(num_tri_faces)
 {
 }
 
-ND_TriDofTransformation::ND_TriDofTransformation(int p)
-   : ND_DofTransformation(p*(p + 2), p)
-{
-}
-
-void ND_TriDofTransformation::TransformPrimal(double *v) const
+void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
+                                                    double *v) const
 {
    // Return immediately when no face DoFs are present
    if (nfdofs < 2) { return; }
 
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   MFEM_VERIFY(Fo.Size() >= nfaces,
+               "Face orientation array is shorter than the number of faces in "
+               "ND_StatelessDofTransformation");
 
    double data[2];
    Vector v2(data, 2);
 
    // Transform face DoFs
-   for (int f=0; f<1; f++)
+   for (int f=0; f<nfaces; f++)
    {
       for (int i=0; i<nfdofs/2; i++)
       {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
       }
    }
 }
 
-void
-ND_TriDofTransformation::InvTransformPrimal(double *v) const
+void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
+                                                       double *v) const
 {
    // Return immediately when no face DoFs are present
    if (nfdofs < 2) { return; }
 
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   MFEM_VERIFY(Fo.Size() >= nfaces,
+               "Face orientation array is shorter than the number of faces in "
+               "ND_StatelessDofTransformation");
 
    double data[2];
    Vector v2(data, 2);
 
    // Transform face DoFs
-   for (int f=0; f<1; f++)
+   for (int f=0; f<nfaces; f++)
    {
       for (int i=0; i<nfdofs/2; i++)
       {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
       }
    }
 }
 
-void
-ND_TriDofTransformation::TransformDual(double *v) const
+void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
+                                                  double *v) const
 {
    // Return immediately when no face DoFs are present
    if (nfdofs < 2) { return; }
 
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   MFEM_VERIFY(Fo.Size() >= nfaces,
+               "Face orientation array is shorter than the number of faces in "
+               "ND_StatelessDofTransformation");
 
    double data[2];
    Vector v2(data, 2);
 
    // Transform face DoFs
-   for (int f=0; f<1; f++)
+   for (int f=0; f<nfaces; f++)
    {
       for (int i=0; i<nfdofs/2; i++)
       {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
       }
    }
 }
 
-void
-ND_TriDofTransformation::InvTransformDual(double *v) const
+void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
+                                                     double *v) const
 {
    // Return immediately when no face DoFs are present
    if (nfdofs < 2) { return; }
 
-   MFEM_VERIFY(Fo.Size() >= 1,
-               "Face orientations are unset in ND_TriDofTransformation");
+   MFEM_VERIFY(Fo.Size() >= nfaces,
+               "Face orientation array is shorter than the number of faces in "
+               "ND_StatelessDofTransformation");
 
    double data[2];
    Vector v2(data, 2);
 
    // Transform face DoFs
-   for (int f=0; f<1; f++)
+   for (int f=0; f<nfaces; f++)
    {
       for (int i=0; i<nfdofs/2; i++)
       {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-ND_TetDofTransformation::ND_TetDofTransformation(int p)
-   : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
-{
-}
-
-void ND_TetDofTransformation::TransformPrimal(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::InvTransformPrimal(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::TransformDual(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::InvTransformDual(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 4,
-               "Face orientations are unset in ND_TetDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-ND_WedgeDofTransformation::ND_WedgeDofTransformation(int p)
-   : ND_DofTransformation(3 * p * ((p + 1) * (p + 2))/2, p)
-{
-}
-
-void ND_WedgeDofTransformation::TransformPrimal(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 2,
-               "Face orientations are unset in ND_WedgeDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform triangular face DoFs
-   for (int f=0; f<2; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[9*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[9*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_WedgeDofTransformation::InvTransformPrimal(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 2,
-               "Face orientations are unset in ND_WedgeDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform triangular face DoFs
-   for (int f=0; f<2; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[9*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[9*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_WedgeDofTransformation::TransformDual(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 2,
-               "Face orientations are unset in ND_WedgeDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform triangular face DoFs
-   for (int f=0; f<2; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[9*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[9*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_WedgeDofTransformation::InvTransformDual(double *v) const
-{
-   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
-
-   MFEM_VERIFY(Fo.Size() >= 2,
-               "Face orientations are unset in ND_WedgeDofTransformation");
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform triangular face DoFs
-   for (int f=0; f<2; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[9*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[9*nedofs + f*nfdofs + 2*i]);
+         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
       }
    }
 }

fem/doftrans.hpp

@@ -15,19 +15,31 @@
 #include "../config/config.hpp"
 #include "../linalg/linalg.hpp"
 #include "intrules.hpp"
-#include "fe.hpp"
 
 namespace mfem
 {
 
-/** The DofTransformation class is an abstract base class for a family of
-    transformations that map local degrees of freedom (DoFs), contained within
-    individual elements, to global degrees of freedom, stored within
-    GridFunction objects. These transformations are necessary to ensure that
-    basis functions in neighboring elements align correctly. Closely related but
+/** The StatelessDofTransformation class is an abstract base class for a family
+    of transformations that map local degrees of freedom (DoFs), contained
+    within individual elements, to global degrees of freedom, stored within
+    GridFunction objects.
+
+    In this context "stateless" means that the concrete classes derived from
+    StatelessDofTransformation do not store information about the relative
+    orientations of the faces with respect to their neighboring elements. In
+    other words there is no information specific to a particular element (aside
+    from the element type e.g. tetrahedron, wedge, or pyramid). The
+    StatelessDofTransformation provides access to the transformation operators
+    for specific relative face orientations. These are useful, for example, when
+    relating DoFs associated with distinct overlapping meshes such as parent and
+    sub-meshes.
+
+    These transformations are necessary to ensure that basis functions in
+    neighboring (or overlapping) elements align correctly. Closely related but
     complementary transformations are required for the entries stored in
-    LinearForm and BilinearForm objects. The DofTransformation class is designed
-    to apply the action of both of these types of DoF transformations.
+    LinearForm and BilinearForm objects. The StatelessDofTransformation class
+    is designed to apply the action of both of these types of DoF
+    transformations.
 
     Let the "primal transformation" be given by the operator T. This means that
     given a local element vector v the data that must be placed into a
@@ -53,24 +65,84 @@ namespace mfem
     D_t = T * D * T^{-1}. This can be accomplished by using a primal
     transformation on the columns of D and a dual transformation on its rows.
 */
-class DofTransformation
+class StatelessDofTransformation
 {
 protected:
    int size_;
 
-   Array<int> Fo;
-
-   DofTransformation(int size)
+   StatelessDofTransformation(int size)
       : size_(size) {}
 
 public:
-
    inline int Size() const { return size_; }
    inline int Height() const { return size_; }
    inline int NumRows() const { return size_; }
    inline int Width() const { return size_; }
    inline int NumCols() const { return size_; }
 
+   /** Transform local DoFs to align with the global DoFs. For example, this
+       transformation can be used to map the local vector computed by
+       FiniteElement::Project() to the transformed vector stored within a
+       GridFunction object. */
+   virtual void TransformPrimal(const Array<int> & face_orientation,
+                                double *v) const = 0;
+   inline void TransformPrimal(const Array<int> & face_orientation,
+                               Vector &v) const
+   { TransformPrimal(face_orientation, v.GetData()); }
+
+   /** Inverse transform local DoFs. Used to transform DoFs from a global vector
+       back to their element-local form. For example, this must be used to
+       transform the vector obtained using GridFunction::GetSubVector before it
+       can be used to compute a local interpolation.
+   */
+   virtual void InvTransformPrimal(const Array<int> & face_orientation,
+                                   double *v) const = 0;
+   inline void InvTransformPrimal(const Array<int> & face_orientation,
+                                  Vector &v) const
+   { InvTransformPrimal(face_orientation, v.GetData()); }
+
+   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
+       into a LinearForm object. */
+   virtual void TransformDual(const Array<int> & face_orientation,
+                              double *v) const = 0;
+   inline void TransformDual(const Array<int> & face_orientation,
+                             Vector &v) const
+   { TransformDual(face_orientation, v.GetData()); }
+
+   /** Inverse Transform dual DoFs */
+   virtual void InvTransformDual(const Array<int> & face_orientation,
+                                 double *v) const = 0;
+   inline void InvTransformDual(const Array<int> & face_orientation,
+                                Vector &v) const
+   { InvTransformDual(face_orientation, v.GetData()); }
+};
+
+/** The DofTransformation class is an extension of the
+    StatelessDofTransformation which stores the face orientations used to
+    select the necessary transformations which allows it to offer a collection
+    of convenience methods.
+
+    DofTransformation objects are provided by the FiniteElementSpace which has
+    access to the mesh and can therefore provide the face orientations. This is
+    convenient when working with GridFunction, LinearForm, or BilinearForm
+    obejcts or their parallel counterparts.
+
+    StatelessDofTransformation objects are provided by FiniteElement or
+    FiniteElementCollection objects which do not have access to face
+    orientation information. This can be useful in non-standard contexts such as
+    transferring finite element degrees of freedom between different meshes.
+    For examples of its use see the TransferMap used by the SubMesh class.
+   */
+class DofTransformation : virtual public StatelessDofTransformation
+{
+protected:
+   Array<int> Fo;
+
+   DofTransformation(int size)
+      : StatelessDofTransformation(size) {}
+
+public:
+
    /** @brief Configure the transformation using face orientations for the
        current element. */
    /// The face_orientation array can be obtained from Mesh::GetElementFaces.
@@ -79,42 +151,82 @@ public:
 
    inline const Array<int> & GetFaceOrientations() const { return Fo; }
 
+   using StatelessDofTransformation::TransformPrimal;
+   using StatelessDofTransformation::InvTransformPrimal;
+   using StatelessDofTransformation::TransformDual;
+   using StatelessDofTransformation::InvTransformDual;
+
    /** Transform local DoFs to align with the global DoFs. For example, this
        transformation can be used to map the local vector computed by
        FiniteElement::Project() to the transformed vector stored within a
        GridFunction object. */
-   virtual void TransformPrimal(double *v) const = 0;
-   virtual void TransformPrimal(Vector &v) const;
+   inline void TransformPrimal(double *v) const
+   { TransformPrimal(Fo, v); }
+   inline void TransformPrimal(Vector &v) const
+   { TransformPrimal(v.GetData()); }
 
    /// Transform groups of DoFs stored as dense matrices
-   virtual void TransformPrimalCols(DenseMatrix &V) const;
+   inline void TransformPrimalCols(DenseMatrix &V) const
+   {
+      for (int c=0; c<V.Width(); c++)
+      {
+         TransformPrimal(V.GetColumn(c));
+      }
+   }
 
    /** Inverse transform local DoFs. Used to transform DoFs from a global vector
        back to their element-local form. For example, this must be used to
        transform the vector obtained using GridFunction::GetSubVector before it
        can be used to compute a local interpolation.
    */
-   virtual void InvTransformPrimal(double *v) const = 0;
-   virtual void InvTransformPrimal(Vector &v) const;
+   inline void InvTransformPrimal(double *v) const
+   { InvTransformPrimal(Fo, v); }
+   inline void InvTransformPrimal(Vector &v) const
+   { InvTransformPrimal(v.GetData()); }
 
    /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
        into a LinearForm object. */
-   virtual void TransformDual(double *v) const = 0;
-   virtual void TransformDual(Vector &v) const;
+   inline void TransformDual(double *v) const
+   { TransformDual(Fo, v); }
+   inline void TransformDual(Vector &v) const
+   { TransformDual(v.GetData()); }
 
    /** Inverse Transform dual DoFs */
-   virtual void InvTransformDual(double *v) const = 0;
-   virtual void InvTransformDual(Vector &v) const;
+   inline void InvTransformDual(double *v) const
+   { InvTransformDual(Fo, v); }
+   inline void InvTransformDual(Vector &v) const
+   { InvTransformDual(v.GetData()); }
 
    /** Transform a matrix of dual DoFs entries as computed by a
        BilinearFormIntegrator before summing into a BilinearForm object. */
-   virtual void TransformDual(DenseMatrix &V) const;
+   inline void TransformDual(DenseMatrix &V) const
+   {
+      TransformDualCols(V);
+      TransformDualRows(V);
+   }
 
-   /// Transform groups of dual DoFs stored as dense matrices
-   virtual void TransformDualRows(DenseMatrix &V) const;
-   virtual void TransformDualCols(DenseMatrix &V) const;
+   /// Transform rows of a dense matrix containing dual DoFs
+   inline void TransformDualRows(DenseMatrix &V) const
+   {
+      Vector row;
+      for (int r=0; r<V.Height(); r++)
+      {
+         V.GetRow(r, row);
+         TransformDual(row);
+         V.SetRow(r, row);
+      }
+   }
 
-   virtual ~DofTransformation() {}
+   /// Transform columns of a dense matrix containing dual DoFs
+   inline void TransformDualCols(DenseMatrix &V) const
+   {
+      for (int c=0; c<V.Width(); c++)
+      {
+         TransformDual(V.GetColumn(c));
+      }
+   }
+
+   virtual ~DofTransformation() = default;
 };
 
 /** Transform a matrix of DoFs entries from different finite element spaces as
@@ -133,66 +245,143 @@ void TransformDual(const DofTransformation *ran_dof_trans,
                    const DofTransformation *dom_dof_trans,
                    DenseMatrix &elmat);
 
-/** The VDofTransformation class implements a nested transformation where an
-    arbitrary DofTransformation is replicated with a vdim >= 1.
+/** The StatelessVDofTransformation class implements a nested transformation
+    where an arbitrary StatelessDofTransformation is replicated with a
+    vdim >= 1.
 */
-class VDofTransformation : public DofTransformation
+class StatelessVDofTransformation : virtual public StatelessDofTransformation
 {
-private:
+protected:
    int vdim_;
    int ordering_;
-   DofTransformation * doftrans_;
+   StatelessDofTransformation * sdoftrans_;
 
 public:
    /** @brief Default constructor which requires that SetDofTransformation be
        called before use. */
-   VDofTransformation(int vdim = 1, int ordering = 0)
-      : DofTransformation(0),
-        vdim_(vdim), ordering_(ordering),
-        doftrans_(NULL) {}
+   StatelessVDofTransformation(int vdim = 1, int ordering = 0)
+      : StatelessDofTransformation(0)
+      , vdim_(vdim)
+      , ordering_(ordering)
+      , sdoftrans_(NULL)
+   {}
 
-   /// Constructor with a known DofTransformation
-   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
-                      int ordering = 0)
-      : DofTransformation(vdim * doftrans.Size()),
-        vdim_(vdim), ordering_(ordering),
-        doftrans_(&doftrans) {}
+   /// Constructor with a known StatelessDofTransformation
+   StatelessVDofTransformation(StatelessDofTransformation & doftrans,
+                               int vdim = 1,
+                               int ordering = 0)
+      : StatelessDofTransformation(vdim * doftrans.Size())
+      , vdim_(vdim)
+      , ordering_(ordering)
+      , sdoftrans_(&doftrans)
+   {}
 
    /// Set or change the vdim parameter
    inline void SetVDim(int vdim)
    {
       vdim_ = vdim;
-      if (doftrans_)
+      if (sdoftrans_)
       {
-         size_ = vdim_ * doftrans_->Size();
+         size_ = vdim_ * sdoftrans_->Size();
       }
    }
 
    /// Return the current vdim value
    inline int GetVDim() const { return vdim_; }
 
-   /// Set or change the nested DofTransformation object
-   inline void SetDofTransformation(DofTransformation & doftrans)
+   /// Set or change the nested StatelessDofTransformation object
+   inline void SetDofTransformation(StatelessDofTransformation & doftrans)
    {
       size_ = vdim_ * doftrans.Size();
+      sdoftrans_ = &doftrans;
+   }
+
+   /// Return the nested StatelessDofTransformation object
+   inline StatelessDofTransformation * GetDofTransformation() const
+   { return sdoftrans_; }
+
+   using StatelessDofTransformation::TransformPrimal;
+   using StatelessDofTransformation::InvTransformPrimal;
+   using StatelessDofTransformation::TransformDual;
+   using StatelessDofTransformation::InvTransformDual;
+
+   /** Specializations of these base class methods which account for the vdim
+       and ordering of the full set of DoFs.
+   */
+   void TransformPrimal(const Array<int> & face_ori, double *v) const;
+   void InvTransformPrimal(const Array<int> & face_ori, double *v) const;
+   void TransformDual(const Array<int> & face_ori, double *v) const;
+   void InvTransformDual(const Array<int> & face_ori, double *v) const;
+};
+
+/** The VDofTransformation class implements a nested transformation where an
+    arbitrary DofTransformation is replicated with a vdim >= 1.
+*/
+class VDofTransformation : public StatelessVDofTransformation,
+   public DofTransformation
+{
+protected:
+   DofTransformation * doftrans_;
+
+public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   VDofTransformation(int vdim = 1, int ordering = 0)
+      : StatelessDofTransformation(0)
+      , StatelessVDofTransformation(vdim, ordering)
+      , DofTransformation(0)
+      , doftrans_(NULL)
+   {}
+
+   /// Constructor with a known DofTransformation
+   /// @note The face orientations in @a doftrans will be copied into the
+   /// new VDofTransformation object.
+   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
+                      int ordering = 0)
+      : StatelessDofTransformation(vdim * doftrans.Size())
+      , StatelessVDofTransformation(doftrans, vdim, ordering)
+      , DofTransformation(vdim * doftrans.Size())
+      , doftrans_(&doftrans)
+   {
+      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
+   }
+
+   using StatelessVDofTransformation::SetDofTransformation;
+
+   /// Set or change the nested DofTransformation object
+   /// @note The face orientations in @a doftrans will be copied into the
+   /// VDofTransformation object.
+   void SetDofTransformation(DofTransformation & doftrans)
+   {
       doftrans_ = &doftrans;
+      StatelessVDofTransformation::SetDofTransformation(doftrans);
+      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
    }
 
    /// Return the nested DofTransformation object
    inline DofTransformation * GetDofTransformation() const { return doftrans_; }
 
-   inline void SetFaceOrientation(const Array<int> & face_orientation)
-   { Fo = face_orientation; doftrans_->SetFaceOrientations(face_orientation); }
+   /// Set new face orientations in both the VDofTransformation and the
+   /// DofTransformation contained within (if there is one).
+   inline void SetFaceOrientations(const Array<int> & face_orientation)
+   {
+      DofTransformation::SetFaceOrientations(face_orientation);
+      if (doftrans_) { doftrans_->SetFaceOrientations(face_orientation); }
+   }
 
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
    using DofTransformation::InvTransformDual;
 
-   void TransformPrimal(double *v) const;
-   void InvTransformPrimal(double *v) const;
-   void TransformDual(double *v) const;
-   void InvTransformDual(double *v) const;
+   inline void TransformPrimal(double *v) const
+   { TransformPrimal(Fo, v); }
+   inline void InvTransformPrimal(double *v) const
+   { InvTransformPrimal(Fo, v); }
+   inline void TransformDual(double *v) const
+   { TransformDual(Fo, v); }
+   inline void InvTransformDual(double *v) const
+   { InvTransformDual(Fo, v); }
 };
 
 /** Abstract base class for high-order Nedelec spaces on elements with
@@ -207,17 +396,22 @@ public:
     be accessed as DenseMatrices using the GetFaceTransform() and
     GetFaceInverseTransform() methods.
 */
-class ND_DofTransformation : public DofTransformation
+class ND_StatelessDofTransformation : virtual public StatelessDofTransformation
 {
-protected:
+private:
    static const double T_data[24];
    static const double TInv_data[24];
    static const DenseTensor T, TInv;
-   int order;
-   int nedofs; // number of DoFs per edge
-   int nfdofs; // number of DoFs per face
 
-   ND_DofTransformation(int size, int order);
+protected:
+   const int order;  // basis function order
+   const int nedofs; // number of DoFs per edge
+   const int nfdofs; // number of DoFs per face
+   const int nedges; // number of edges per element
+   const int nfaces; // number of triangular faces per element
+
+   ND_StatelessDofTransformation(int size, int order,
+                                 int num_edges, int num_tri_faces);
 
 public:
    // Return the 2x2 transformation operator for the given face orientation
@@ -226,67 +420,117 @@ public:
    // Return the 2x2 inverse transformation operator
    static const DenseMatrix & GetFaceInverseTransform(int ori)
    { return TInv(ori); }
+
+   void TransformPrimal(const Array<int> & face_orientation,
+                        double *v) const;
+
+   void InvTransformPrimal(const Array<int> & face_orientation,
+                           double *v) const;
+
+   void TransformDual(const Array<int> & face_orientation,
+                      double *v) const;
+
+   void InvTransformDual(const Array<int> & face_orientation,
+                         double *v) const;
+};
+
+/// Stateless DoF transformation implementation for the Nedelec basis on
+/// triangles
+class ND_TriStatelessDofTransformation : public ND_StatelessDofTransformation
+{
+public:
+   ND_TriStatelessDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2))
+      , ND_StatelessDofTransformation(order*(order + 2), order, 3, 1)
+   {}
 };
 
 /// DoF transformation implementation for the Nedelec basis on triangles
-class ND_TriDofTransformation : public ND_DofTransformation
+class ND_TriDofTransformation : public DofTransformation,
+   public ND_TriStatelessDofTransformation
 {
 public:
-   ND_TriDofTransformation(int order);
+   ND_TriDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2))
+      , DofTransformation(order*(order + 2))
+      , ND_TriStatelessDofTransformation(order)
+   {}
 
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-
-   void InvTransformDual(double *v) const;
    using DofTransformation::InvTransformDual;
+
+   using ND_TriStatelessDofTransformation::TransformPrimal;
+   using ND_TriStatelessDofTransformation::InvTransformPrimal;
+   using ND_TriStatelessDofTransformation::TransformDual;
+   using ND_TriStatelessDofTransformation::InvTransformDual;
 };
 
 /// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetDofTransformation : public ND_DofTransformation
+class ND_TetStatelessDofTransformation : public ND_StatelessDofTransformation
 {
 public:
-   ND_TetDofTransformation(int order);
+   ND_TetStatelessDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
+      , ND_StatelessDofTransformation(order*(order + 2)*(order + 3)/2, order,
+                                      6, 4)
+   {}
+};
+
+/// DoF transformation implementation for the Nedelec basis on tetrahedra
+class ND_TetDofTransformation : public DofTransformation,
+   public ND_TetStatelessDofTransformation
+{
+public:
+   ND_TetDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
+      , DofTransformation(order*(order + 2)*(order + 3)/2)
+      , ND_TetStatelessDofTransformation(order)
+   {}
 
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
    using DofTransformation::InvTransformDual;
 
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-
-   void InvTransformDual(double *v) const;
+   using ND_TetStatelessDofTransformation::TransformPrimal;
+   using ND_TetStatelessDofTransformation::InvTransformPrimal;
+   using ND_TetStatelessDofTransformation::TransformDual;
+   using ND_TetStatelessDofTransformation::InvTransformDual;
 };
 
 /// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeDofTransformation : public ND_DofTransformation
+class ND_WedgeStatelessDofTransformation : public ND_StatelessDofTransformation
 {
 public:
-   ND_WedgeDofTransformation(int order);
+   ND_WedgeStatelessDofTransformation(int order)
+      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , ND_StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2,
+                                      order, 9, 2)
+   {}
+};
+
+/// DoF transformation implementation for the Nedelec basis on wedge elements
+class ND_WedgeDofTransformation : public DofTransformation,
+   public ND_WedgeStatelessDofTransformation
+{
+public:
+   ND_WedgeDofTransformation(int order)
+      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , DofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , ND_WedgeStatelessDofTransformation(order)
+   {}
 
    using DofTransformation::TransformPrimal;
    using DofTransformation::InvTransformPrimal;
    using DofTransformation::TransformDual;
    using DofTransformation::InvTransformDual;
 
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-
-   void InvTransformDual(double *v) const;
-
+   using ND_WedgeStatelessDofTransformation::TransformPrimal;
+   using ND_WedgeStatelessDofTransformation::InvTransformPrimal;
+   using ND_WedgeStatelessDofTransformation::TransformDual;
+   using ND_WedgeStatelessDofTransformation::InvTransformDual;
 };
 
 } // namespace mfem

fem/fe/fe_base.hpp

@@ -14,6 +14,7 @@
 
 #include "../intrules.hpp"
 #include "../geom.hpp"
+#include "../doftrans.hpp"
 
 #include <map>
 
@@ -576,6 +577,7 @@ public:
    virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                          DofToQuad::Mode mode) const;
 
+
    /** @brief Return the mapping from lexicographic face DOFs to lexicographic
        element DOFs for the given local face @a face_id. */
    /** Given the @a ith DOF (lexicographically ordered) on the face referenced
@@ -590,6 +592,12 @@ public:
        when simplex elements are supported in the future. */
    virtual void GetFaceMap(const int face_id, Array<int> &face_map) const;
 
+   /** @brief Return a DoF transformation object for this particular type of
+       basis.
+   */
+   virtual StatelessDofTransformation * GetDofTransformation() const
+   { return NULL; }
+
    /// Deconstruct the FiniteElement
    virtual ~FiniteElement();
 

fem/fe/fe_nd.cpp

@@ -845,7 +845,7 @@ const double ND_TetrahedronElement::c = 1./4.;
 
 ND_TetrahedronElement::ND_TetrahedronElement(const int p)
    : VectorFiniteElement(3, Geometry::TETRAHEDRON, p*(p + 2)*(p + 3)/2, p,
-                         H_CURL, FunctionSpace::Pk), dof2tk(dof)
+                         H_CURL, FunctionSpace::Pk), dof2tk(dof), doftrans(p)
 {
    const double *eop = poly1d.OpenPoints(p - 1);
    const double *fop = (p > 1) ? poly1d.OpenPoints(p - 2) : NULL;
@@ -1108,7 +1108,7 @@ const double ND_TriangleElement::c = 1./3.;
 ND_TriangleElement::ND_TriangleElement(const int p)
    : VectorFiniteElement(2, Geometry::TRIANGLE, p*(p + 2), p,
                          H_CURL, FunctionSpace::Pk),
-     dof2tk(dof)
+     dof2tk(dof), doftrans(p)
 {
    const double *eop = poly1d.OpenPoints(p - 1);
    const double *iop = (p > 1) ? poly1d.OpenPoints(p - 2) : NULL;
@@ -1302,6 +1302,7 @@ ND_WedgeElement::ND_WedgeElement(const int p,
      dof2tk(dof),
      t_dof(dof),
      s_dof(dof),
+     doftrans(p),
      H1TriangleFE(p, cb_type),
      NDTriangleFE(p),
      H1SegmentFE(p, cb_type),

fem/fe/fe_nd.hpp

@@ -179,6 +179,8 @@ class ND_TetrahedronElement : public VectorFiniteElement
    Array<int> dof2tk;
    DenseMatrixInverse Ti;
 
+   mutable ND_TetStatelessDofTransformation doftrans;
+
 public:
    /// Construct the ND_TetrahedronElement of order @a p
    ND_TetrahedronElement(const int p);
@@ -199,6 +201,8 @@ public:
                                   ElementTransformation &Trans,
                                   DenseMatrix &I) const
    { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+   virtual StatelessDofTransformation * GetDofTransformation() const
+   { return &doftrans; }
    using FiniteElement::Project;
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
@@ -238,6 +242,8 @@ class ND_TriangleElement : public VectorFiniteElement
    Array<int> dof2tk;
    DenseMatrixInverse Ti;
 
+   mutable ND_TriStatelessDofTransformation doftrans;
+
 public:
    /// Construct the ND_TriangleElement of order @a p
    ND_TriangleElement(const int p);
@@ -258,6 +264,8 @@ public:
                                   ElementTransformation &Trans,
                                   DenseMatrix &I) const
    { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
+   virtual StatelessDofTransformation * GetDofTransformation() const
+   { return &doftrans; }
    using FiniteElement::Project;
    virtual void Project(VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const
@@ -338,6 +346,8 @@ private:
 #endif
    Array<int> dof2tk, t_dof, s_dof;
 
+   mutable ND_WedgeStatelessDofTransformation doftrans;
+
    H1_TriangleElement H1TriangleFE;
    ND_TriangleElement NDTriangleFE;
    H1_SegmentElement  H1SegmentFE;
@@ -369,6 +379,9 @@ public:
                                   DenseMatrix &I) const
    { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
 
+   virtual StatelessDofTransformation * GetDofTransformation() const
+   { return &doftrans; }
+
    using FiniteElement::Project;
 
    virtual void Project(VectorCoefficient &vc,

fem/fe_coll.cpp

@@ -2886,6 +2886,19 @@ ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
    }
 }
 
+StatelessDofTransformation *
+ND_FECollection::DofTransformationForGeometry(Geometry::Type GeomType) const
+{
+   if (!Geometry::IsTensorProduct(GeomType) && this->GetOrder() > 1)
+   {
+      return FiniteElementForGeometry(GeomType)->GetDofTransformation();
+   }
+   else
+   {
+      return NULL;
+   }
+}
+
 const int *ND_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                    int Or) const
 {

fem/fe_coll.hpp

@@ -61,6 +61,13 @@ public:
 
    virtual int DofForGeometry(Geometry::Type GeomType) const = 0;
 
+   /** @brief Returns a DoF transformation object compatible with this basis
+       and geometry type.
+   */
+   virtual StatelessDofTransformation *
+   DofTransformationForGeometry(Geometry::Type GeomType) const
+   { return NULL; }
+
    /** @brief Returns an array, say p, that maps a local permuted index i to a
        local base index: base_i = p[i].
 
@@ -466,8 +473,12 @@ public:
    virtual int DofForGeometry(Geometry::Type GeomType) const
    { return ND_dof[GeomType]; }
 
+   virtual StatelessDofTransformation *
+   DofTransformationForGeometry(Geometry::Type GeomType) const;
+
    virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
                                              int Or) const;
+
    virtual const char *Name() const { return nd_name; }
    virtual int GetContType() const { return TANGENTIAL; }
    FiniteElementCollection *GetTraceCollection() const;

fem/fespace.hpp

@@ -377,17 +377,6 @@ protected:
    /// Return number of possible DOF variants for edge/face (var. order spaces).
    int GetNVariants(int entity, int index) const;
 
-   /// Helper to encode a sign flip into a DOF index (for Hcurl/Hdiv shapes).
-   static inline int EncodeDof(int entity_base, int idx)
-   { return (idx >= 0) ? (entity_base + idx) : (-1-(entity_base + (-1-idx))); }
-
-   /// Helpers to remove encoded sign from a DOF
-   static inline int DecodeDof(int dof)
-   { return (dof >= 0) ? dof : (-1 - dof); }
-
-   static inline int DecodeDof(int dof, double& sign)
-   { return (dof >= 0) ? (sign = 1, dof) : (sign = -1, (-1 - dof)); }
-
    /// Helper to get vertex, edge or face DOFs (entity=0,1,2 resp.).
    int GetEntityDofs(int entity, int index, Array<int> &dofs,
                      Geometry::Type master_geom = Geometry::INVALID,
@@ -985,6 +974,18 @@ public:
    /// well on sets of @ref ldof "Local Dofs".
    static void AdjustVDofs(Array<int> &vdofs);
 
+   /// Helper to encode a sign flip into a DOF index (for Hcurl/Hdiv shapes).
+   static inline int EncodeDof(int entity_base, int idx)
+   { return (idx >= 0) ? (entity_base + idx) : (-1-(entity_base + (-1-idx))); }
+
+   /// Helper to return the DOF associated with a sign encoded DOF
+   static inline int DecodeDof(int dof)
+   { return (dof >= 0) ? dof : (-1 - dof); }
+
+   /// Helper to determine the DOF and sign of a sign encoded DOF
+   static inline int DecodeDof(int dof, double& sign)
+   { return (dof >= 0) ? (sign = 1, dof) : (sign = -1, (-1 - dof)); }
+
    /// @anchor getvdof @name Local Vector DoF Access Members
    /// These member functions produce arrays of local vector degree of freedom
    /// indices, see @ref ldof and @ref vdof. These indices can be used to

fem/gslib.cpp

@@ -1308,6 +1308,298 @@ void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
    Interpolate(field_in, field_out);
 }
 
+#ifdef MFEM_USE_MPI
+
+GSLIBCommunicator::GSLIBCommunicator(MPI_Comm comm_)
+   : cr(NULL), gsl_comm(NULL)
+{
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
+   comm_init(gsl_comm, comm_);
+   crystal_init(cr, gsl_comm);
+}
+
+void GSLIBCommunicator::SendData(int dim, const Array<unsigned int> & gsl_proc,
+                                 const Array<unsigned int> & elem_send,
+                                 const Vector &ref_send,
+                                 const Vector &coords_send,
+                                 const Array<int> &s_conn_send,
+                                 Array<unsigned int> & proc_recv,
+                                 Array<unsigned int> & index_recv,
+                                 Array<unsigned int> & elem_recv,
+                                 Vector &ref_recv,
+                                 Vector &coords_recv,
+                                 Array<int> &s_conn_recv)
+{
+   int nptsend = gsl_proc.Size();
+   int nptElem   = elem_send.Size();
+   int nptRST    = ref_send.Size();
+
+   MFEM_VERIFY(nptElem == nptsend,
+               "Incompatible Elem size.");
+   MFEM_VERIFY(nptsend*dim == nptRST,
+               "Incompatible nptRST size.");
+   MFEM_VERIFY(dim <= 3,
+               "Incompatible dimension.");
+
+   // Pack data to send via crystal router
+   struct gslib::array *outpt = new gslib::array;
+
+   struct out_pt { double rst[3], coords[3]; int s_conn; uint index, elem, proc; };
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->index = index;
+      pt->elem = elem_send[index];
+      pt->proc  = gsl_proc[index];
+      pt->s_conn = s_conn_send[index];
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->rst[d]= ref_send(index*dim + d);
+         pt->coords[d]= coords_send(index + d*nptsend);
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+
+   // unpack
+   int npt = outpt->n;
+   proc_recv.SetSize(npt);
+   elem_recv.SetSize(npt);
+   index_recv.SetSize(npt);
+   ref_recv.SetSize(npt*dim);
+   coords_recv.SetSize(npt*dim);
+   s_conn_recv.SetSize(npt);
+
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      index_recv[index] = pt->index;
+      elem_recv[index] = pt->elem;
+      proc_recv[index] = pt->proc;
+      s_conn_recv[index] = pt->s_conn;
+      for (int d = 0; d < dim; ++d)
+      {
+         ref_recv(index*dim + d)= pt->rst[d]; // by VDIM
+         coords_recv(index + d*npt)= pt->coords[d]; // by NODES
+      }
+      ++pt;
+   }
+
+   array_free(outpt);
+   delete outpt;
+}
+
+void GSLIBCommunicator::SendData2(int dim,
+                                  const Array<unsigned int> & gsl_proc,
+                                  const Vector &xyz_send,
+                                  const Vector &xi_send,
+                                  const Array<int> &s_conn_send,
+                                  const Array<int> &conn_send,
+                                  const DenseMatrix &coords_send,
+                                  Vector &xyz_recv,
+                                  Vector &xi_recv,
+                                  Array<int> &s_conn_recv,
+                                  Array<int> &conn_recv,
+                                  DenseMatrix &coords_recv)
+{
+   int nptsend = gsl_proc.Size();
+
+   struct gslib::array *outpt = new gslib::array;
+   struct out_pt {double xyz[3], xi[2], coords[12]; int s_conn; int conn[4]; uint proc;};
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->proc  = gsl_proc[index];
+      pt->s_conn  = s_conn_send[index];
+      for (int d = 0; d < dim-1; ++d)
+      {
+         pt->xi[d]= xi_send(index*(dim-1) + d);
+      }
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->xyz[d]= xyz_send(index + d*nptsend);
+      }
+      for (int j = 0; j<4; j++)
+      {
+         pt->conn[j] = conn_send[index*4+j];
+         for (int d = 0; d < dim; ++d)
+         {
+            pt->coords[j*dim+d]= coords_send(index*4+j,d);
+         }
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+   // unpack
+   int npt = outpt->n;
+   xi_recv.SetSize(npt*(dim-1));
+   xyz_recv.SetSize(npt*dim);
+   s_conn_recv.SetSize(npt);
+   conn_recv.SetSize(npt*4);
+   coords_recv.SetSize(npt*4,dim);
+
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      s_conn_recv[index] = pt->s_conn;
+      for (int d = 0; d < dim-1; ++d)
+      {
+         xi_recv(index*(dim-1) + d) = pt->xi[d];
+      }
+      for (int d = 0; d < dim; ++d)
+      {
+         xyz_recv(index + d*npt)= pt->xyz[d]; // by NODES
+      }
+      for (int j = 0; j<4; j++)
+      {
+         conn_recv[index*4+j] = pt->conn[j];
+         for (int d = 0; d < dim; ++d)
+         {
+            coords_recv(index*4+j,d) = pt->coords[j*dim+d];
+         }
+      }
+      ++pt;
+   }
+   array_free(outpt);
+   delete outpt;
+}
+
+
+void GSLIBCommunicator::ExchangeNormal(Mesh & mesh,
+                                       const Array<unsigned int> &gsl_proc,
+                                       const Array<unsigned int> &gsl_mfem_elem,
+                                       const Vector &gsl_mfem_ref,
+                                       Vector &recv_normals)
+{
+   int dim = mesh.Dimension();
+   int nptsend = gsl_proc.Size();
+   int nptElem   = gsl_mfem_elem.Size();
+   int nptRST    = gsl_mfem_ref.Size();
+
+   recv_normals.SetSize(nptRST);
+   int nptNormal    = recv_normals.Size();
+
+   MFEM_VERIFY(nptElem == nptsend,
+               "Incompatible Elem size.");
+   MFEM_VERIFY(nptsend*dim == nptRST,
+               "Incompatible nptRST size.");
+   MFEM_VERIFY(dim <= 3,
+               "Incompatible dimension.");
+
+   // Pack data to send via crystal router
+   struct gslib::array *outpt = new gslib::array;
+
+   struct out_pt { double rst[3]; uint index, elem, proc; };
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->index = index;
+      pt->elem = gsl_mfem_elem[index];
+      pt->proc  = gsl_proc[index];
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->rst[d]= gsl_mfem_ref(index*dim + d);
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+
+   // Get normal vector
+   int npt = outpt->n;
+   pt = (struct out_pt *)outpt->ptr;
+   Vector normal(npt*dim);
+   for (int index = 0; index < npt; index++)
+   {
+      IntegrationPoint ip;
+      ip.Set3(&pt->rst[0]);
+      Vector localval(normal.GetData()+index*dim, dim);
+      // get the normal at this integration point here
+      // for now I just put back this proc's rank + the input rst coordinates
+      for (int d = 0; d < dim; d++)
+      {
+         localval(d) = gsl_comm->id + pt->rst[d];
+      }
+      ++pt;
+   }
+
+   // Save index and proc data in a struct
+   struct gslib::array *savpt = new gslib::array;
+   struct sav_pt { uint index, proc; };
+   struct sav_pt *spt;
+   array_init(struct sav_pt, savpt, npt);
+   savpt->n=npt;
+   spt = (struct sav_pt *)savpt->ptr;
+   pt  = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      spt->index = pt->index;
+      spt->proc  = pt->proc;
+      ++pt; ++spt;
+   }
+
+   array_free(outpt);
+   delete outpt;
+
+   // Copy data from save struct to send struct and send component wise
+   struct gslib::array *sendpt = new gslib::array;
+   struct send_pt { double ival; uint index, proc; };
+   struct send_pt *sdpt;
+   for (int j = 0; j < dim; j++)
+   {
+      array_init(struct send_pt, sendpt, npt);
+      sendpt->n=npt;
+      spt  = (struct sav_pt *)savpt->ptr;
+      sdpt = (struct send_pt *)sendpt->ptr;
+      for (int index = 0; index < npt; index++)
+      {
+         sdpt->index = spt->index;
+         sdpt->proc  = spt->proc;
+         sdpt->ival  = normal(j + index*dim);
+         ++sdpt; ++spt;
+      }
+
+      sarray_transfer(struct send_pt, sendpt, proc, 1, cr);
+      sdpt = (struct send_pt *)sendpt->ptr;
+      for (int index = 0; index < static_cast<int>(sendpt->n); index++)
+      {
+         int idx = sdpt->index*dim + j;
+         recv_normals(idx) = sdpt->ival;
+         ++sdpt;
+      }
+      array_free(sendpt);
+   }
+   array_free(savpt);
+   delete sendpt;
+   delete savpt;
+}
+
+void GSLIBCommunicator::FreeData()
+{
+   crystal_free(cr);
+}
+
+GSLIBCommunicator::~GSLIBCommunicator()
+{
+   delete gsl_comm;
+   delete cr;
+}
+#endif
 
 } // namespace mfem
 

fem/gslib.hpp

@@ -290,6 +290,55 @@ public:
    using FindPointsGSLIB::Interpolate;
 };
 
+#ifdef MFEM_USE_MPI
+// Use to send info to certain processes
+class GSLIBCommunicator
+{
+protected:
+   struct gslib::crystal *cr;             // gslib's internal data
+   struct gslib::comm *gsl_comm;          // gslib's internal data
+
+public:
+   GSLIBCommunicator(MPI_Comm comm_);
+
+   virtual ~GSLIBCommunicator();
+
+   void ExchangeNormal(Mesh& mesh,
+                       const Array<unsigned int> &gsl_proc,
+                       const Array<unsigned int> &gsl_mfem_elem,
+                       const Vector &gsl_mfem_ref,
+                       Vector &recv_normals); //npt*dim
+
+   void SendData(int dim,
+                 const Array<unsigned int> & gsl_proc,
+                 const Array<unsigned int> & elem_send,
+                 const Vector &ref_send,
+                 const Vector &coords_send,
+                 const Array<int> &s_conn_send,
+                 Array<unsigned int> & proc_recv,
+                 Array<unsigned int> & index_recv,
+                 Array<unsigned int> & elem_recv,
+                 Vector &ref_recv,
+                 Vector &coords_recv,
+                 Array<int> & s_conn_recv);
+
+   void SendData2(int dim,
+                  const Array<unsigned int> & gsl_proc,
+                  const Vector &xyz_send,
+                  const Vector &xi_send,
+                  const Array<int> &s_conn_send,
+                  const Array<int> &conn_send,
+                  const DenseMatrix &coords_send,
+                  Vector &xyz_recv,
+                  Vector &ref_recv,
+                  Array<int> &s_conn_recv,
+                  Array<int> &conn_recv,
+                  DenseMatrix &coords_recv);
+
+   virtual void FreeData();
+};
+#endif
+
 } // namespace mfem
 
 #endif // MFEM_USE_GSLIB

fem/linearform.cpp

@@ -36,9 +36,6 @@ LinearForm::LinearForm(FiniteElementSpace *f, LinearForm *lf)
 
    boundary_face_integs = lf->boundary_face_integs;
    boundary_face_integs_marker = lf->boundary_face_integs_marker;
-
-   internal_boundary_face_integs = lf->internal_boundary_face_integs;
-   internal_boundary_face_integs_marker = lf->internal_boundary_face_integs_marker;
 }
 
 void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi)
@@ -104,20 +101,6 @@ void LinearForm::AddInteriorFaceIntegrator(LinearFormIntegrator *lfi)
    interior_face_integs.Append(lfi);
 }
 
-void LinearForm::AddInternalBoundaryFaceIntegrator(LinearFormIntegrator *lfi)
-{
-   internal_boundary_face_integs.Append(lfi);
-   // nullptr -> all attributes are active
-   internal_boundary_face_integs_marker.Append(nullptr);
-}
-
-void LinearForm::AddInternalBoundaryFaceIntegrator(LinearFormIntegrator *lfi,
-                                                   Array<int> &internal_bdr_attr_marker)
-{
-   internal_boundary_face_integs.Append(lfi);
-   internal_boundary_face_integs_marker.Append(&internal_bdr_attr_marker);
-}
-
 bool LinearForm::SupportsDevice() const
 {
    // return false for NURBS meshes, so we don’t convert it to non-NURBS
@@ -138,10 +121,7 @@ bool LinearForm::SupportsDevice() const
    if (!IntegratorsSupportDevice(domain_integs)) { return false; }
    if (!IntegratorsSupportDevice(boundary_integs)) { return false; }
    if (boundary_face_integs.Size() > 0 || interior_face_integs.Size() > 0 ||
-       domain_delta_integs.Size() > 0 || internal_boundary_face_integs.Size() > 0)
-   {
-      return false;
-   }
+       domain_delta_integs.Size() > 0) { return false; }
 
    if (boundary_integs.Size() > 0)
    {
@@ -359,61 +339,6 @@ void LinearForm::Assemble()
          }
       }
    }
-
-   if (internal_boundary_face_integs.Size())
-   {
-      auto *mesh = fes->GetMesh();
-
-      // Which internal boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < internal_boundary_face_integs.Size(); k++)
-      {
-         if (internal_boundary_face_integs_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         auto &bdr_marker = *internal_boundary_face_integs_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for internal boundary face "
-                     "integrator #" << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      Array<int> vdofs2;
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         auto *tr = mesh->GetInternalBdrFaceTransformations(i);
-         if (tr != nullptr)
-         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            fes->GetElementVDofs(tr->Elem2No, vdofs2);
-            vdofs.Append(vdofs2);
-            const auto *fe1 = fes->GetFE(tr->Elem1No);
-            const auto *fe2 = fes->GetFE(tr->Elem2No);
-            for (int k = 0; k < internal_boundary_face_integs.Size(); k++)
-            {
-               if (internal_boundary_face_integs_marker[k] &&
-                   (*internal_boundary_face_integs_marker[k])[bdr_attr - 1] == 0)
-               {
-                  continue;
-               }
-
-               internal_boundary_face_integs[k]->AssembleRHSElementVect(
-                  *fe1, *fe2, *tr, elemvect);
-               AddElementVector(vdofs, elemvect);
-            }
-         }
-      }
-   }
 }
 
 void LinearForm::Update()
@@ -504,10 +429,6 @@ LinearForm::~LinearForm()
       { delete boundary_face_integs[k]; }
       for (k=0; k < interior_face_integs.Size(); k++)
       { delete interior_face_integs[k]; }
-      for (int i = 0; i < internal_boundary_face_integs.Size(); i++)
-      {
-         delete internal_boundary_face_integs[i];
-      }
    }
 
    delete ext;

fem/linearform.hpp

@@ -65,10 +65,6 @@ protected:
    /// Set of Internal Face Integrators to be applied.
    Array<LinearFormIntegrator*> interior_face_integs;
 
-   /// Set of internal boundary face integrators to be applied.
-   Array<LinearFormIntegrator*> internal_boundary_face_integs;
-   Array<Array<int>*> internal_boundary_face_integs_marker; ///< Entries not owned.
-
    /// The element ids where the centers of the delta functions lie
    Array<int> domain_delta_integs_elem_id;
 
@@ -166,18 +162,6 @@ public:
    /// Adds new Interior Face Integrator. Assumes ownership of @a lfi.
    void AddInteriorFaceIntegrator(LinearFormIntegrator *lfi);
 
-   /// @brief Add new internal boundary face integrator. Assumes ownership of
-   /// @a lfi.
-   void AddInternalBoundaryFaceIntegrator(LinearFormIntegrator *lfi);
-
-   /** @brief Add new internal boundary face integrator, restricted to the given
-       boundary attributes.
-
-       Assumes ownership of @a lfi. The array @a internal_bdr_attr_marker is
-       stored internally as a pointer to the given Array<int> object. */
-   void AddInternalBoundaryFaceIntegrator(LinearFormIntegrator *lfi,
-                                          Array<int> &internal_bdr_attr_marker);
-
    /** @brief Access all integrators added with AddDomainIntegrator() which are
        not DeltaLFIntegrator%s or they are DeltaLFIntegrator%s with non-delta
        coefficients. */

fem/nonlinearform.cpp

@@ -89,8 +89,6 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
    {
       MFEM_VERIFY(!fnfi.Size(), "Interior faces terms not yet implemented!");
       MFEM_VERIFY(!bfnfi.Size(), "Boundary face terms not yet implemented!");
-      MFEM_VERIFY(!internal_boundary_face_integs.Size(),
-                  "Internal boundary face terms not yet implemented!");
       return ext->GetGridFunctionEnergy(x);
    }
 
@@ -127,11 +125,6 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
       MFEM_ABORT("TODO: add energy contribution from boundary face terms");
    }
 
-   if (internal_boundary_face_integs.Size())
-   {
-      MFEM_ABORT("TODO: add energy contribution from internal boundary face terms");
-   }
-
    return energy;
 }
 
@@ -281,62 +274,6 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
       }
    }
 
-   if (internal_boundary_face_integs.Size())
-   {
-      // Which internal boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < internal_boundary_face_integs.Size(); k++)
-      {
-         if (internal_boundary_face_integs_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         auto &bdr_marker = *internal_boundary_face_integs_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for internal boundary face "
-                     "integrator #" << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      Array<int> vdofs2;
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         auto *tr = mesh->GetInternalBdrFaceTransformations(i);
-         if (tr != nullptr)
-         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            fes->GetElementVDofs(tr->Elem2No, vdofs2);
-            vdofs.Append(vdofs2);
-
-            px.GetSubVector(vdofs, el_x);
-
-            const auto *fe1 = fes->GetFE(tr->Elem1No);
-            const auto *fe2 = fes->GetFE(tr->Elem2No);
-            for (int k = 0; k < internal_boundary_face_integs.Size(); k++)
-            {
-               if (internal_boundary_face_integs_marker[k] &&
-                   (*internal_boundary_face_integs_marker[k])[bdr_attr - 1] == 0)
-               {
-                  continue;
-               }
-
-               internal_boundary_face_integs[k]->AssembleFaceVector(
-                  *fe1, *fe2, *tr, el_x, el_y);
-               py.AddElementVector(vdofs, el_y);
-            }
-         }
-      }
-   }
-
    if (Serial())
    {
       if (cP) { cP->MultTranspose(py, y); }
@@ -485,62 +422,6 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const
       }
    }
 
-   if (internal_boundary_face_integs.Size())
-   {
-      // Which internal boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < internal_boundary_face_integs.Size(); k++)
-      {
-         if (internal_boundary_face_integs_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         auto &bdr_marker = *internal_boundary_face_integs_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for internal boundary face "
-                     "integrator #" << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      Array<int> vdofs2;
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         auto *tr = mesh->GetInternalBdrFaceTransformations(i);
-         if (tr != nullptr)
-         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            fes->GetElementVDofs(tr->Elem2No, vdofs2);
-            vdofs.Append(vdofs2);
-
-            px.GetSubVector(vdofs, el_x);
-
-            const auto *fe1 = fes->GetFE(tr->Elem1No);
-            const auto *fe2 = fes->GetFE(tr->Elem2No);
-            for (int k = 0; k < internal_boundary_face_integs.Size(); k++)
-            {
-               if (internal_boundary_face_integs_marker[k] &&
-                   (*internal_boundary_face_integs_marker[k])[bdr_attr - 1] == 0)
-               {
-                  continue;
-               }
-
-               internal_boundary_face_integs[k]->AssembleFaceGrad(
-                  *fe1, *fe2, *tr, el_x, elmat);
-               Grad->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
-            }
-         }
-      }
-   }
-
    if (!Grad->Finalized())
    {
       Grad->Finalize(skip_zeros);
@@ -593,10 +474,6 @@ NonlinearForm::~NonlinearForm()
    for (int i = 0; i <  dnfi.Size(); i++) { delete  dnfi[i]; }
    for (int i = 0; i <  fnfi.Size(); i++) { delete  fnfi[i]; }
    for (int i = 0; i < bfnfi.Size(); i++) { delete bfnfi[i]; }
-   for (int i = 0; i < internal_boundary_face_integs.Size(); i++)
-   {
-      delete internal_boundary_face_integs[i];
-   }
    delete ext;
 }
 

fem/nonlinearform.hpp

@@ -45,10 +45,6 @@ protected:
    Array<NonlinearFormIntegrator*> bfnfi; // owned
    Array<Array<int>*>              bfnfi_marker; // not owned
 
-   /// Set of internal boundary face integrators to be applied.
-   Array<NonlinearFormIntegrator*> internal_boundary_face_integs;
-   Array<Array<int>*> internal_boundary_face_integs_marker; ///< Entries not owned.
-
    mutable SparseMatrix *Grad, *cGrad; // owned
    /// Gradient Operator when not assembled as a matrix.
    mutable OperatorHandle hGrad; // has internal ownership flag
@@ -142,33 +138,6 @@ public:
    const Array<NonlinearFormIntegrator*> &GetBdrFaceIntegrators() const
    { return bfnfi; }
 
-   /// @brief Add new internal boundary face integrator. Assumes ownership of
-   /// @a nfi.
-   void AddInternalBoundaryFaceIntegrator(NonlinearFormIntegrator *nfi)
-   {
-      internal_boundary_face_integs.Append(nfi);
-      // nullptr -> all attributes are active
-      internal_boundary_face_integs_marker.Append(nullptr);
-   }
-
-   /** @brief Add new internal boundary face integrator, restricted to the given
-       boundary attributes.
-
-       Assumes ownership of @a nfi. The array @a internal_bdr_attr_marker is
-       stored internally as a pointer to the given Array<int> object. */
-   void AddInternalBoundaryFaceIntegrator(NonlinearFormIntegrator *nfi,
-                                          Array<int> &internal_bdr_attr_marker)
-   {
-      internal_boundary_face_integs.Append(nfi);
-      internal_boundary_face_integs_marker.Append(&internal_bdr_attr_marker);
-   }
-
-   /** @brief Access all boundary face integrators added with
-       AddBdrFaceIntegrator(). */
-   const Array<NonlinearFormIntegrator*> &GetInternalBoundaryFaceIntegrators()
-   const
-   { return internal_boundary_face_integs; }
-
    /// Specify essential boundary conditions.
    /** This method calls FiniteElementSpace::GetEssentialTrueDofs() and stores
        the result internally for use by other methods. If the @a rhs pointer is

fem/pbilinearform.cpp

@@ -264,11 +264,6 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
 
 void ParBilinearForm::Assemble(int skip_zeros)
 {
-   if (internal_boundary_face_integs.Size())
-   {
-      MFEM_ABORT("TODO: assemble contributions from shared internal boundary terms");
-   }
-
    if (interior_face_integs.Size())
    {
       pfes->ExchangeFaceNbrData();

fem/pfespace.cpp

@@ -940,7 +940,7 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
          }
          else if (i_offd[i+1] == i_offd[i] + 2)
          {
-            const double * T = ND_DofTransformation
+            const double * T = ND_StatelessDofTransformation
                                ::GetFaceTransform(ltori[i]).GetData();
             j_offd[i_offd[i] + 1] = j_offd[i_offd[i]] + 1;
             d_offd[i_offd[i]] = T[0]; d_offd[i_offd[i] + 1] = T[2];

fem/pnonlinearform.cpp

@@ -38,11 +38,6 @@ double ParNonlinearForm::GetParGridFunctionEnergy(const Vector &x) const
       MFEM_ABORT("TODO: add energy contribution from shared faces");
    }
 
-   if (internal_boundary_face_integs.Size())
-   {
-      MFEM_ABORT("TODO: add energy contributions from shared internal boundary terms");
-   }
-
    MPI_Allreduce(&loc_energy, &glob_energy, 1, MPI_DOUBLE, MPI_SUM,
                  ParFESpace()->GetComm());
 
@@ -51,11 +46,6 @@ double ParNonlinearForm::GetParGridFunctionEnergy(const Vector &x) const
 
 void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   if (internal_boundary_face_integs.Size() != 0)
-   {
-      MFEM_ABORT("TODO: assemble contributions from shared internal boundary terms");
-   }
-
    NonlinearForm::Mult(x, y); // x --(P)--> aux1 --(A_local)--> aux2
 
    if (fnfi.Size())
@@ -126,11 +116,6 @@ Operator &ParNonlinearForm::GetGradient(const Vector &x) const
 
    OperatorHandle dA(pGrad.Type()), Ph(pGrad.Type());
 
-   if (internal_boundary_face_integs.Size() != 0)
-   {
-      MFEM_ABORT("TODO: assemble contributions from shared internal boundary terms");
-   }
-
    if (fnfi.Size() == 0)
    {
       dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),

fem/tmop/tmop_pa_da3.cpp

@@ -21,6 +21,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, DatcSize,
                            const int NE,
                            const int ncomp,
                            const int sizeidx,
+                           const double input_min_size,
                            const DenseMatrix &w_,
                            const Array<double> &b_,
                            const Vector &x_,
@@ -97,7 +98,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, DatcSize,
          MFEM_SYNC_THREAD;
       }
       min = min_size[0];
-
+      if (input_min_size > 0.) { min = input_min_size; }
       kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
       kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);
@@ -162,6 +163,7 @@ void DiscreteAdaptTC::ComputeAllElementTargets(const FiniteElementSpace &pa_fes,
    const Array<double> &B = maps.B;
    const int D1D = maps.ndof;
    const int Q1D = maps.nqpt;
+   const double input_min_size = lim_min_size;
 
    Vector nc_size_red(NE, Device::GetDeviceMemoryType());
    nc_size_red.HostWrite();
@@ -181,7 +183,7 @@ void DiscreteAdaptTC::ComputeAllElementTargets(const FiniteElementSpace &pa_fes,
    tspec.UseDevice(true);
    R->Mult(tspec, tspec_e);
    const int id = (D1D << 4 ) | Q1D;
-   MFEM_LAUNCH_TMOP_KERNEL(DatcSize,id,NE,ncomp,sizeidx,W,B,
+   MFEM_LAUNCH_TMOP_KERNEL(DatcSize,id,NE,ncomp,sizeidx,input_min_size,W,B,
                            tspec_e, nc_size_red, Jtr);
 }
 

general/mem_manager.cpp

@@ -851,7 +851,8 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
 
 void MemoryManager::Register2_(void *h_ptr, void *d_ptr, size_t bytes,
                                MemoryType h_mt, MemoryType d_mt,
-                               bool own, bool alias, unsigned &flags)
+                               bool own, bool alias, unsigned &flags,
+                               unsigned valid_flags)
 {
    MFEM_CONTRACT_VAR(alias);
    MFEM_ASSERT(exists, "Internal error!");
@@ -871,7 +872,7 @@ void MemoryManager::Register2_(void *h_ptr, void *d_ptr, size_t bytes,
    mm.InsertDevice(d_ptr, h_ptr, bytes, h_mt, d_mt);
    flags = (own ? flags | (Mem::OWNS_HOST | Mem::OWNS_DEVICE) :
             flags & ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE)) |
-           Mem::VALID_HOST;
+           valid_flags;
 
    CheckHostMemoryType_(h_mt, h_ptr, alias);
 }

general/mem_manager.hpp

@@ -367,7 +367,8 @@ public:
 
    /** Wrap an externally pair of allocated pointers, @a h_ptr and @a d_ptr,
        of the given host MemoryType @a h_mt. */
-   /** The new memory object will have the device MemoryType set as valid.
+   /** The new memory object will have the device MemoryType set as valid unless
+       specified otherwise by the parameters @a valid_host and @a valid_device.
 
        The given @a h_ptr and @a d_ptr must be allocated appropriately for the
        given host MemoryType and its dual device MemoryType as defined by
@@ -376,13 +377,18 @@ public:
        The parameter @a own determines whether both @a h_ptr and @a d_ptr will
        be deleted when the method Delete() is called.
 
+       The parameters @a valid_host and @a valid_device determine which
+       pointers, host and/or device, will be marked as valid; at least one of
+       the two parameters must be set to true.
+
        @note Ownership can also be controlled by using the following methods:
          - ClearOwnerFlags,
          - SetHostPtrOwner,
          - SetDevicePtrOwner.
 
        @note The current memory is NOT deleted by this method. */
-   inline void Wrap(T *h_ptr, T *d_ptr, int size, MemoryType h_mt, bool own);
+   inline void Wrap(T *h_ptr, T *d_ptr, int size, MemoryType h_mt, bool own,
+                    bool valid_host = false, bool valid_device = true);
 
    /// Create a memory object that points inside the memory object @a base.
    /** The new Memory object uses the same MemoryType(s) as @a base.
@@ -645,7 +651,8 @@ private: // Static methods used by the Memory<T> class
    /// Register a pair of external host and device pointers
    static void Register2_(void *h_ptr, void *d_ptr, size_t bytes,
                           MemoryType h_mt, MemoryType d_mt,
-                          bool own, bool alias, unsigned &flags);
+                          bool own, bool alias, unsigned &flags,
+                          unsigned valid_flags);
 
    /// Register an alias. Note: base_h_ptr may be an alias.
    static void Alias_(void *base_h_ptr, size_t offset, size_t bytes,
@@ -958,17 +965,20 @@ inline void Memory<T>::Wrap(T *ptr, int size, MemoryType mt, bool own)
 }
 
 template <typename T>
-inline void Memory<T>::Wrap(T *ptr, T *d_ptr, int size, MemoryType mt, bool own)
+inline void Memory<T>::Wrap(T *h_ptr_, T *d_ptr, int size, MemoryType h_mt_,
+                            bool own, bool valid_host, bool valid_device)
 {
-   h_mt = mt;
+   h_mt = h_mt_;
    flags = 0;
-   h_ptr = ptr;
+   h_ptr = h_ptr_;
    capacity = size;
    MFEM_ASSERT(IsHostMemory(h_mt),"");
+   MFEM_ASSERT(valid_host || valid_device,"");
    const size_t bytes = size*sizeof(T);
    const MemoryType d_mt = MemoryManager::GetDualMemoryType(h_mt);
    MemoryManager::Register2_(h_ptr, d_ptr, bytes, h_mt, d_mt,
-                             own, false, flags);
+                             own, false, flags,
+                             valid_host*VALID_HOST|valid_device*VALID_DEVICE);
 }
 
 template <typename T>

linalg/hypre.cpp

@@ -5340,20 +5340,8 @@ void HypreAMS::MakeGradientAndInterpolation(
    rt_trace_space = dynamic_cast<const RT_Trace_FECollection*>(edge_fec);
    trace_space = trace_space || rt_trace_space;
 
-   int p = 1;
-   if (edge_fespace->GetNE() > 0)
-   {
-      MFEM_VERIFY(!edge_fespace->IsVariableOrder(), "");
-      if (trace_space)
-      {
-         p = edge_fespace->GetFaceOrder(0);
-         if (dim == 2) { p++; }
-      }
-      else
-      {
-         p = edge_fespace->GetElementOrder(0);
-      }
-   }
+   MFEM_VERIFY(!edge_fespace->IsVariableOrder(), "");
+   int p = edge_fec->GetOrder();
 
    ParMesh *pmesh = edge_fespace->GetParMesh();
    if (rt_trace_space)
@@ -5742,19 +5730,9 @@ void HypreADS::MakeDiscreteMatrices(ParFiniteElementSpace *face_fespace)
    const FiniteElementCollection *face_fec = face_fespace->FEColl();
    bool trace_space =
       (dynamic_cast<const RT_Trace_FECollection*>(face_fec) != NULL);
-   int p = 1;
-   if (face_fespace->GetNE() > 0)
-   {
-      MFEM_VERIFY(!face_fespace->IsVariableOrder(), "");
-      if (trace_space)
-      {
-         p = face_fespace->GetFaceOrder(0) + 1;
-      }
-      else
-      {
-         p = face_fespace->GetElementOrder(0);
-      }
-   }
+
+   MFEM_VERIFY(!face_fespace->IsVariableOrder(), "");
+   int p = face_fec->GetOrder();
 
    // define the nodal and edge finite element spaces associated with face_fespace
    ParMesh *pmesh = (ParMesh *) face_fespace->GetMesh();

linalg/sundials.cpp

@@ -20,8 +20,10 @@
 
 // SUNDIALS vectors
 #include <nvector/nvector_serial.h>
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA)
 #include <nvector/nvector_cuda.h>
+#elif defined(MFEM_USE_HIP)
+#include <nvector/nvector_hip.h>
 #endif
 #ifdef MFEM_USE_MPI
 #include <nvector/nvector_mpiplusx.h>
@@ -35,6 +37,14 @@
 // Access SUNDIALS object's content pointer
 #define GET_CONTENT(X) ( X->content )
 
+#if defined(MFEM_USE_CUDA)
+#define SUN_Hip_OR_Cuda(X) X##_Cuda
+#define SUN_HIP_OR_CUDA(X) X##_CUDA
+#elif defined(MFEM_USE_HIP)
+#define SUN_Hip_OR_Cuda(X) X##_Hip
+#define SUN_HIP_OR_CUDA(X) X##_HIP
+#endif
+
 using namespace std;
 
 #if (SUNDIALS_VERSION_MAJOR < 6)
@@ -112,16 +122,16 @@ MFEM_DEPRECATED N_Vector N_VNewEmpty_Parallel(MPI_Comm comm,
 
 #endif // MFEM_USE_MPI
 
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
 
 /// (DEPRECATED) Wrapper function for backwards compatibility with SUNDIALS
 /// version < 6
-MFEM_DEPRECATED N_Vector N_VNewWithMemHelp_Cuda(sunindextype length,
-                                                booleantype use_managed_mem,
-                                                SUNMemoryHelper helper,
-                                                SUNContext)
+MFEM_DEPRECATED N_Vector SUN_Hip_OR_Cuda(N_VNewWithMemHelp)(sunindextype length,
+                                                            booleantype use_managed_mem,
+                                                            SUNMemoryHelper helper,
+                                                            SUNContext)
 {
-   return N_VNewWithMemHelp_Cuda(length, use_managed_mem, helper);
+   return SUN_Hip_OR_Cuda(N_VNewWithMemHelp)(length, use_managed_mem, helper);
 }
 
 /// (DEPRECATED) Wrapper function for backwards compatibility with SUNDIALS
@@ -131,9 +141,9 @@ MFEM_DEPRECATED SUNMemoryHelper SUNMemoryHelper_NewEmpty(SUNContext)
    return SUNMemoryHelper_NewEmpty();
 }
 
-#endif // MFEM_USE_CUDA
+#endif // MFEM_USE_CUDA || MFEM_USE_HIP
 
-#if defined(MFEM_USE_MPI) && defined(MFEM_USE_CUDA)
+#if defined(MFEM_USE_MPI) && (defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
 
 /// (DEPRECATED) Wrapper function for backwards compatibility with SUNDIALS
 /// version < 6
@@ -143,7 +153,7 @@ MFEM_DEPRECATED N_Vector N_VMake_MPIPlusX(MPI_Comm comm, N_Vector local_vector,
    return N_VMake_MPIPlusX(comm, local_vector);
 }
 
-#endif // MFEM_USE_MPI && MFEM_USE_CUDA
+#endif // MFEM_USE_MPI && (MFEM_USE_CUDA || MFEM_USE_HIP)
 
 #endif // SUNDIALS_VERSION_MAJOR < 6
 
@@ -206,7 +216,7 @@ Sundials::~Sundials()
 
 #endif // SUNDIALS_VERSION_MAJOR >= 6
 
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
 SundialsMemHelper::SundialsMemHelper(SUNContext context)
 {
    /* Allocate helper */
@@ -215,8 +225,8 @@ SundialsMemHelper::SundialsMemHelper(SUNContext context)
    /* Set the ops */
    h->ops->alloc     = SundialsMemHelper_Alloc;
    h->ops->dealloc   = SundialsMemHelper_Dealloc;
-   h->ops->copy      = SUNMemoryHelper_Copy_Cuda;
-   h->ops->copyasync = SUNMemoryHelper_CopyAsync_Cuda;
+   h->ops->copy      = SUN_Hip_OR_Cuda(SUNMemoryHelper_Copy);
+   h->ops->copyasync = SUN_Hip_OR_Cuda(SUNMemoryHelper_CopyAsync);
 }
 
 SundialsMemHelper::SundialsMemHelper(SundialsMemHelper&& that_helper)
@@ -240,25 +250,25 @@ int SundialsMemHelper::SundialsMemHelper_Alloc(SUNMemoryHelper helper,
 #endif
                                               )
 {
-   int length = memsize/sizeof(double);
    SUNMemory sunmem = SUNMemoryNewEmpty();
 
    sunmem->ptr = NULL;
    sunmem->own = SUNTRUE;
 
+   // memsize is the number of bytes to allocate, so we use Memory<char>
    if (mem_type == SUNMEMTYPE_HOST)
    {
-      Memory<double> mem(length, Device::GetHostMemoryType());
+      Memory<char> mem(memsize, Device::GetHostMemoryType());
       mem.SetHostPtrOwner(false);
-      sunmem->ptr  = mfem::HostReadWrite(mem, length);
+      sunmem->ptr  = mfem::HostReadWrite(mem, memsize);
       sunmem->type = SUNMEMTYPE_HOST;
       mem.Delete();
    }
    else if (mem_type == SUNMEMTYPE_DEVICE || mem_type == SUNMEMTYPE_UVM)
    {
-      Memory<double> mem(length, Device::GetDeviceMemoryType());
+      Memory<char> mem(memsize, Device::GetDeviceMemoryType());
       mem.SetDevicePtrOwner(false);
-      sunmem->ptr  = mfem::ReadWrite(mem, length);
+      sunmem->ptr  = mfem::ReadWrite(mem, memsize);
       sunmem->type = mem_type;
       mem.Delete();
    }
@@ -283,14 +293,14 @@ int SundialsMemHelper::SundialsMemHelper_Dealloc(SUNMemoryHelper helper,
    {
       if (sunmem->type == SUNMEMTYPE_HOST)
       {
-         Memory<double> mem(static_cast<double*>(sunmem->ptr), 1,
-                            Device::GetHostMemoryType(), true);
+         Memory<char> mem(static_cast<char*>(sunmem->ptr), 1,
+                          Device::GetHostMemoryType(), true);
          mem.Delete();
       }
       else if (sunmem->type == SUNMEMTYPE_DEVICE || sunmem->type == SUNMEMTYPE_UVM)
       {
-         Memory<double> mem(static_cast<double*>(sunmem->ptr), 1,
-                            Device::GetDeviceMemoryType(), true);
+         Memory<char> mem(static_cast<char*>(sunmem->ptr), 1,
+                          Device::GetDeviceMemoryType(), true);
          mem.Delete();
       }
       else
@@ -303,7 +313,7 @@ int SundialsMemHelper::SundialsMemHelper_Dealloc(SUNMemoryHelper helper,
    return 0;
 }
 
-#endif // MFEM_USE_CUDA
+#endif // MFEM_USE_CUDA || MFEM_USE_HIP
 
 
 // ---------------------------------------------------------------------------
@@ -329,12 +339,13 @@ void SundialsNVector::_SetNvecDataAndSize_(long glob_size)
          NV_LENGTH_S(local_x) = size;
          break;
       }
-#ifdef MFEM_USE_CUDA
-      case SUNDIALS_NVEC_CUDA:
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
+      case SUN_HIP_OR_CUDA(SUNDIALS_NVEC):
       {
-         N_VSetHostArrayPointer_Cuda(HostReadWrite(), local_x);
-         N_VSetDeviceArrayPointer_Cuda(ReadWrite(), local_x);
-         static_cast<N_VectorContent_Cuda>(GET_CONTENT(local_x))->length = size;
+         SUN_Hip_OR_Cuda(N_VSetHostArrayPointer)(HostReadWrite(), local_x);
+         SUN_Hip_OR_Cuda(N_VSetDeviceArrayPointer)(ReadWrite(), local_x);
+         static_cast<SUN_Hip_OR_Cuda(N_VectorContent)>(GET_CONTENT(
+                                                          local_x))->length = size;
          break;
       }
 #endif
@@ -403,14 +414,14 @@ void SundialsNVector::_SetDataAndSize_()
          if (known) { data.ClearOwnerFlags(); }
          break;
       }
-#ifdef MFEM_USE_CUDA
-      case SUNDIALS_NVEC_CUDA:
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
+      case SUN_HIP_OR_CUDA(SUNDIALS_NVEC):
       {
-         double *h_ptr = N_VGetHostArrayPointer_Cuda(local_x);
-         double *d_ptr = N_VGetDeviceArrayPointer_Cuda(local_x);
+         double *h_ptr = SUN_Hip_OR_Cuda(N_VGetHostArrayPointer)(local_x);
+         double *d_ptr = SUN_Hip_OR_Cuda(N_VGetDeviceArrayPointer)(local_x);
          const bool known = mm.IsKnown(h_ptr);
-         size = N_VGetLength_Cuda(local_x);
-         data.Wrap(h_ptr, d_ptr, size, Device::GetHostMemoryType(), false);
+         size = SUN_Hip_OR_Cuda(N_VGetLength)(local_x);
+         data.Wrap(h_ptr, d_ptr, size, Device::GetHostMemoryType(), false, false, true);
          if (known) { data.ClearOwnerFlags(); }
          UseDevice(true);
          break;
@@ -525,11 +536,12 @@ void SundialsNVector::SetDataAndSize(double *d, int s, long glob_size)
 N_Vector SundialsNVector::MakeNVector(bool use_device)
 {
    N_Vector x;
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
    if (use_device)
    {
-      x = N_VNewWithMemHelp_Cuda(0, UseManagedMemory(), Sundials::GetMemHelper(),
-                                 Sundials::GetContext());
+      x = SUN_Hip_OR_Cuda(N_VNewWithMemHelp)(0, UseManagedMemory(),
+                                             Sundials::GetMemHelper(),
+                                             Sundials::GetContext());
    }
    else
    {
@@ -555,12 +567,13 @@ N_Vector SundialsNVector::MakeNVector(MPI_Comm comm, bool use_device)
    }
    else
    {
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
       if (use_device)
       {
-         x = N_VMake_MPIPlusX(comm, N_VNewWithMemHelp_Cuda(0, UseManagedMemory(),
-                                                           Sundials::GetMemHelper(),
-                                                           Sundials::GetContext()),
+         x = N_VMake_MPIPlusX(comm, SUN_Hip_OR_Cuda(N_VNewWithMemHelp)(0,
+                                                                       UseManagedMemory(),
+                                                                       Sundials::GetMemHelper(),
+                                                                       Sundials::GetContext()),
                               Sundials::GetContext());
       }
       else
@@ -569,7 +582,7 @@ N_Vector SundialsNVector::MakeNVector(MPI_Comm comm, bool use_device)
       }
 #else
       x = N_VNewEmpty_Parallel(comm, 0, 0, Sundials::GetContext());
-#endif // MFEM_USE_CUDA
+#endif // MFEM_USE_CUDA || MFEM_USE_HIP
    }
 
    MFEM_VERIFY(x, "Error in SundialsNVector::MakeNVector.");

linalg/sundials.hpp

@@ -32,13 +32,24 @@
 #if defined(MFEM_USE_CUDA) && ((SUNDIALS_VERSION_MAJOR == 5) && (SUNDIALS_VERSION_MINOR < 4))
 #error MFEM requires SUNDIALS version 5.4.0 or newer when MFEM_USE_CUDA=TRUE!
 #endif
+#if defined(MFEM_USE_HIP) && ((SUNDIALS_VERSION_MAJOR == 5) && (SUNDIALS_VERSION_MINOR < 7))
+#error MFEM requires SUNDIALS version 5.7.0 or newer when MFEM_USE_HIP=TRUE!
+#endif
+#if defined(MFEM_USE_CUDA) && !defined(SUNDIALS_NVECTOR_CUDA)
+#error MFEM_USE_CUDA=TRUE requires SUNDIALS to be built with CUDA support
+#endif
+#if defined(MFEM_USE_HIP) && !defined(SUNDIALS_NVECTOR_HIP)
+#error MFEM_USE_HIP=TRUE requires SUNDIALS to be built with HIP support
+#endif
 #include <sundials/sundials_matrix.h>
 #include <sundials/sundials_linearsolver.h>
 #include <arkode/arkode_arkstep.h>
 #include <cvodes/cvodes.h>
 #include <kinsol/kinsol.h>
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA)
 #include <sunmemory/sunmemory_cuda.h>
+#elif defined(MFEM_USE_HIP)
+#include <sunmemory/sunmemory_hip.h>
 #endif
 
 #include <functional>
@@ -62,10 +73,10 @@ using SUNContext = void*;
 namespace mfem
 {
 
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
 
 // ---------------------------------------------------------------------------
-// SUNMemory interface class (used when CUDA is enabled)
+// SUNMemory interface class (used when CUDA or HIP is enabled)
 // ---------------------------------------------------------------------------
 class SundialsMemHelper
 {
@@ -113,10 +124,10 @@ public:
 
 };
 
-#else // MFEM_USE_CUDA
+#else // MFEM_USE_CUDA || MFEM_USE_HIP
 
 // ---------------------------------------------------------------------------
-// Dummy SUNMemory interface class (used when CUDA is not enabled)
+// Dummy SUNMemory interface class (used when CUDA or HIP is not enabled)
 // ---------------------------------------------------------------------------
 class SundialsMemHelper
 {
@@ -130,7 +141,7 @@ public:
    }
 };
 
-#endif // MFEM_USE_CUDA
+#endif // MFEM_USE_CUDA || MFEM_USE_HIP
 
 
 /// Singleton class for SUNContext and SundialsMemHelper objects
@@ -290,17 +301,17 @@ public:
 #endif
 
    /// Create a N_Vector.
-   /** @param[in] use_device  If true, use the SUNDIALS CUDA N_Vector. */
+   /** @param[in] use_device  If true, use the SUNDIALS CUDA or HIP N_Vector. */
    static N_Vector MakeNVector(bool use_device);
 
 #ifdef MFEM_USE_MPI
    /// Create a parallel N_Vector.
    /** @param[in] comm  The MPI communicator to use.
-       @param[in] use_device  If true, use the SUNDIALS CUDA N_Vector. */
+       @param[in] use_device  If true, use the SUNDIALS CUDA or HIP N_Vector. */
    static N_Vector MakeNVector(MPI_Comm comm, bool use_device);
 #endif
 
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
    static bool UseManagedMemory()
    {
       return Device::GetDeviceMemoryType() == MemoryType::MANAGED;

linalg/vector.cpp

@@ -15,13 +15,6 @@
 #include "vector.hpp"
 #include "../general/forall.hpp"
 
-#if defined(MFEM_USE_SUNDIALS)
-#include "sundials.hpp"
-#if defined(MFEM_USE_MPI)
-#include <nvector/nvector_parallel.h>
-#endif
-#endif
-
 #ifdef MFEM_USE_OPENMP
 #include <omp.h>
 #endif

linalg/vector.hpp

@@ -19,9 +19,7 @@
 #include "../general/globals.hpp"
 #include "../general/mem_manager.hpp"
 #include "../general/device.hpp"
-#ifdef MFEM_USE_SUNDIALS
-#include <nvector/nvector_serial.h>
-#endif
+
 #include <cmath>
 #include <iostream>
 #include <limits>

makefile

@@ -125,7 +125,7 @@ EXAMPLE_TEST_DIRS := examples
 
 MINIAPP_SUBDIRS = common electromagnetics meshing navier performance tools \
  toys nurbs gslib adjoint solvers shifted mtop parelag autodiff hooke \
- multidomain dpg hdiv-linear-solver spde
+ multidomain dpg hdiv-linear-solver spde contact
 MINIAPP_DIRS := $(addprefix miniapps/,$(MINIAPP_SUBDIRS))
 MINIAPP_TEST_DIRS := $(filter-out %/common,$(MINIAPP_DIRS))
 MINIAPP_USE_COMMON := $(addprefix miniapps/,electromagnetics meshing tools \

mesh/mesh.cpp

@@ -1117,25 +1117,6 @@ FaceElementTransformations *Mesh::GetBdrFaceTransformations(int BdrElemNo)
    return tr;
 }
 
-FaceElementTransformations *Mesh::GetInternalBdrFaceTransformations(
-   int IntBdrElemNo)
-{
-   int fn = GetBdrFace(IntBdrElemNo);
-
-   // Check if the face is not interior
-   if (!FaceIsTrueInterior(fn))
-   {
-      return nullptr;
-   }
-
-   auto *tr = GetFaceElementTransformations(fn, 31);
-   tr->Attribute = boundary[IntBdrElemNo]->GetAttribute();
-   tr->ElementNo = IntBdrElemNo;
-   tr->ElementType = ElementTransformation::BDR_FACE;
-   tr->mesh = this;
-   return tr;
-}
-
 int Mesh::GetBdrFace(int BdrElemNo) const
 {
    int fn;
@@ -3039,6 +3020,10 @@ void Mesh::FinalizeTopology(bool generate_bdr)
    if (Dim == 1)
    {
       GenerateFaces();
+      if (NumOfBdrElements == 0 && generate_bdr)
+      {
+         GenerateBoundaryElements();
+      }
    }
 
    if (ncmesh)
@@ -5679,13 +5664,54 @@ int Mesh::GetTriOrientation(const int *base, const int *test)
    for (int j = 0; j < 3; j++)
       if (test[aor[j]] != base[j])
       {
-         mfem_error("Mesh::GetTriOrientation(...)");
+         mfem::err << "Mesh::GetTriOrientation(...)" << endl;
+         mfem::err << " base = [";
+         for (int k = 0; k < 3; k++)
+         {
+            mfem::err << " " << base[k];
+         }
+         mfem::err << " ]\n test = [";
+         for (int k = 0; k < 3; k++)
+         {
+            mfem::err << " " << test[k];
+         }
+         mfem::err << " ]" << endl;
+         mfem_error();
       }
 #endif
 
    return orient;
 }
 
+int Mesh::ComposeTriOrientations(int ori_a_b, int ori_b_c)
+{
+   // Static method.
+   // Given three, possibly different, configurations of triangular face
+   // vertices: va, vb, and vc.  This function returns the relative orientation
+   // GetTriOrientation(va, vc) by composing previously computed orientations
+   // ori_a_b = GetTriOrientation(va, vb) and
+   // ori_b_c = GetTriOrientation(vb, vc) without accessing the vertices.
+
+   const int oo[6][6] =
+   {
+      {0, 1, 2, 3, 4, 5},
+      {1, 0, 5, 4, 3, 2},
+      {2, 3, 4, 5, 0, 1},
+      {3, 2, 1, 0, 5, 4},
+      {4, 5, 0, 1, 2, 3},
+      {5, 4, 3, 2, 1, 0}
+   };
+
+   int ori_a_c = oo[ori_a_b][ori_b_c];
+   return ori_a_c;
+}
+
+int Mesh::InvertTriOrientation(int ori)
+{
+   const int inv_ori[6] = {0, 1, 4, 3, 2, 5};
+   return inv_ori[ori];
+}
+
 int Mesh::GetQuadOrientation(const int *base, const int *test)
 {
    int i;
@@ -5734,6 +5760,37 @@ int Mesh::GetQuadOrientation(const int *base, const int *test)
    return 2*i+1;
 }
 
+int Mesh::ComposeQuadOrientations(int ori_a_b, int ori_b_c)
+{
+   // Static method.
+   // Given three, possibly different, configurations of quadrilateral face
+   // vertices: va, vb, and vc.  This function returns the relative orientation
+   // GetQuadOrientation(va, vc) by composing previously computed orientations
+   // ori_a_b = GetQuadOrientation(va, vb) and
+   // ori_b_c = GetQuadOrientation(vb, vc) without accessing the vertices.
+
+   const int oo[8][8] =
+   {
+      {0, 1, 2, 3, 4, 5, 6, 7},
+      {1, 0, 3, 2, 5, 4, 7, 6},
+      {2, 7, 4, 1, 6, 3, 0, 5},
+      {3, 6, 5, 0, 7, 2, 1, 4},
+      {4, 5, 6, 7, 0, 1, 2, 3},
+      {5, 4, 7, 6, 1, 0, 3, 2},
+      {6, 3, 0, 5, 2, 7, 4, 1},
+      {7, 2, 1, 4, 3, 6, 5, 0}
+   };
+
+   int ori_a_c = oo[ori_a_b][ori_b_c];
+   return ori_a_c;
+}
+
+int Mesh::InvertQuadOrientation(int ori)
+{
+   const int inv_ori[8] = {0, 1, 6, 3, 4, 5, 2, 7};
+   return inv_ori[ori];
+}
+
 int Mesh::GetTetOrientation(const int *base, const int *test)
 {
    // Static method.
@@ -6549,9 +6606,9 @@ const Table & Mesh::ElementToEdgeTable() const
 
 void Mesh::AddPointFaceElement(int lf, int gf, int el)
 {
-   if (faces_info[gf].Elem1No == -1)  // this will be elem1
+   if (faces[gf] == NULL)  // this will be elem1
    {
-      // faces[gf] = new Point(&gf);
+      faces[gf] = new Point(&gf);
       faces_info[gf].Elem1No  = el;
       faces_info[gf].Elem1Inf = 64 * lf; // face lf with orientation 0
       faces_info[gf].Elem2No  = -1; // in case there's no other side
@@ -7962,7 +8019,7 @@ void Mesh::GetNode(int i, double *coord) const
       FiniteElementSpace *fes = Nodes->FESpace();
       for (int j = 0; j < spaceDim; j++)
       {
-         coord[j] = (*Nodes)(fes->DofToVDof(i, j));
+         coord[j] = AsConst(*Nodes)(fes->DofToVDof(i, j));
       }
    }
    else

mesh/submesh/psubmesh.cpp

@@ -84,7 +84,8 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       GetEdgeVertices(i, lv);
 
       // Find vertices/edge in parent mesh
-      int parent_edge_id = v2v(parent_vertex_ids_[lv[0]], parent_vertex_ids_[lv[1]]);
+      int parent_edge_id = v2v(parent_vertex_ids_[lv[0]],
+                               parent_vertex_ids_[lv[1]]);
       parent_edge_ids_.Append(parent_edge_id);
    }
 
@@ -106,6 +107,73 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       {
          parent_to_submesh_face_ids_[parent_face_ids_[i]] = i;
       }
+
+      parent_face_ori_.SetSize(NumOfFaces);
+
+      for (int i = 0; i < NumOfFaces; i++)
+      {
+         Array<int> sub_vert;
+         GetFaceVertices(i, sub_vert);
+
+         Array<int> sub_par_vert(sub_vert.Size());
+         for (int j = 0; j < sub_vert.Size(); j++)
+         {
+            sub_par_vert[j] = parent_vertex_ids_[sub_vert[j]];
+         }
+
+         Array<int> par_vert;
+         parent.GetFaceVertices(parent_face_ids_[i], par_vert);
+
+         if (par_vert.Size() == 3)
+         {
+            parent_face_ori_[i] = GetTriOrientation(par_vert, sub_par_vert);
+         }
+         else
+         {
+            parent_face_ori_[i] = GetQuadOrientation(par_vert, sub_par_vert);
+         }
+      }
+   }
+   else if (Dim == 2)
+   {
+      parent_face_ori_.SetSize(NumOfElements);
+
+      for (int i = 0; i < NumOfElements; i++)
+      {
+         Array<int> sub_vert;
+         GetElementVertices(i, sub_vert);
+
+         Array<int> sub_par_vert(sub_vert.Size());
+         for (int j = 0; j < sub_vert.Size(); j++)
+         {
+            sub_par_vert[j] = parent_vertex_ids_[sub_vert[j]];
+         }
+
+         Array<int> par_vert;
+         int be_ori = 0;
+         if (from == SubMesh::From::Boundary)
+         {
+            parent.GetBdrElementVertices(parent_element_ids_[i], par_vert);
+
+            int f = -1;
+            parent.GetBdrElementFace(parent_element_ids_[i], &f, &be_ori);
+         }
+         else
+         {
+            parent.GetElementVertices(parent_element_ids_[i], par_vert);
+         }
+
+         if (par_vert.Size() == 3)
+         {
+            int se_ori = GetTriOrientation(par_vert, sub_par_vert);
+            parent_face_ori_[i] = ComposeTriOrientations(be_ori, se_ori);
+         }
+         else
+         {
+            int se_ori = GetQuadOrientation(par_vert, sub_par_vert);
+            parent_face_ori_[i] = ComposeQuadOrientations(be_ori, se_ori);
+         }
+      }
    }
 
    ListOfIntegerSets groups;
@@ -145,7 +213,7 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
    {
       BuildFaceGroup(ngroups, rht, nstrias, rhq, nsquads);
    }
-   else
+   else if (Dim == 2)
    {
       group_stria.MakeI(ngroups);
       group_stria.MakeJ();
@@ -167,7 +235,9 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
 
    // Add boundaries
    {
-      int num_of_faces_or_edges = (Dim == 2) ? NumOfEdges : NumOfFaces;
+      int num_of_faces_or_edges =
+         (Dim == 3) ? NumOfFaces :
+         ((Dim == 2) ? NumOfEdges : NumOfVertices);
       Array<int> &be2face = (Dim == 2) ? be_to_edge : be_to_face;
 
       if (Dim == 3)
@@ -190,9 +260,11 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       boundary.SetSize(NumOfBdrElements);
       be2face.SetSize(NumOfBdrElements);
       Array<int> parent_face_to_be;
+      int max_bdr_attr = -1;
       if (Dim == 3)
       {
          parent_face_to_be = parent.GetFaceToBdrElMap();
+         max_bdr_attr = parent.bdr_attributes.Max();
       }
       for (int i = 0, j = 0; i < num_of_faces_or_edges; i++)
       {
@@ -209,7 +281,7 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
                }
                else
                {
-                  boundary[j]->SetAttribute(SubMesh::GENERATED_ATTRIBUTE);
+                  boundary[j]->SetAttribute(max_bdr_attr + 1);
                }
             }
             else
@@ -743,9 +815,14 @@ void ParSubMesh::BuildSharedEdgesMapping(const int sedges_ct,
          else
          {
             Array<int> vert;
-            GetEdgeVertices(submesh_edge_id, vert);
+            parent_.GetEdgeVertices(ple, vert);
+            // Swap order of vertices if orientation in parent group is -1
+            int v0 = parent_to_submesh_vertex_ids_[vert[(1-o)/2]];
+            int v1 = parent_to_submesh_vertex_ids_[vert[(1+o)/2]];
 
-            shared_edges.Append(new Segment(vert[0], vert[1], 1));
+            // The orienation of the shared edge relative to the local edge
+            // will be determined by whether v0 < v1 or v1 < v0
+            shared_edges.Append(new Segment(v0, v1, 1));
             sedge_ledge.Append(submesh_edge_id);
          }
       }
@@ -760,6 +837,53 @@ void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
    shared_quads.Reserve(nsquads);
    sface_lface.Reserve(nstrias + nsquads);
 
+   // sface_lface should list the triangular shared faces first
+   // followed by the quadrilateral shared faces.
+
+   for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
+   {
+      for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
+      {
+         int plt, o;
+         parent_.GroupTriangle(g, gt, plt, o);
+         int submesh_face_id = parent_to_submesh_face_ids_[plt];
+         if ((submesh_face_id == -1) || rht[st] == -1)
+         {
+            // parent shared face is not in SubMesh or is not shared
+         }
+         else
+         {
+            Array<int> vert;
+
+            GetFaceVertices(submesh_face_id, vert);
+
+            int v0 = vert[0];
+            int v1 = vert[1];
+            int v2 = vert[2];
+
+            // See Mesh::GetTriOrientation for info on interpretting "o"
+            switch (o)
+            {
+               case 1:
+                  std::swap(v0,v1);
+                  break;
+               case 3:
+                  std::swap(v2,v0);
+                  break;
+               case 5:
+                  std::swap(v1,v2);
+                  break;
+               default:
+                  // Do nothing
+                  break;
+            }
+
+            shared_trias.Append(Vert3(v0, v1, v2));
+            sface_lface.Append(submesh_face_id);
+         }
+      }
+   }
+
    for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
    {
       for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
@@ -776,29 +900,34 @@ void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
             Array<int> vert;
             GetFaceVertices(submesh_face_id, vert);
 
-            shared_quads.Append(Vert4(vert[0], vert[1], vert[2], vert[3]));
-            sface_lface.Append(submesh_face_id);
-         }
-      }
-   }
+            int v0 = vert[0];
+            int v1 = vert[1];
+            int v2 = vert[2];
+            int v3 = vert[3];
 
-   for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
-   {
-      for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
-      {
-         int plt, o;
-         parent_.GroupTriangle(g, gt, plt, o);
-         int submesh_face_id = parent_to_submesh_face_ids_[plt];
-         if ((submesh_face_id == -1) || rht[st] == -1)
-         {
-            // parent shared face is not in SubMesh or is not shared
-         }
-         else
-         {
-            Array<int> vert;
-            GetFaceVertices(submesh_face_id, vert);
+            // See Mesh::GetQuadOrientation for info on interpretting "o"
+            switch (o)
+            {
+               case 1:
+                  std::swap(v1,v3);
+                  break;
+               case 3:
+                  std::swap(v0,v1);
+                  std::swap(v2,v3);
+                  break;
+               case 5:
+                  std::swap(v0,v2);
+                  break;
+               case 7:
+                  std::swap(v0,v3);
+                  std::swap(v1,v2);
+                  break;
+               default:
+                  // Do nothing
+                  break;
+            }
 
-            shared_trias.Append(Vert3(vert[0], vert[1], vert[2]));
+            shared_quads.Append(Vert4(v0, v1, v2, v3));
             sface_lface.Append(submesh_face_id);
          }
       }

mesh/submesh/psubmesh.hpp

@@ -128,6 +128,16 @@ public:
       return parent_face_ids_;
    }
 
+   /**
+    * @brief Get the relative face orientations
+    *
+    * ParSubMesh element id (array index) to parent ParMesh face orientation.
+    */
+   const Array<int>& GetParentFaceOrientations() const
+   {
+      return parent_face_ori_;
+   }
+
    /**
     * @brief Get the ParSubMesh face id map.
     *
@@ -359,6 +369,10 @@ private:
    /// ParMesh face ids.
    Array<int> parent_face_ids_;
 
+   /// Mapping from SubMesh face ids (index of the array), to the orientation
+   /// of the face relative to the parent face.
+   Array<int> parent_face_ori_;
+
    /// Mapping from parent ParMesh vertex ids (index of the array), to the
    /// ParSubMesh vertex ids. Inverse map of parent_vertex_ids_.
    Array<int> parent_to_submesh_vertex_ids_;

mesh/submesh/ptransfermap.cpp

@@ -41,9 +41,68 @@ ParTransferMap::ParTransferMap(const ParGridFunction &src,
 
       category_ = TransferCategory::SubMeshToSubMesh;
 
-      root_fes_.reset(new ParFiniteElementSpace(
-                         *src.ParFESpace(),
-                         *const_cast<ParMesh *>(SubMeshUtils::GetRootParent(*src_sm))));
+      {
+         ParMesh * parent_mesh =
+            const_cast<ParMesh *>(SubMeshUtils::GetRootParent(*src_sm));
+
+         int parent_dim = parent_mesh->Dimension();
+         int src_sm_dim = src_sm->Dimension();
+         int dst_sm_dim = dst_sm->Dimension();
+
+         bool root_fes_reset = false;
+         if (src_sm_dim == parent_dim - 1 && dst_sm_dim == parent_dim - 1)
+         {
+            const ParFiniteElementSpace *src_fes = src.ParFESpace();
+            const ParFiniteElementSpace *dst_fes = dst.ParFESpace();
+
+            const FiniteElementCollection *src_fec = src_fes->FEColl();
+            const FiniteElementCollection *dst_fec = dst_fes->FEColl();
+
+            const L2_FECollection *src_l2_fec =
+               dynamic_cast<const L2_FECollection*>(src_fec);
+            const L2_FECollection *dst_l2_fec =
+               dynamic_cast<const L2_FECollection*>(dst_fec);
+
+            if (src_l2_fec != NULL && dst_l2_fec != NULL)
+            {
+               // Source and destination are both lower dimension L2 spaces.
+               // Transfer them as the trace of an RT space if possible.
+
+               int src_mt = src_fec->GetMapType(src_sm_dim);
+               int dst_mt = dst_fec->GetMapType(dst_sm_dim);
+
+               int src_bt = src_l2_fec->GetBasisType();
+               int dst_bt = dst_l2_fec->GetBasisType();
+
+               int src_p = src_fec->GetOrder();
+               int dst_p = dst_fec->GetOrder();
+
+               if (src_mt == FiniteElement::INTEGRAL &&
+                   dst_mt == FiniteElement::INTEGRAL &&
+                   src_bt == BasisType::GaussLegendre &&
+                   dst_bt == BasisType::GaussLegendre &&
+                   src_p == dst_p)
+               {
+                  // The subspaces are consistent with the trace of an RT space
+                  root_fec_.reset(new RT_FECollection(src_p, parent_dim));
+                  root_fes_.reset(new ParFiniteElementSpace(
+                                     const_cast<ParMesh *>(
+                                        SubMeshUtils::GetRootParent(*src_sm)),
+                                     root_fec_.get()));
+                  root_fes_reset = true;
+               }
+            }
+         }
+
+         if (!root_fes_reset)
+         {
+            root_fes_.reset(new ParFiniteElementSpace(
+                               *src.ParFESpace(),
+                               const_cast<ParMesh *>(
+                                  SubMeshUtils::GetRootParent(*src_sm))));
+         }
+      }
+
       subfes1 = src.ParFESpace();
       subfes2 = dst.ParFESpace();
 
@@ -105,8 +164,12 @@ void ParTransferMap::Transfer(const ParGridFunction &src,
       // dst = S1^T src
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
-         dst(i) = src(sub1_to_parent_map_[i]);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub1_to_parent_map_[i], s);
+         dst(i) = s * src(j);
       }
+
+      CorrectFaceOrientations(*dst.ParFESpace(), src, dst);
    }
    else if (category_ == TransferCategory::SubMeshToParent)
    {
@@ -117,10 +180,15 @@ void ParTransferMap::Transfer(const ParGridFunction &src,
 
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
-         dst(sub1_to_parent_map_[i]) = src(i);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub1_to_parent_map_[i], s);
+         dst(j) = s * src(i);
       }
 
-      CommunicateSharedVdofs(dst);
+      CorrectFaceOrientations(*src.ParFESpace(), src, dst,
+                              &sub1_to_parent_map_);
+
+      // CommunicateSharedVdofs(dst);
    }
    else if (category_ == TransferCategory::SubMeshToSubMesh)
    {
@@ -132,20 +200,34 @@ void ParTransferMap::Transfer(const ParGridFunction &src,
 
       for (int i = 0; i < sub2_to_parent_map_.Size(); i++)
       {
-         z_(sub2_to_parent_map_[i]) = dst(i);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub2_to_parent_map_[i], s);
+         z_(j) = s * dst(i);
       }
 
+      CorrectFaceOrientations(*dst.ParFESpace(), dst, z_,
+                              &sub2_to_parent_map_);
+
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
-         z_(sub1_to_parent_map_[i]) = src(i);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub1_to_parent_map_[i], s);
+         z_(j) = s * src(i);
       }
 
-      CommunicateSharedVdofs(z_);
+      CorrectFaceOrientations(*src.ParFESpace(), src, z_,
+                              &sub1_to_parent_map_);
+
+      // CommunicateSharedVdofs(z_);
 
       for (int i = 0; i < sub2_to_parent_map_.Size(); i++)
       {
-         dst(i) = z_(sub2_to_parent_map_[i]);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub2_to_parent_map_[i], s);
+         dst(i) = s * z_(j);
       }
+
+      CorrectFaceOrientations(*dst.ParFESpace(), z_, dst);
    }
    else
    {
@@ -159,7 +241,7 @@ void ParTransferMap::CommunicateIndicesSet(Array<int> &map, int dst_sz)
    indices_set_local_ = 0;
    for (int i = 0; i < map.Size(); i++)
    {
-      indices_set_local_[map[i]] = 1;
+      indices_set_local_[(map[i]>=0)?map[i]:(-map[i]-1)] = 1;
    }
    indices_set_global_ = indices_set_local_;
    root_gc_->Reduce(indices_set_global_, GroupCommunicator::Sum);
@@ -214,4 +296,84 @@ void ParTransferMap::CommunicateSharedVdofs(Vector &f) const
    root_gc_->Bcast<double>(f.HostReadWrite());
 }
 
+void
+ParTransferMap::CorrectFaceOrientations(const ParFiniteElementSpace &fes,
+                                        const Vector &src,
+                                        Vector &dst,
+                                        const Array<int> *sub_to_parent_map)
+{
+   const FiniteElementCollection * fec = fes.FEColl();
+
+   ParSubMesh * mesh = dynamic_cast<ParSubMesh*>(fes.GetParMesh());
+
+   const Array<int>& parent_face_ori = mesh->GetParentFaceOrientations();
+
+   if (parent_face_ori.Size() == 0) { return; }
+
+   VDofTransformation vdoftrans(fes.GetVDim(),
+                                fes.GetOrdering());
+
+   int dim = mesh->Dimension();
+   bool face = (dim == 3);
+
+   Array<int> vdofs;
+   Array<int> Fo(1);
+   Vector face_vector;
+
+   for (int i = 0; i < (face ? mesh->GetNumFaces() : mesh->GetNE()); i++)
+   {
+      if (parent_face_ori[i] == 0) { continue; }
+
+      Geometry::Type geom = face ? mesh->GetFaceGeometry(i) :
+                            mesh->GetElementGeometry(i);;
+
+      StatelessDofTransformation * doftrans =
+         fec->DofTransformationForGeometry(geom);
+
+      if (doftrans == NULL) { continue; }
+
+      vdoftrans.SetDofTransformation(*doftrans);
+
+      Fo[0] = parent_face_ori[i];
+      vdoftrans.SetFaceOrientations(Fo);
+
+      if (face)
+      {
+         fes.GetFaceVDofs(i, vdofs);
+      }
+      else
+      {
+         fes.GetElementVDofs(i, vdofs);
+      }
+
+      if (sub_to_parent_map)
+      {
+         src.GetSubVector(vdofs, face_vector);
+         vdoftrans.TransformPrimal(face_vector);
+      }
+      else
+      {
+         dst.GetSubVector(vdofs, face_vector);
+         vdoftrans.InvTransformPrimal(face_vector);
+      }
+
+      for (int j = 0; j < vdofs.Size(); j++)
+      {
+         double s = 1.0;
+         int k = FiniteElementSpace::DecodeDof(vdofs[j], s);
+
+         if (sub_to_parent_map)
+         {
+            double sps = 1.0;
+            int spk = FiniteElementSpace::DecodeDof((*sub_to_parent_map)[k],
+                                                    sps);
+            s *= sps;
+            k = spk;
+         }
+
+         dst[k] = s * face_vector[j];
+      }
+   }
+}
+
 #endif // MFEM_USE_MPI

mesh/submesh/ptransfermap.hpp

@@ -75,6 +75,11 @@ private:
     */
    void CommunicateSharedVdofs(Vector &f) const;
 
+   static void CorrectFaceOrientations(const ParFiniteElementSpace &fes,
+                                       const Vector &src,
+                                       Vector &dst,
+                                       const Array<int> *s2p_map = NULL);
+
    TransferCategory category_;
 
    /// Mapping of the ParGridFunction defined on the SubMesh to the
@@ -98,6 +103,13 @@ private:
    /// ParSubMesh to ParSubMesh transfer.
    std::unique_ptr<const ParFiniteElementSpace> root_fes_;
 
+   /// Pointer to the supplemental FiniteElementCollection used with root_fes_.
+   /// This is only used if this TransferMap represents a SubMesh to
+   /// SubMesh transfer where the root requires a different type of collection
+   /// than the SubMesh objects. For example, when the subpaces are L2 on
+   /// boundaries of the parent mesh and the root space can be RT.
+   std::unique_ptr<const FiniteElementCollection> root_fec_;
+
    const GroupCommunicator *root_gc_ = nullptr;
 
    /// Temporary vector

mesh/submesh/submesh.cpp

@@ -61,6 +61,7 @@ SubMesh::SubMesh(const Mesh &parent, From from,
                                                     parent_element_ids_);
 
       Array<int> parent_face_to_be = parent.GetFaceToBdrElMap();
+      int max_bdr_attr = parent.bdr_attributes.Max();
 
       for (int i = 0; i < NumOfBdrElements; i++)
       {
@@ -75,7 +76,73 @@ SubMesh::SubMesh(const Mesh &parent, From from,
             // This case happens when a domain is extracted, but the root parent
             // mesh didn't have a boundary element on the surface that defined
             // it's boundary. It still creates a valid mesh, so we allow it.
-            GetBdrElement(i)->SetAttribute(GENERATED_ATTRIBUTE);
+            GetBdrElement(i)->SetAttribute(max_bdr_attr + 1);
+         }
+      }
+
+      parent_face_ori_.SetSize(NumOfFaces);
+
+      for (int i = 0; i < NumOfFaces; i++)
+      {
+         Array<int> sub_vert;
+         GetFaceVertices(i, sub_vert);
+
+         Array<int> sub_par_vert(sub_vert.Size());
+         for (int j = 0; j < sub_vert.Size(); j++)
+         {
+            sub_par_vert[j] = parent_vertex_ids_[sub_vert[j]];
+         }
+
+         Array<int> par_vert;
+         parent.GetFaceVertices(parent_face_ids_[i], par_vert);
+
+         if (par_vert.Size() == 3)
+         {
+            parent_face_ori_[i] = GetTriOrientation(par_vert, sub_par_vert);
+         }
+         else
+         {
+            parent_face_ori_[i] = GetQuadOrientation(par_vert, sub_par_vert);
+         }
+      }
+   }
+   else if (Dim == 2)
+   {
+      parent_face_ori_.SetSize(NumOfElements);
+
+      for (int i = 0; i < NumOfElements; i++)
+      {
+         Array<int> sub_vert;
+         GetElementVertices(i, sub_vert);
+
+         Array<int> sub_par_vert(sub_vert.Size());
+         for (int j = 0; j < sub_vert.Size(); j++)
+         {
+            sub_par_vert[j] = parent_vertex_ids_[sub_vert[j]];
+         }
+
+         Array<int> par_vert;
+         int be_ori = 0;
+         if (from == From::Boundary)
+         {
+            parent.GetBdrElementVertices(parent_element_ids_[i], par_vert);
+
+            int f = -1;
+            parent.GetBdrElementFace(parent_element_ids_[i], &f, &be_ori);
+         }
+         else
+         {
+            parent.GetElementVertices(parent_element_ids_[i], par_vert);
+         }
+
+         if (par_vert.Size() == 3)
+         {
+            int se_ori = GetTriOrientation(par_vert, sub_par_vert);
+            parent_face_ori_[i] = ComposeTriOrientations(be_ori, se_ori);
+         }
+         else
+         {
+            parent_face_ori_[i] = GetQuadOrientation(par_vert, sub_par_vert);
          }
       }
    }

mesh/submesh/submesh.hpp

@@ -120,6 +120,16 @@ public:
       return parent_face_ids_;
    }
 
+   /**
+    * @brief Get the relative face orientations
+    *
+    * SubMesh element id (array index) to parent Mesh face orientation.
+    */
+   const Array<int>& GetParentFaceOrientations() const
+   {
+      return parent_face_ori_;
+   }
+
    /**
     * @brief Get the parent vertex id map.
     *
@@ -193,6 +203,10 @@ private:
    /// face ids.
    Array<int> parent_face_ids_;
 
+   /// Mapping from SubMesh face ids (index of the array), to the orientation
+   /// of the face relative to the parent face.
+   Array<int> parent_face_ori_;
+
    Array<int> face_to_be;
 };
 

mesh/submesh/submesh_utils.cpp

@@ -127,7 +127,8 @@ void BuildVdofToVdofMap(const FiniteElementSpace& subfes,
                Tr.Transf,
                face_info);
 
-            Geometry::Type face_geom = pm->GetBdrElementBaseGeometry(i);
+            Geometry::Type face_geom =
+               pm->GetBdrElementBaseGeometry(parent_element_ids[i]);
             const FiniteElement *face_el =
                parentfes.GetTraceElement(parent_element_ids[i], face_geom);
             MFEM_VERIFY(dynamic_cast<const NodalFiniteElement*>(face_el),
@@ -169,10 +170,18 @@ void BuildVdofToVdofMap(const FiniteElementSpace& subfes,
 
       Array<int> sub_vdofs;
       subfes.GetElementVDofs(i, sub_vdofs);
+
       MFEM_ASSERT(parent_vdofs.Size() == sub_vdofs.Size(), "internal error");
       for (int j = 0; j < parent_vdofs.Size(); j++)
       {
-         vdof_to_vdof_map[sub_vdofs[j]] = parent_vdofs[j];
+         double sub_sign = 1.0;
+         int sub_vdof = subfes.DecodeDof(sub_vdofs[j], sub_sign);
+
+         double parent_sign = 1.0;
+         int parent_vdof = parentfes.DecodeDof(parent_vdofs[j], parent_sign);
+
+         vdof_to_vdof_map[sub_vdof] =
+            (sub_sign * parent_sign > 0.0) ? parent_vdof : (-1-parent_vdof);
       }
    }
 }

mesh/submesh/transfermap.cpp

@@ -37,9 +37,68 @@ TransferMap::TransferMap(const GridFunction &src,
 
       category_ = TransferCategory::SubMeshToSubMesh;
 
-      root_fes_.reset(new FiniteElementSpace(
-                         *src.FESpace(),
-                         const_cast<Mesh *>(SubMeshUtils::GetRootParent(*src_sm))));
+      {
+         Mesh * parent_mesh =
+            const_cast<Mesh *>(SubMeshUtils::GetRootParent(*src_sm));
+
+         int parent_dim = parent_mesh->Dimension();
+         int src_sm_dim = src_sm->Dimension();
+         int dst_sm_dim = dst_sm->Dimension();
+
+         bool root_fes_reset = false;
+         if (src_sm_dim == parent_dim - 1 && dst_sm_dim == parent_dim - 1)
+         {
+            const FiniteElementSpace *src_fes = src.FESpace();
+            const FiniteElementSpace *dst_fes = dst.FESpace();
+
+            const FiniteElementCollection *src_fec = src_fes->FEColl();
+            const FiniteElementCollection *dst_fec = dst_fes->FEColl();
+
+            const L2_FECollection *src_l2_fec =
+               dynamic_cast<const L2_FECollection*>(src_fec);
+            const L2_FECollection *dst_l2_fec =
+               dynamic_cast<const L2_FECollection*>(dst_fec);
+
+            if (src_l2_fec != NULL && dst_l2_fec != NULL)
+            {
+               // Source and destination are both lower dimension L2 spaces.
+               // Transfer them as the trace of an RT space if possible.
+
+               int src_mt = src_fec->GetMapType(src_sm_dim);
+               int dst_mt = dst_fec->GetMapType(dst_sm_dim);
+
+               int src_bt = src_l2_fec->GetBasisType();
+               int dst_bt = dst_l2_fec->GetBasisType();
+
+               int src_p = src_fec->GetOrder();
+               int dst_p = dst_fec->GetOrder();
+
+               if (src_mt == FiniteElement::INTEGRAL &&
+                   dst_mt == FiniteElement::INTEGRAL &&
+                   src_bt == BasisType::GaussLegendre &&
+                   dst_bt == BasisType::GaussLegendre &&
+                   src_p == dst_p)
+               {
+                  // The subspaces are consistent with the trace of an RT space
+                  root_fec_.reset(new RT_FECollection(src_p, parent_dim));
+                  root_fes_.reset(new FiniteElementSpace(
+                                     const_cast<Mesh *>(
+                                        SubMeshUtils::GetRootParent(*src_sm)),
+                                     root_fec_.get()));
+                  root_fes_reset = true;
+               }
+            }
+         }
+
+         if (!root_fes_reset)
+         {
+            root_fes_.reset(new FiniteElementSpace(
+                               *src.FESpace(),
+                               const_cast<Mesh *>(
+                                  SubMeshUtils::GetRootParent(*src_sm))));
+         }
+      }
+
       subfes1 = src.FESpace();
       subfes2 = dst.FESpace();
 
@@ -95,8 +154,12 @@ void TransferMap::Transfer(const GridFunction &src,
       // dst = S1^T src
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
-         dst(i) = src(sub1_to_parent_map_[i]);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub1_to_parent_map_[i], s);
+         dst(i) = s * src(j);
       }
+
+      CorrectFaceOrientations(*dst.FESpace(), src, dst);
    }
    else if (category_ == TransferCategory::SubMeshToParent)
    {
@@ -107,8 +170,13 @@ void TransferMap::Transfer(const GridFunction &src,
 
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
-         dst(sub1_to_parent_map_[i]) = src(i);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub1_to_parent_map_[i], s);
+         dst(j) = s * src(i);
       }
+
+      CorrectFaceOrientations(*src.FESpace(), src, dst,
+                              &sub1_to_parent_map_);
    }
    else if (category_ == TransferCategory::SubMeshToSubMesh)
    {
@@ -120,21 +188,114 @@ void TransferMap::Transfer(const GridFunction &src,
 
       for (int i = 0; i < sub2_to_parent_map_.Size(); i++)
       {
-         z_(sub2_to_parent_map_[i]) = dst(i);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub2_to_parent_map_[i], s);
+         z_(j) = s * dst(i);
       }
 
+      CorrectFaceOrientations(*dst.FESpace(), dst, z_,
+                              &sub2_to_parent_map_);
+
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
-         z_(sub1_to_parent_map_[i]) = src(i);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub1_to_parent_map_[i], s);
+         z_(j) = s * src(i);
       }
 
+      CorrectFaceOrientations(*src.FESpace(), src, z_,
+                              &sub1_to_parent_map_);
+
       for (int i = 0; i < sub2_to_parent_map_.Size(); i++)
       {
-         dst(i) = z_(sub2_to_parent_map_[i]);
+         double s = 1.0;
+         int j = FiniteElementSpace::DecodeDof(sub2_to_parent_map_[i], s);
+         dst(i) = s * z_(j);
       }
+
+      CorrectFaceOrientations(*dst.FESpace(), z_, dst);
    }
    else
    {
       MFEM_ABORT("unknown TransferCategory: " << category_);
    }
 }
+
+void TransferMap::CorrectFaceOrientations(const FiniteElementSpace &fes,
+                                          const Vector &src,
+                                          Vector &dst,
+                                          const Array<int> *sub_to_parent_map)
+{
+   const FiniteElementCollection * fec = fes.FEColl();
+
+   SubMesh * mesh = dynamic_cast<SubMesh*>(fes.GetMesh());
+
+   const Array<int>& parent_face_ori = mesh->GetParentFaceOrientations();
+
+   if (parent_face_ori.Size() == 0) { return; }
+
+   VDofTransformation vdoftrans(fes.GetVDim(),
+                                fes.GetOrdering());
+
+   int dim = mesh->Dimension();
+   bool face = (dim == 3);
+
+   Array<int> vdofs;
+   Array<int> Fo(1);
+   Vector face_vector;
+
+   for (int i = 0; i < (face ? mesh->GetNumFaces() : mesh->GetNE()); i++)
+   {
+      if (parent_face_ori[i] == 0) { continue; }
+
+      Geometry::Type geom = face ? mesh->GetFaceGeometry(i) :
+                            mesh->GetElementGeometry(i);;
+
+      StatelessDofTransformation * doftrans =
+         fec->DofTransformationForGeometry(geom);
+
+      if (doftrans == NULL) { continue; }
+
+      vdoftrans.SetDofTransformation(*doftrans);
+
+      Fo[0] = parent_face_ori[i];
+      vdoftrans.SetFaceOrientations(Fo);
+
+      if (face)
+      {
+         fes.GetFaceVDofs(i, vdofs);
+      }
+      else
+      {
+         fes.GetElementVDofs(i, vdofs);
+      }
+
+      if (sub_to_parent_map)
+      {
+         src.GetSubVector(vdofs, face_vector);
+         vdoftrans.TransformPrimal(face_vector);
+      }
+      else
+      {
+         dst.GetSubVector(vdofs, face_vector);
+         vdoftrans.InvTransformPrimal(face_vector);
+      }
+
+      for (int j = 0; j < vdofs.Size(); j++)
+      {
+         double s = 1.0;
+         int k = FiniteElementSpace::DecodeDof(vdofs[j], s);
+
+         if (sub_to_parent_map)
+         {
+            double sps = 1.0;
+            int spk = FiniteElementSpace::DecodeDof((*sub_to_parent_map)[k],
+                                                    sps);
+            s *= sps;
+            k = spk;
+         }
+
+         dst[k] = s * face_vector[j];
+      }
+   }
+}

mesh/submesh/transfermap.hpp

@@ -52,6 +52,12 @@ public:
    void Transfer(const GridFunction &src, GridFunction &dst) const;
 
 private:
+
+   static void CorrectFaceOrientations(const FiniteElementSpace &fes,
+                                       const Vector &src,
+                                       Vector &dst,
+                                       const Array<int> *s2p_map = NULL);
+
    TransferCategory category_;
 
    /// Mapping of the GridFunction defined on the SubMesh to the Gridfunction
@@ -68,6 +74,13 @@ private:
    /// SubMesh transfer.
    std::unique_ptr<const FiniteElementSpace> root_fes_;
 
+   /// Pointer to the supplemental FiniteElementCollection used with root_fes_.
+   /// This is only used if this TransferMap represents a SubMesh to
+   /// SubMesh transfer where the root requires a different type of collection
+   /// than the SubMesh objects. For example, when the subpaces are L2 on
+   /// boundaries of the parent mesh and the root space can be RT.
+   std::unique_ptr<const FiniteElementCollection> root_fec_;
+
    /// Temporary vector
    mutable Vector z_;
 };

miniapps/contact/contact_driver.cpp

@@ -0,0 +1,151 @@
+//                                Contact example
+//
+// Compile with: make contact
+//
+// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
+// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ipsolver/IPsolver.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file1 = "meshes/block1.mesh";
+   const char *mesh_file2 = "meshes/rotatedblock2.mesh";
+   int order = 1;
+   int ref = 0;
+   Array<int> attr;
+   Array<int> m_attr;
+   int linSolver = 2;
+   bool paraview = false;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file1, "-m1", "--mesh1",
+                  "First mesh file to use.");
+   args.AddOption(&mesh_file2, "-m2", "--mesh2",
+                  "Second mesh file to use.");
+   args.AddOption(&attr, "-at", "--attributes-surf",
+                  "Attributes of boundary faces on contact surface for mesh 2.");
+   args.AddOption(&ref, "-r", "--refinements",
+                  "Number of uniform refinements.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   ElasticityProblem prob1(mesh_file1,ref,order);
+   ElasticityProblem prob2(mesh_file2,ref,order);
+
+   ContactProblem contact(&prob1, &prob2);
+   QPOptContactProblem qpopt(&contact);
+   int numconstr = contact.GetNumConstraints();
+
+   InteriorPointSolver optimizer(&qpopt);
+   optimizer.SetTol(1e-6);
+   optimizer.SetMaxIter(50);
+   optimizer.SetLinearSolver(linSolver);
+   optimizer.SetLinearSolveTol(1e-10);
+
+   GridFunction x1 = prob1.GetDisplacementGridFunction();
+   GridFunction x2 = prob2.GetDisplacementGridFunction();
+
+   int ndofs1 = prob1.GetNumDofs();
+   int ndofs2 = prob2.GetNumDofs();
+   int ndofs = ndofs1 + ndofs2;
+
+   Vector x0(ndofs); x0 = 0.0;
+   x0.SetVector(x1,0);
+   x0.SetVector(x2,x1.Size());
+
+   Vector xf(ndofs); xf = 0.0;
+   optimizer.Mult(x0, xf);
+   Array<int> & CGiterations = optimizer.GetCGIterNumbers();
+
+   double Einitial = contact.E(x0);
+   double Efinal = contact.E(xf);
+
+   mfem::out << endl;
+   mfem::out << " Initial Energy objective     = " << Einitial << endl;
+   mfem::out << " Final Energy objective       = " << Efinal << endl;
+   mfem::out << " Global number of dofs        = " << ndofs1 + ndofs2 << endl;
+   mfem::out << " Global number of constraints = " << numconstr << endl;
+   mfem::out << " CG iteration numbers         = " ;
+   CGiterations.Print(mfem::out, CGiterations.Size());
+
+   MFEM_VERIFY(optimizer.GetConverged(),
+               "Interior point solver did not converge.");
+
+   if (visualization || paraview)
+   {
+      FiniteElementSpace * fes1 = prob1.GetFESpace();
+      FiniteElementSpace * fes2 = prob2.GetFESpace();
+
+      Mesh * mesh1 = fes1->GetMesh();
+      Mesh * mesh2 = fes2->GetMesh();
+
+      GridFunction x1_gf(fes1,xf.GetData());
+      GridFunction x2_gf(fes2,&xf.GetData()[fes1->GetTrueVSize()]);
+
+      mesh1->MoveNodes(x1_gf);
+      mesh2->MoveNodes(x2_gf);
+
+      if (paraview)
+      {
+         ParaViewDataCollection paraview_dc1("QPContactBody1", mesh1);
+         paraview_dc1.SetPrefixPath("ParaView");
+         paraview_dc1.SetLevelsOfDetail(1);
+         paraview_dc1.SetDataFormat(VTKFormat::BINARY);
+         paraview_dc1.SetHighOrderOutput(true);
+         paraview_dc1.SetCycle(0);
+         paraview_dc1.SetTime(0.0);
+         paraview_dc1.RegisterField("Body1", &x1_gf);
+         paraview_dc1.Save();
+
+         ParaViewDataCollection paraview_dc2("QPContactBody2", mesh2);
+         paraview_dc2.SetPrefixPath("ParaView");
+         paraview_dc2.SetLevelsOfDetail(1);
+         paraview_dc2.SetDataFormat(VTKFormat::BINARY);
+         paraview_dc2.SetHighOrderOutput(true);
+         paraview_dc2.SetCycle(0);
+         paraview_dc2.SetTime(0.0);
+         paraview_dc2.RegisterField("Body2", &x2_gf);
+         paraview_dc2.Save();
+      }
+
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int visport = 19916;
+         {
+            socketstream sol_sock(vishost, visport);
+            sol_sock.precision(8);
+            sol_sock << "parallel " << 2 << " " << 0 << "\n"
+                     << "solution\n" << *mesh1 << x1_gf << flush;
+         }
+         {
+            socketstream sol_sock(vishost, visport);
+            sol_sock.precision(8);
+            sol_sock << "parallel " << 2 << " " << 1 << "\n"
+                     << "solution\n" << *mesh2 << x2_gf << flush;
+         }
+      }
+   }
+
+   return 0;
+}

miniapps/contact/ipsolver/IPsolver.cpp

@@ -0,0 +1,818 @@
+#include "mfem.hpp"
+#include "IPsolver.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstdlib>
+
+using namespace std;
+using namespace mfem;
+
+
+
+InteriorPointSolver::InteriorPointSolver(QPOptContactProblem * Problem) 
+: optProblem(Problem), block_offsetsumlz(5), block_offsetsuml(4), block_offsetsx(3),
+saveLogBarrierIterates(false)
+{
+   rel_tol  = 1.e-2;
+   max_iter = 20;
+   mu_k     = 1.0;
+
+   sMax     = 1.e2;
+   kSig     = 1.e10;   // control deviation from primal Hessian
+   tauMin   = 0.8;     // control rate at which iterates can approach the boundary
+   eta      = 1.e-4;   // backtracking constant
+   thetaMin = 1.e-4;   // allowed violation of the equality constraints
+
+   // constants in line-step A-5.4
+   delta    = 1.0;
+   sTheta   = 1.1;
+   sPhi     = 2.3;
+
+   // control the rate at which the penalty parameter is decreased
+   kMu     = 0.2;
+   thetaMu = 1.5;
+
+   // TO DO -- include the filter
+
+   thetaMax = 1.e6; // maximum constraint violation
+   // data for the second order correction
+   kSoc     = 0.99;
+
+   // equation (18)
+   gTheta = 1.e-5;
+   gPhi   = 1.e-5;
+
+   kEps   = 1.e1;
+
+   dimU = optProblem->GetDimU();
+   dimM = optProblem->GetDimM();
+   dimC = optProblem->GetDimC();
+   ckSoc.SetSize(dimC);
+   
+   block_offsetsumlz[0] = 0;
+   block_offsetsumlz[1] = dimU; // u
+   block_offsetsumlz[2] = dimM; // m
+   block_offsetsumlz[3] = dimC; // lambda
+   block_offsetsumlz[4] = dimM; // zl
+   block_offsetsumlz.PartialSum();
+
+   for(int i = 0; i < block_offsetsuml.Size(); i++)  { block_offsetsuml[i] = block_offsetsumlz[i]; }
+   for(int i = 0; i < block_offsetsx.Size(); i++)    { block_offsetsx[i]   = block_offsetsuml[i] ; }
+
+   // lower-bound for the inequality constraint m >= ml
+   ml = optProblem->Getml();
+   
+   lk.SetSize(dimC);  lk  = 0.0;
+   zlk.SetSize(dimM); zlk = 0.0;
+
+   linSolver = 0;
+   MyRank = 0;
+   iAmRoot = MyRank == 0 ? true : false;
+}
+
+double InteriorPointSolver::MaxStepSize(Vector &x, Vector &xl, Vector &xhat, double tau)
+{
+   double alphaMaxloc = 1.0;
+   double alphaTmp;
+   for(int i = 0; i < x.Size(); i++)
+   {
+      if( xhat(i) < 0. )
+      {
+         alphaTmp = -1. * tau * (x(i) - xl(i)) / xhat(i);
+         alphaMaxloc = min(alphaMaxloc, alphaTmp);
+      } 
+   }
+
+   // alphaMaxloc is the local maximum step size which is
+   // distinct on each MPI process. Need to compute
+   // the global maximum step size 
+   double alphaMaxglb;
+   alphaMaxglb = alphaMaxloc;
+   return alphaMaxglb;
+}
+
+double InteriorPointSolver::MaxStepSize(Vector &x, Vector &xhat, double tau)
+{
+   Vector zero(x.Size()); zero = 0.0;
+   return MaxStepSize(x, zero, xhat, tau);
+}
+
+
+void InteriorPointSolver::Mult(const Vector &x0, Vector &xf)
+{
+   BlockVector x0block(block_offsetsx); x0block = 0.0;
+   x0block.GetBlock(0).Set(1.0, x0);
+   // To do: give options for user specificiation of initialization m0
+   x0block.GetBlock(1) = 1.0;
+   x0block.GetBlock(1).Add(1.0, ml);
+   BlockVector xfblock(block_offsetsx); xfblock = 0.0;
+   Mult(x0block, xfblock);
+   xf.Set(1.0, xfblock.GetBlock(0));
+}
+
+void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
+{
+   converged = false;
+   
+   BlockVector xk(block_offsetsx), xhat(block_offsetsx); xk = 0; xhat = 0.0;
+   BlockVector Xk(block_offsetsumlz), Xhat(block_offsetsumlz); Xk = 0.0; Xhat = 0.0;
+   BlockVector Xhatuml(block_offsetsuml); Xhatuml = 0.0;
+   Vector zlhat(dimM); zlhat = 0.0;
+
+   xk.GetBlock(0).Set(1.0, x0.GetBlock(0));
+   xk.GetBlock(1).Set(1.0, x0.GetBlock(1));
+   // running estimate of the final values of the Lagrange multipliers
+   lk  = 0.0;
+   zlk = 0.0;
+
+   for(int i = 0; i < dimM; i++)
+   {
+      zlk(i) = 1.e1 * mu_k / (xk(i+dimU) - ml(i));
+   }
+
+   Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+   Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+   Xk.GetBlock(2).Set(1.0, lk);
+   Xk.GetBlock(3).Set(1.0, zlk);
+
+   /* set theta0 = theta(x0)
+      *     thetaMin
+      *     thetaMax
+      * when theta(xk) < thetaMin and the switching condition holds
+      * then we ask for the Armijo sufficient decrease of the barrier
+      * objective to be satisfied, in order to accept the trial step length alphakl
+      * 
+      * thetaMax controls how the filter is initialized for each log-barrier subproblem
+      * F0 = {(th, phi) s.t. th > thetaMax}
+      * that is the filter does not allow for iterates where the constraint violation
+      * is larger than that of thetaMax
+      */
+   double theta0 = theta(xk);
+   thetaMin = 1.e-4 * max(1.0, theta0);
+   thetaMax = 1.e8  * thetaMin;
+
+   double Eeval, maxBarrierSolves, Eevalmu0;
+   bool printOptimalityError; // control optimality error print to console for log-barrier subproblems
+   
+   maxBarrierSolves = 10;
+
+   for(jOpt = 0; jOpt < max_iter; jOpt++)
+   {
+      mfem::out << "interior-point solve step " << jOpt << endl;
+      // A-2. Check convergence of overall optimization problem
+      printOptimalityError = false;
+      Eevalmu0 = E(xk, lk, zlk, printOptimalityError);
+      if(Eevalmu0 < rel_tol)
+      {
+         converged = true;
+         mfem::out << "solved optimization problem :)\n";
+         break;
+      }
+      
+      if(jOpt > 0) { maxBarrierSolves = 1; }
+      
+      for(int i = 0; i < maxBarrierSolves; i++)
+      {
+         // A-3. Check convergence of the barrier subproblem
+         printOptimalityError = true;
+         Eeval = E(xk, lk, zlk, mu_k, printOptimalityError);
+         if(Eeval < kEps * mu_k)
+         {
+            mfem::out << "solved barrier subproblem, for mu = " << mu_k << endl;
+            // A-3.1. Recompute the barrier parameter
+            mu_k  = max(rel_tol / 10., min(kMu * mu_k, pow(mu_k, thetaMu)));
+            // A-3.2. Re-initialize the filter
+            F1.DeleteAll();
+            F2.DeleteAll();
+         }
+         else
+         {
+            break;
+         }
+      }
+      // A-4. Compute the search direction
+      // solve for (uhat, mhat, lhat)
+      mfem::out << "\n** A-4. IP-Newton solve **\n";
+      zlhat = 0.0; Xhatuml = 0.0;
+      // why do we have Xhatuml ....???
+      // TO DO: remove Xhatuml in favor of passing Xhat
+      IPNewtonSolve(xk, lk, zlk, zlhat, Xhatuml, mu_k, false); 
+
+      // assign data stack, X = (u, m, l, zl)
+      Xk = 0.0;
+      Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+      Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+      Xk.GetBlock(2).Set(1.0, lk);
+      Xk.GetBlock(3).Set(1.0, zlk);
+
+      // assign data stack, Xhat = (uhat, mhat, lhat, zlhat)
+      Xhat = 0.0;
+      for(int i = 0; i < 3; i++)
+      {
+         Xhat.GetBlock(i).Set(1.0, Xhatuml.GetBlock(i));
+      }
+      Xhat.GetBlock(3).Set(1.0, zlhat);
+
+      // A-5. Backtracking line search.
+      mfem::out << "\n** A-5. Linesearch **\n";
+      mfem::out << "mu = " << mu_k << endl;
+      
+      lineSearch(Xk, Xhat, mu_k);
+      if(lineSearchSuccess)
+      {
+         if(!switchCondition || !sufficientDecrease)
+         {
+            F1.Append( (1. - gTheta) * thx0);
+            F2.Append( phx0 - gPhi * thx0);
+         }
+         // ----- A-6: Accept the trial point
+         // print info regarding zl...
+         xk.GetBlock(0).Add(alpha, Xhat.GetBlock(0));
+         xk.GetBlock(1).Add(alpha, Xhat.GetBlock(1));
+         lk.Add(alpha,   Xhat.GetBlock(2));
+         zlk.Add(alphaz, Xhat.GetBlock(3));
+         projectZ(xk, zlk, mu_k);
+      }
+      else
+      {
+         mfem::out << "lineSearch not successful :(\n";
+         mfem::out << "attempting feasibility restoration with theta = " << thx0 << endl;
+         mfem::out << "no feasibility restoration implemented, exiting now \n";
+         break;
+      }
+      //
+      if(jOpt + 1 == max_iter) 
+      { 
+         mfem::out << "maximum optimization iterations :(\n";
+      }
+   }
+   // done with optimization routine, just reassign data to xf reference so
+   // that the application code has access to the optimal point
+   xf = 0.0;
+   xf.GetBlock(0).Set(1.0, xk.GetBlock(0));
+   xf.GetBlock(1).Set(1.0, xk.GetBlock(1));
+}
+
+void InteriorPointSolver::FormIPNewtonMat(BlockVector & x, Vector & l, Vector &zl, BlockOperator &Ak)
+{
+   // WARNING: Huu, Hum, Hmu, Hmm should all be Hessian terms of the Lagrangian, currently we 
+   //          them by Hessian terms of the objective function and neglect the Hessian of l^T c
+
+   Huu = optProblem->Duuf(x);
+   Hum = optProblem->Dumf(x);
+   Hmu = optProblem->Dmuf(x);
+   Hmm = optProblem->Dmmf(x);
+
+   Vector DiagLogBar(dimM); DiagLogBar = 0.0;
+   for(int ii = 0; ii < dimM; ii++)
+   {
+      DiagLogBar(ii) = zl(ii) / (x(ii+dimU) - ml(ii));
+   }
+
+   if(saveLogBarrierIterates)
+   {
+      std::ofstream diagStream;
+      char diagString[100];
+      snprintf(diagString, 100, "logBarrierHessiandata/D%d.dat", jOpt);
+      diagStream.open(diagString, ios::out | ios::trunc);
+      for(int ii = 0; ii < dimM; ii++)
+      {
+         diagStream << setprecision(30) << DiagLogBar(ii) << endl;
+      }
+      diagStream.close();
+   } 
+
+   delete Wmm;
+   if(Hmm != nullptr)
+   {
+      SparseMatrix * D = new SparseMatrix(DiagLogBar);
+      Wmm = Add(*Hmm, *D);
+      delete D;
+   }
+   else
+   {
+      Wmm = new SparseMatrix(DiagLogBar);
+   }
+
+   delete JuT;
+   delete JmT;
+   Ju = optProblem->Duc(x); JuT = Transpose(*Ju);
+   Jm = optProblem->Dmc(x); JmT = Transpose(*Jm);
+   
+   Huucl = optProblem->lDuuc(x, l);
+   if(Huucl != nullptr)
+   {
+      delete HLuucl;
+      HLuucl = Add(*Huucl, *Huu);
+      Ak.SetBlock(0, 0, HLuucl);
+   } 
+   else
+   {
+      Ak.SetBlock(0, 0, Huu);
+   }
+
+   //         IP-Newton system matrix
+   //    Ak = [[H_(u,u)  H_(u,m)   J_u^T]
+   //          [H_(m,u)  W_(m,m)   J_m^T]
+   //          [ J_u      J_m       0  ]]
+
+   Ak.SetBlock(0, 0, Huu);
+   Ak.SetBlock(0, 2, JuT);
+   Ak.SetBlock(1, 1, Wmm);   
+   Ak.SetBlock(1, 2, JmT);
+   Ak.SetBlock(2, 0, Ju); 
+   Ak.SetBlock(2, 1, Jm); 
+
+   if(Hum != nullptr) { Ak.SetBlock(0, 1, Hum); Ak.SetBlock(1, 0, Hmu); }
+}
+
+// perturbed KKT system solve
+// determine the search direction
+void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, Vector &zlhat, BlockVector &Xhat, double mu, bool socSolve)
+{
+   // solve A x = b, where A is the IP-Newton matrix
+   BlockOperator A(block_offsetsuml, block_offsetsuml); BlockVector b(block_offsetsuml); b = 0.0;
+   FormIPNewtonMat(x, l, zl, A);
+
+   //       [grad_u phi + Ju^T l]
+   // b = - [grad_m phi + Jm^T l]
+   //       [          c        ]
+   BlockVector gradphi(block_offsetsx); gradphi = 0.0;
+   BlockVector JTl(block_offsetsx); JTl = 0.0;
+   Dxphi(x, mu, gradphi);
+  
+   (A.GetBlock(0,2)).Mult(l, JTl.GetBlock(0));
+   (A.GetBlock(1,2)).Mult(l, JTl.GetBlock(1));
+
+   
+   for(int ii = 0; ii < 2; ii++)
+   {
+      b.GetBlock(ii).Set(1.0, gradphi.GetBlock(ii));
+      b.GetBlock(ii).Add(1.0, JTl.GetBlock(ii));
+   }
+   if(!socSolve) 
+   {
+      optProblem->c(x, b.GetBlock(2));
+   }
+   else
+   {
+      b.GetBlock(2).Set(1.0, ckSoc);
+   }
+   b *= -1.0; 
+   Xhat = 0.0;
+
+
+#ifdef MFEM_USE_SUITESPARSE
+   // Direct solve for IP-Newton saddle-point system
+   // A = [ [ Huu   0    Ju^T]
+   //       [  0    D     -I ]
+   //       [ Ju   -I      0 ]]
+   // if(linSolver == 0)
+   // {
+   //    BlockMatrix ABlockMatrix(block_offsetsuml, block_offsetsuml);
+   //    for(int ii = 0; ii < 3; ii++)
+   //    {
+   //       for(int jj = 0; jj < 3; jj++)
+   //       {
+   //          if(!A.IsZeroBlock(ii, jj))
+   //          {
+   //             ABlockMatrix.SetBlock(ii, jj, dynamic_cast<SparseMatrix *>(&(A.GetBlock(ii, jj))));
+   //          }
+   //       }
+   //    }
+   //    /* direct solve of the 3x3 IP-Newton linear system */
+   //    UMFPackSolver ASolver;
+   //    SparseMatrix *ASparse = ABlockMatrix.CreateMonolithic();
+   //    ASolver.SetOperator(*ASparse);
+   //    ASolver.Mult(b, Xhat);
+
+   //    Vector residual(Xhat.Size());
+   //    ASparse->Mult(Xhat, residual);
+   //    residual.Add(-1.0, b);
+   //    delete ASparse;
+   // }
+   // else if(linSolver == 1)
+   // {
+   //    // Direct solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+   //    // where Wmm = D for contact problems
+   //    SparseMatrix * Huuloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0)));
+   //    SparseMatrix * Wmmloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1)));
+   //    SparseMatrix * Juloc  = dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0)));
+   //    SparseMatrix * JuTloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2)));
+   //    Vector DVec(dimM); DVec = 0.0;
+   //    Vector one(dimM); one = 1.0;
+   //    D->Mult(one, DVec);
+   //    SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, DVec);     // Ju^T D Ju
+   //    SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+      
+   //    /* prepare the reduced rhs */
+   //    // breduced = bu + Ju^T (bm + Wmm bl)
+   //    Vector breduced(dimU); breduced = 0.0;
+   //    Vector tempVec(dimM); tempVec = 0.0;
+   //    Wmmloc->Mult(b.GetBlock(2), tempVec);
+   //    tempVec.Add(1.0, b.GetBlock(1));
+   //    JuTloc->Mult(tempVec, breduced);
+   //    breduced.Add(1.0, b.GetBlock(0));
+
+   //    // solve the reduced linear system
+   //    UMFPackSolver AreducedSolver;
+   //    AreducedSolver.SetOperator(*Areduced);
+   //    AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+
+   //    // now propagate solved uhat to obtain mhat and lhat
+   //    // xm = Ju xu - bl
+   //    Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+   //    Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+   //    // xl = Wmm xm - bm
+   //    Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+   //    Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+   //    delete JuTDJu;
+   //    delete Areduced;
+   // }
+#else
+   MFEM_VERIFY(linSolver > 1, "linSolver = 0, 1 require MFEM_USE_SUITESPARSE=YES");
+#endif
+   // if(linSolver ==2)
+   { 
+      // Iterative solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+      // where Wmm = D for contact problems
+      // here the iterative solver is a Jacobi-preconditioned CG-solve
+      SparseMatrix * Huuloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0)));
+      SparseMatrix * Wmmloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1)));
+      SparseMatrix * Juloc  = dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0)));
+      SparseMatrix * JuTloc = dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2)));
+
+      SparseMatrix *JuTDJu   = RAP(*Juloc,*Wmmloc,*Juloc);     // Ju^T D Ju
+      SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+
+      /* prepare the reduced rhs */
+      // breduced = bu + Ju^T (bm + Wmm bl)
+      Vector breduced(dimU); breduced = 0.0;
+      Vector tempVec(dimM); tempVec = 0.0;
+      Wmmloc->SortColumnIndices();
+
+      Wmmloc->Mult(b.GetBlock(2), tempVec);
+      tempVec.Add(1.0, b.GetBlock(1));
+      JuTloc->Mult(tempVec, breduced);
+
+
+      breduced.Add(1.0, b.GetBlock(0));
+      int globalNumRows = dimU;
+      HYPRE_BigInt rowStarts[2];
+      rowStarts[0] = 0;
+      rowStarts[1] = dimU;
+
+      HypreParMatrix Ahypre(MPI_COMM_WORLD, globalNumRows, rowStarts, Areduced);
+      HypreBoomerAMG Aprec(Ahypre);
+      Aprec.SetPrintLevel(0);
+      Aprec.SetSystemsOptions(3,false);
+      HyprePCG AreducedSolver(MPI_COMM_WORLD);
+      AreducedSolver.SetOperator(Ahypre);
+      // AreducedSolver.SetRelTol(linSolveTol);
+      // AreducedSolver.SetRelTol(1e-6);
+      AreducedSolver.SetTol(1e-6);
+      AreducedSolver.SetMaxIter(1000);
+      AreducedSolver.SetPreconditioner(Aprec);
+      // AreducedSolver.SetResidualConvergenceOptions();
+      AreducedSolver.SetPrintLevel(2);
+
+      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+      int num_iterations;
+      AreducedSolver.GetNumIterations(num_iterations);
+      cgnum_iterations.Append(num_iterations);
+    
+      // now propagate solved uhat to obtain mhat and lhat
+      // xm = Ju xu - bl
+      Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+      Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+      // xl = Wmm xm - bm
+      Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+      Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+      delete JuTDJu;
+      delete Areduced;
+   }
+    
+   /* backsolve to determine zlhat */
+   for(int ii = 0; ii < dimM; ii++)
+   {
+      zlhat(ii) = -1.*(zl(ii) + (zl(ii) * Xhat(ii + dimU) - mu) / (x(ii + dimU) - ml(ii)) );
+   }
+}
+
+// here Xhat, X will be BlockVectors w.r.t. the 4 partitioning X = (u, m, l, zl)
+
+void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double mu)
+{
+  double tau  = max(tauMin, 1.0 - mu);
+  Vector u0   = X0.GetBlock(0);
+  Vector m0   = X0.GetBlock(1);
+  Vector l0   = X0.GetBlock(2);
+  Vector z0   = X0.GetBlock(3);
+  Vector uhat = Xhat.GetBlock(0);
+  Vector mhat = Xhat.GetBlock(1);
+  Vector lhat = Xhat.GetBlock(2);
+  Vector zhat = Xhat.GetBlock(3);
+  double alphaMax  = MaxStepSize(m0, ml, mhat, tau);
+  double alphaMaxz = MaxStepSize(z0, zhat, tau);
+  alphaz = alphaMaxz;
+
+
+  BlockVector x0(block_offsetsx); x0 = 0.0;
+  x0.GetBlock(0).Set(1.0, u0);
+  x0.GetBlock(1).Set(1.0, m0);
+  
+  BlockVector xhat(block_offsetsx); xhat = 0.0;
+  xhat.GetBlock(0).Set(1.0, uhat);
+  xhat.GetBlock(1).Set(1.0, mhat);
+  
+  BlockVector xtrial(block_offsetsx); xtrial = 0.0;
+  BlockVector Dxphi0(block_offsetsx); Dxphi0 = 0.0;
+  int maxBacktrack = 20;
+  alpha = alphaMax;
+
+
+  Vector ck0(dimC); ck0 = 0.0;
+  Vector zhatsoc(dimM); zhatsoc = 0.0;
+  BlockVector Xhatumlsoc(block_offsetsuml); Xhatumlsoc = 0.0;
+  BlockVector xhatsoc(block_offsetsx); xhatsoc = 0.0;
+  Vector uhatsoc(dimU); uhatsoc = 0.0;
+  Vector mhatsoc(dimM); mhatsoc = 0.0;
+
+  Dxphi(x0, mu, Dxphi0);
+  Dxphi0_xhat = InnerProduct(Dxphi0, xhat);
+  descentDirection = Dxphi0_xhat < 0. ? true : false;
+  if(descentDirection)
+  {
+    mfem::out << "is a descent direction for the log-barrier objective\n";
+  }
+  else
+  {
+    mfem::out << "is not a descent direction for the log-barrier objective\n";
+  }
+  mfem::out << "Dxphi^T xhat / (|| Dxphi||_2 * || xhat ||_2) = " << Dxphi0_xhat / (xhat.Norml2() * Dxphi0.Norml2()) << endl;
+  thx0 = theta(x0);
+  phx0 = phi(x0, mu);
+
+  lineSearchSuccess = false;
+  for(int i = 0; i < maxBacktrack; i++)
+  {
+    mfem::out << "\n--------- alpha = " << alpha << " ---------\n";
+
+    // ----- A-5.2. Compute trial point: xtrial = x0 + alpha_i xhat
+    xtrial.Set(1.0, x0);
+    xtrial.Add(alpha, xhat);
+
+    // ------ A-5.3. if not in filter region go to A.5.4 otherwise go to A-5.5.
+    thxtrial = theta(xtrial);
+    phxtrial = phi(xtrial, mu);
+
+    filterCheck(thxtrial, phxtrial);    
+    if(!inFilterRegion)
+    {
+      mfem::out << "not in filter region :)\n";
+      // ------ A.5.4: Check sufficient decrease
+      if(!descentDirection)
+      {
+        switchCondition = false;
+      }
+      else
+      {
+        switchCondition = (alpha * pow(abs(Dxphi0_xhat), sPhi) > delta * pow(thx0, sTheta)) ? true : false;
+      }
+      mfem::out << "theta(x0) = "     << thx0     << ", thetaMin = "                  << thetaMin             << endl;
+      mfem::out << "theta(xtrial) = " << thxtrial << ", (1-gTheta) *theta(x0) = "     << (1. - gTheta) * thx0 << endl;
+      mfem::out << "phi(xtrial) = "   << phxtrial << ", phi(x0) - gPhi *theta(x0) = " << phx0 - gPhi * thx0   << endl;
+      
+      // Case I      
+      if(thx0 <= thetaMin && switchCondition)
+      {
+        sufficientDecrease = phxtrial <= phx0 + eta * alpha * Dxphi0_xhat ? true : false;
+        if(sufficientDecrease)
+        {
+          mfem::out << "Accepted step length -- sufficient decrease in log-barrier objective.\n"; 
+          // accept the trial step
+          lineSearchSuccess = true;
+          break;
+        }
+      }
+      else
+      {
+        if(thxtrial <= (1. - gTheta) * thx0 || phxtrial <= phx0 - gPhi * thx0)
+        {
+          mfem::out << "Accepted step length -- decrease in either constraint violation or log-barrier objective.\n"; 
+          // accept the trial step
+          lineSearchSuccess = true;
+          break;
+        }
+      }
+      // A-5.5: Initialize the second-order correction
+      if((!(thx0 < thxtrial)) && i == 0)
+      {
+        mfem::out << "second order correction\n";
+        optProblem->c(xtrial, ckSoc);
+        optProblem->c(x0, ck0);
+        ckSoc.Add(alphaMax, ck0);
+        // A-5.6 Compute the second-order correction.
+        IPNewtonSolve(x0, l0, z0, zhatsoc, Xhatumlsoc, mu, true);
+        mhatsoc.Set(1.0, Xhatumlsoc.GetBlock(1));
+        // alphasoc = MaxStepSize(m0, ml, mhatsoc, tau);
+        //WARNING: not complete but currently solver isn't entering this region
+      }
+    }
+    else
+    {
+      mfem::out << "in filter region\n";
+    }
+
+    // include more if needed
+    alpha *= 0.5;
+
+  } 
+}
+
+
+void InteriorPointSolver::projectZ(const Vector &x, Vector &z, double mu)
+{
+  double zi;
+  double mudivmml;
+  for(int i = 0; i < dimM; i++)
+  {
+    zi = z(i);
+    mudivmml = mu / (x(i + dimU) - ml(i));
+    z(i) = max(min(zi, kSig * mudivmml), mudivmml / kSig);
+  }
+}
+
+void InteriorPointSolver::filterCheck(double th, double ph)
+{
+  inFilterRegion = false;
+  if(th > thetaMax)
+  {
+    inFilterRegion = true;
+  }
+  else
+  {
+    for(int i = 0; i < F1.Size(); i++)
+    {
+      if(th >= F1[i] && ph >= F2[i])
+      {
+        inFilterRegion = true;
+        break;
+      }
+    }
+  }
+}
+
+double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, double mu, bool print)
+{
+  double E1, E2, E3;
+  double sc, sd;
+  BlockVector gradL(block_offsetsx); gradL = 0.0; // stationarity grad L = grad f + J^T l - z
+  Vector cx(dimC); cx = 0.0;     // feasibility c = c(x)
+  Vector comp(dimM); comp = 0.0; // complementarity M Z - mu 1
+
+  DxL(x, l, zl, gradL);
+  E1 = gradL.Normlinf();
+
+  optProblem->c(x, cx);
+  E2 = cx.Normlinf();
+
+  for(int ii = 0; ii < dimM; ii++) 
+  { 
+    comp(ii) = x(dimU + ii) * zl(ii) - mu;
+  }
+  E3 = comp.Normlinf();
+
+  double ll1, zl1;
+  zl1 = zl.Norml1() / double(dimC + dimM);
+  ll1 =  l.Norml1();
+  sc = max(sMax, zl1 / (double(dimM)) ) / sMax;
+  sd = max(sMax, (ll1 + zl1) / (double(dimC + dimM))) / sMax;
+  if(print)
+  {
+    mfem::out << "evaluating optimality error for mu = " << mu << endl;
+    mfem::out << "stationarity measure = "    << E1 / sd << endl;
+    mfem::out << "feasibility measure  = "    << E2      << endl;
+    mfem::out << "complimentarity measure = " << E3 / sc << endl;
+  }
+  return max(max(E1 / sd, E2), E3 / sc);
+}
+
+double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, bool print)
+{
+  return E(x, l, zl, 0.0, print);
+}
+
+double InteriorPointSolver::theta(const BlockVector &x)
+{
+  Vector cx(dimC); cx = 0.0;
+  optProblem->c(x, cx);
+  return cx.Norml2();
+}
+
+// log-barrier objective
+double InteriorPointSolver::phi(const BlockVector &x, double mu)
+{
+  double fx = optProblem->CalcObjective(x); 
+  double logBarrierLoc = 0.0;
+  for(int i = 0; i < dimM; i++) 
+  { 
+    logBarrierLoc += log(x(dimU+i)-ml(i));
+  }
+  double logBarrierGlb = 0.0;
+  logBarrierGlb = logBarrierLoc;
+  return fx - mu * logBarrierGlb;
+}
+
+// gradient of log-barrier objective with respect to x = (u, m)
+void InteriorPointSolver::Dxphi(const BlockVector &x, double mu, BlockVector &y)
+{
+  optProblem->CalcObjectiveGrad(x, y);
+  for(int i = 0; i < dimM; i++) 
+  { 
+    y(dimU + i) -= mu / (x(dimU + i));
+  } 
+}
+
+// Lagrangian function evaluation
+// L(x, l, zl) = f(x) + l^T c(x) - zl^T m
+double InteriorPointSolver::L(const BlockVector &x, const Vector &l, const Vector &zl)
+{
+  double fx = optProblem->CalcObjective(x);
+  Vector cx(dimC); optProblem->c(x, cx);
+  return (fx + InnerProduct(cx, l) - InnerProduct(x.GetBlock(1), zl));
+}
+
+void InteriorPointSolver::DxL(const BlockVector &x, const Vector &l, const Vector &zl, BlockVector &y)
+{
+  // evaluate the gradient of the objective with respect to the primal variables x = (u, m)
+  BlockVector gradxf(block_offsetsx); gradxf = 0.0;
+  optProblem->CalcObjectiveGrad(x, gradxf);
+  
+  SparseMatrix *Jacu, *Jacm, *JacuT, *JacmT;
+  Jacu = optProblem->Duc(x); Jacm = optProblem->Dmc(x);
+  JacuT = Transpose(*Jacu);
+  JacmT = Transpose(*Jacm);
+  JacuT->Mult(l, y.GetBlock(0));
+  JacmT->Mult(l, y.GetBlock(1));
+  delete JacuT; 
+  delete JacmT;
+  y.Add(1.0, gradxf);
+  (y.GetBlock(1)).Add(-1.0, zl);
+}
+
+
+bool InteriorPointSolver::GetConverged() const
+{
+  return converged;
+}
+
+void InteriorPointSolver::SetTol(double Tol)
+{
+  rel_tol = Tol;
+}
+
+void InteriorPointSolver::SetMaxIter(int max_it)
+{
+  max_iter = max_it;
+}
+
+void InteriorPointSolver::SetBarrierParameter(double mu_0)
+{
+  mu_k = mu_0;
+}
+
+void InteriorPointSolver::SaveLogBarrierHessianIterates(bool save)
+{
+   MFEM_ASSERT(MyRank == 0 || save == false, "currently can only save logbarrier hessian in serial codes");
+   saveLogBarrierIterates = save;
+}
+
+void InteriorPointSolver::SetLinearSolver(int LinSolver)
+{
+   linSolver = LinSolver;
+}
+
+void InteriorPointSolver::SetLinearSolveTol(double Tol)
+{
+   linSolveTol = Tol;
+}
+
+
+InteriorPointSolver::~InteriorPointSolver() 
+{
+   delete HLuucl;
+   delete JuT;
+   delete JmT; 
+   delete Wmm;
+
+   F1.DeleteAll();
+   F2.DeleteAll();
+   block_offsetsx.DeleteAll();
+   block_offsetsumlz.DeleteAll();
+   block_offsetsuml.DeleteAll();
+   ml.SetSize(0);
+}

miniapps/contact/ipsolver/IPsolver.hpp

@@ -0,0 +1,94 @@
+#include "mfem.hpp"
+#include "../problems/problems.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+#ifndef IPSOLVER 
+#define IPSOLVER
+
+class InteriorPointSolver
+{
+protected:
+    QPOptContactProblem * optProblem;
+    double rel_tol;
+    int  max_iter;
+    double mu_k; // \mu_k
+    Vector lk, zlk;
+
+    double sMax, kSig, tauMin, eta, thetaMin, delta, sTheta, sPhi, kMu, thetaMu;
+    double thetaMax, kSoc, gTheta, gPhi, kEps;
+	
+    // filter
+    Array<double> F1, F2;
+	
+    // quantities computed in lineSearch
+    double alpha, alphaz;
+    double thx0, thxtrial;
+    double phx0, phxtrial;
+    bool descentDirection, switchCondition, sufficientDecrease, lineSearchSuccess, inFilterRegion;
+    double Dxphi0_xhat;
+
+    int dimU, dimM, dimC;
+    Array<int> block_offsetsumlz, block_offsetsuml, block_offsetsx;
+    Vector ml;
+
+    Vector ckSoc;
+    SparseMatrix * Huu = nullptr; 
+    SparseMatrix * Hum = nullptr; 
+    SparseMatrix * Hmu = nullptr; 
+    SparseMatrix * Hmm = nullptr; 
+    SparseMatrix * Wmm = nullptr; 
+    SparseMatrix * Ju  = nullptr; 
+    SparseMatrix * Jm  = nullptr; 
+    SparseMatrix * JmT  = nullptr; 
+    SparseMatrix * JuT = nullptr; 
+    SparseMatrix * Huucl = nullptr;
+    SparseMatrix * HLuucl = nullptr;
+
+    int jOpt;
+    bool converged;
+    
+    int MyRank;
+    bool iAmRoot;
+
+    bool saveLogBarrierIterates;
+
+    int linSolver;
+    double linSolveTol;
+    Array<int> cgnum_iterations;
+
+public:
+    InteriorPointSolver(QPOptContactProblem*);
+    void Mult(const BlockVector& , BlockVector&); // used when the user wants to be aware of bound-constrained variable m >= ml
+    void Mult(const Vector&, Vector &); // useful when the user doesn't need to know about bound-constrained variable   m >= ml, e.g., when m is a slack variable
+    double MaxStepSize(Vector& , Vector& , Vector& , double);
+    double MaxStepSize(Vector& , Vector& , double);
+    void FormIPNewtonMat(BlockVector& , Vector& , Vector& , BlockOperator &);
+    void IPNewtonSolve(BlockVector& , Vector& , Vector& , Vector&, BlockVector& , double, bool);
+    void lineSearch(BlockVector& , BlockVector& , double);
+    void projectZ(const Vector & , Vector &, double);
+    void filterCheck(double, double);
+    double E(const BlockVector &, const Vector &, const Vector &, double, bool);
+    double E(const BlockVector &, const Vector &, const Vector &, bool);
+    Array<int> & GetCGIterNumbers() {return cgnum_iterations;}
+    bool GetConverged() const;
+    // TO DO: include Hessian of Lagrangian
+    double theta(const BlockVector &);
+    double phi(const BlockVector &, double);
+    void Dxphi(const BlockVector &, double, BlockVector &);
+    double L(const BlockVector &, const Vector &, const Vector &);
+    void DxL(const BlockVector &, const Vector &, const Vector &, BlockVector &);
+    void SetTol(double);
+    void SetMaxIter(int);
+    void SetBarrierParameter(double);    
+    void SaveLogBarrierHessianIterates(bool);
+    void SetLinearSolver(int);
+    void SetLinearSolveTol(double);
+    virtual ~InteriorPointSolver();
+};
+
+#endif

miniapps/contact/ipsolver/ParIPsolver.cpp

@@ -0,0 +1,864 @@
+#include "mfem.hpp"
+#include "ParIPsolver.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstdlib>
+
+using namespace std;
+using namespace mfem;
+
+
+ParInteriorPointSolver::ParInteriorPointSolver(QPOptParContactProblem * problem_) 
+                     : problem(problem_)
+{
+   OptTol  = 1.e-2;
+   max_iter = 20;
+   mu_k     = 1.0;
+
+   sMax     = 1.e2;
+   kSig     = 1.e10;   // control deviation from primal Hessian
+   tauMin   = 0.8;     // control rate at which iterates can approach the boundary
+   eta      = 1.e-4;   // backtracking constant
+   thetaMin = 1.e-4;   // allowed violation of the equality constraints
+
+   // constants in line-step A-5.4
+   delta    = 1.0;
+   sTheta   = 1.1;
+   sPhi     = 2.3;
+
+   // control the rate at which the penalty parameter is decreased
+   kMu     = 0.2;
+   thetaMu = 1.5;
+
+   thetaMax = 1.e6; // maximum constraint violation
+   // data for the second order correction
+   kSoc     = 0.99;
+
+   // equation (18)
+   gTheta = 1.e-5;
+   gPhi   = 1.e-5;
+
+   kEps   = 1.e1;
+
+   dimU = problem->GetDimU();
+   dimM = problem->GetDimM();
+   dimC = problem->GetDimC();
+
+   MPI_Allreduce(&dimU,&gdimU,1,MPI_INT,MPI_SUM,problem->GetComm());
+   MPI_Allreduce(&dimM,&gdimM,1,MPI_INT,MPI_SUM,problem->GetComm());
+   MPI_Allreduce(&dimC,&gdimC,1,MPI_INT,MPI_SUM,problem->GetComm());
+
+   ckSoc.SetSize(dimC);
+
+   block_offsetsumlz.SetSize(5);
+   block_offsetsuml.SetSize(4);
+   block_offsetsx.SetSize(3);
+  
+   block_offsetsumlz[0] = 0;
+   block_offsetsumlz[1] = dimU; // u
+   block_offsetsumlz[2] = dimM; // m
+   block_offsetsumlz[3] = dimC; // lambda
+   block_offsetsumlz[4] = dimM; // zl
+   block_offsetsumlz.PartialSum();
+
+   for(int i = 0; i < block_offsetsuml.Size(); i++)  
+   { 
+      block_offsetsuml[i] = block_offsetsumlz[i]; 
+   }
+   for(int i = 0; i < block_offsetsx.Size(); i++)    
+   { 
+      block_offsetsx[i] = block_offsetsuml[i] ; 
+   }
+
+   ml = problem->Getml();
+  
+   lk.SetSize(dimC);  lk  = 0.0;
+   zlk.SetSize(dimM); zlk = 0.0;
+
+   linSolver = 0;
+   linSolveTol = 1.e-8;
+   MyRank = Mpi::WorldRank();
+   iAmRoot = MyRank == 0 ? true : false;
+}
+
+double ParInteriorPointSolver::MaxStepSize(Vector &x, Vector &xl, Vector &xhat, double tau)
+{
+   double alphaMaxloc = 1.0;
+   double alphaTmp;
+   for(int i = 0; i < x.Size(); i++)
+   {   
+      if( xhat(i) < 0. )
+      {
+         alphaTmp = -1. * tau * (x(i) - xl(i)) / xhat(i);
+         alphaMaxloc = min(alphaMaxloc, alphaTmp);
+      } 
+   }
+
+   // alphaMaxloc is the local maximum step size which is
+   // distinct on each MPI process. Need to compute
+   // the global maximum step size 
+   double alphaMaxglb;
+   MPI_Allreduce(&alphaMaxloc, &alphaMaxglb, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
+   return alphaMaxglb;
+}
+
+double ParInteriorPointSolver::MaxStepSize(Vector &x, Vector &xhat, double tau)
+{
+   Vector zero(x.Size()); zero = 0.0;
+   return MaxStepSize(x, zero, xhat, tau);
+}
+
+
+void ParInteriorPointSolver::Mult(const Vector &x0, Vector &xf)
+{
+   BlockVector x0block(block_offsetsx); x0block = 0.0;
+   x0block.GetBlock(0).Set(1.0, x0);
+   x0block.GetBlock(1) = 1.0;
+   x0block.GetBlock(1).Add(1.0, ml);
+   BlockVector xfblock(block_offsetsx); xfblock = 0.0;
+
+   Mult(x0block, xfblock);
+   xf.Set(1.0, xfblock.GetBlock(0));
+}
+
+
+void ParInteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
+{
+   converged = false;
+   
+   BlockVector xk(block_offsetsx), xhat(block_offsetsx); xk = 0; xhat = 0.0;
+   BlockVector Xk(block_offsetsumlz), Xhat(block_offsetsumlz); Xk = 0.0; Xhat = 0.0;
+   BlockVector Xhatuml(block_offsetsuml); Xhatuml = 0.0;
+   Vector zlhat(dimM); zlhat = 0.0;
+
+   xk.GetBlock(0).Set(1.0, x0.GetBlock(0));
+   xk.GetBlock(1).Set(1.0, x0.GetBlock(1));
+   // running estimate of the final values of the Lagrange multipliers
+   lk  = 0.0;
+   zlk = 0.0;
+
+   for(int i = 0; i < dimM; i++)
+   {
+      zlk(i) = 1.e1 * mu_k / (xk(i+dimU) - ml(i));
+   }
+
+   Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+   Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+   Xk.GetBlock(2).Set(1.0, lk);
+   Xk.GetBlock(3).Set(1.0, zlk);
+
+  /* set theta0 = theta(x0)
+   *     thetaMin
+   *     thetaMax
+   * when theta(xk) < thetaMin and the switching condition holds
+   * then we ask for the Armijo sufficient decrease of the barrier
+   * objective to be satisfied, in order to accept the trial step length alphakl
+   * 
+   * thetaMax controls how the filter is initialized for each log-barrier subproblem
+   * F0 = {(th, phi) s.t. th > thetaMax}
+   * that is the filter does not allow for iterates where the constraint violation
+   * is larger than that of thetaMax
+   */
+   double theta0 = theta(xk);
+   thetaMin = 1.e-4 * max(1.0, theta0);
+   thetaMax = 1.e8  * thetaMin; // 1.e4 * max(1.0, theta0)
+
+   double Eeval, maxBarrierSolves, Eevalmu0;
+   bool printOptimalityError; // control optimality error print to console for log-barrier subproblems
+   
+   maxBarrierSolves = 10;
+
+   for(jOpt = 0; jOpt < max_iter; jOpt++)
+   {
+      if(iAmRoot)
+      {
+         cout << "interior-point solve step " << jOpt << endl;
+      }
+      // A-2. Check convergence of overall optimization problem
+      printOptimalityError = false;
+      Eevalmu0 = E(xk, lk, zlk, printOptimalityError);
+      if(Eevalmu0 < OptTol)
+      {
+         converged = true;
+         if(iAmRoot)
+         {
+            cout << "solved optimization problem :)\n";
+         }
+         break;
+      }
+      
+      if(jOpt > 0) { maxBarrierSolves = 1; }
+      
+      for(int i = 0; i < maxBarrierSolves; i++)
+      {
+         // A-3. Check convergence of the barrier subproblem
+         printOptimalityError = true;
+         Eeval = E(xk, lk, zlk, mu_k, printOptimalityError);
+         if(iAmRoot)
+         {
+            cout << "E = " << Eeval << endl;
+         }
+         if(Eeval < kEps * mu_k)
+         {
+            if(iAmRoot)
+            {
+               cout << "solved barrier subproblem :), for mu = " << mu_k << endl;
+            }
+            // A-3.1. Recompute the barrier parameter
+            mu_k  = max(OptTol / 10., min(kMu * mu_k, pow(mu_k, thetaMu)));
+            // A-3.2. Re-initialize the filter
+            F1.DeleteAll();
+            F2.DeleteAll();
+         }
+         else
+         {
+            break;
+         }
+      }
+    
+      // A-4. Compute the search direction
+      // solve for (uhat, mhat, lhat)
+      if(iAmRoot)
+      {
+         cout << "\n** A-4. IP-Newton solve **\n";
+      }
+      zlhat = 0.0; Xhatuml = 0.0;
+      // why do we have Xhatuml ....???
+      // TO DO: remove Xhatuml in favor of passing Xhat
+      IPNewtonSolve(xk, lk, zlk, zlhat, Xhatuml, mu_k, false); 
+
+      // assign data stack, X = (u, m, l, zl)
+      Xk = 0.0;
+      Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+      Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+      Xk.GetBlock(2).Set(1.0, lk);
+      Xk.GetBlock(3).Set(1.0, zlk);
+
+      // assign data stack, Xhat = (uhat, mhat, lhat, zlhat)
+      Xhat = 0.0;
+      for(int i = 0; i < 3; i++)
+      {
+         Xhat.GetBlock(i).Set(1.0, Xhatuml.GetBlock(i));
+      }
+      Xhat.GetBlock(3).Set(1.0, zlhat);
+
+      // A-5. Backtracking line search.
+      if(iAmRoot)
+      {
+         cout << "\n** A-5. Linesearch **\n";
+         cout << "mu = " << mu_k << endl;
+      }
+      lineSearch(Xk, Xhat, mu_k);
+
+      if(lineSearchSuccess)
+      {
+         if(iAmRoot)
+         {
+            cout << "lineSearch successful :)\n";
+         }
+         if(!switchCondition || !sufficientDecrease)
+         {
+            F1.Append( (1. - gTheta) * thx0);
+            F2.Append( phx0 - gPhi * thx0);
+         }
+         // ----- A-6: Accept the trial point
+         // print info regarding zl...
+         xk.GetBlock(0).Add(alpha, Xhat.GetBlock(0));
+         xk.GetBlock(1).Add(alpha, Xhat.GetBlock(1));
+         lk.Add(alpha,   Xhat.GetBlock(2));
+         zlk.Add(alphaz, Xhat.GetBlock(3));
+         projectZ(xk, zlk, mu_k);
+      }
+      else
+      {
+         if(iAmRoot)
+         {
+            cout << "lineSearch not successful :(\n";
+            cout << "attempting feasibility restoration with theta = " << thx0 << endl;
+            cout << "no feasibility restoration implemented, exiting now \n";
+         }
+         break;
+      }
+      if(jOpt + 1 == max_iter && iAmRoot) 
+      {  
+         cout << "maximum optimization iterations :(\n";
+      }
+   }
+   // done with optimization routine, just reassign data to xf reference so
+   // that the application code has access to the optimal point
+   xf = 0.0;
+   xf.GetBlock(0).Set(1.0, xk.GetBlock(0));
+   xf.GetBlock(1).Set(1.0, xk.GetBlock(1));
+}
+
+void ParInteriorPointSolver::FormIPNewtonMat(BlockVector & x, Vector & l, Vector &zl, 
+                                             BlockOperator &Ak)
+{
+   // WARNING: Huu, Hum, Hmu, Hmm should all be Hessian terms of the Lagrangian, currently we 
+   //          them by Hessian terms of the objective function and neglect the Hessian of l^T c
+
+   Huu = problem->Duuf(x); 
+   Hum = problem->Dumf(x);
+   Hmu = problem->Dmuf(x);
+   Hmm = problem->Dmmf(x);
+
+   Vector DiagLogBar(dimM); DiagLogBar = 0.0;
+   for(int ii = 0; ii < dimM; ii++)
+   {
+      DiagLogBar(ii) = zl(ii) / (x(ii+dimU) - ml(ii));
+   }
+   if(saveLogBarrierIterates)
+   {
+      std::ofstream diagStream;
+      char diagString[100];
+      snprintf(diagString, 100, "logBarrierHessiandata/D%d.dat", jOpt);
+      diagStream.open(diagString, ios::out | ios::trunc);
+      for(int ii = 0; ii < dimM; ii++)
+      {
+         diagStream << setprecision(30) << DiagLogBar(ii) << endl;
+         // mfem::out << DiagLogBar(ii) << endl;
+      }
+      diagStream.close();
+   } 
+
+   
+   int gsize = problem->GetGlobalNumConstraints();
+   int * rows = problem->GetConstraintsStarts();
+
+   delete Wmm;
+   if(Hmm != nullptr)
+   {
+      SparseMatrix * Ds = new SparseMatrix(DiagLogBar);
+      HypreParMatrix * D = new HypreParMatrix(problem->GetComm(), gsize, rows, Ds);
+      HypreStealOwnership(*D,*Ds);
+      delete Ds;
+      Wmm = ParAdd(Hmm,D);
+      delete D;
+   }
+   else
+   {
+      SparseMatrix * Ds = new SparseMatrix(DiagLogBar);
+      Wmm = new HypreParMatrix(problem->GetComm(), gsize, rows, Ds);
+      HypreStealOwnership(*Wmm,*Ds);
+      delete Ds;
+   }
+
+   delete JuT;
+   delete JmT;
+   Ju = problem->Duc(x); JuT = Ju->Transpose();
+   Jm = problem->Dmc(x); JmT = Jm->Transpose();
+
+   //         IP-Newton system matrix
+   //    Ak = [[H_(u,u)  H_(u,m)   J_u^T]
+   //          [H_(m,u)  W_(m,m)   J_m^T]
+   //          [ J_u      J_m       0  ]]
+
+   Ak.SetBlock(0, 0, Huu);                         Ak.SetBlock(0, 2, JuT);
+                           Ak.SetBlock(1, 1, Wmm); Ak.SetBlock(1, 2, JmT);
+   Ak.SetBlock(2, 0,  Ju); Ak.SetBlock(2, 1,  Jm);
+   if(Hum != nullptr) { Ak.SetBlock(0, 1, Hum); Ak.SetBlock(1, 0, Hmu); }
+}
+
+// perturbed KKT system solve
+// determine the search direction
+void ParInteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, Vector &zlhat, BlockVector &Xhat, double mu, bool socSolve)
+{
+   // solve A x = b, where A is the IP-Newton matrix
+   BlockOperator A(block_offsetsuml, block_offsetsuml); 
+   BlockVector b(block_offsetsuml); b = 0.0;
+   FormIPNewtonMat(x, l, zl, A);
+
+   //       [grad_u phi + Ju^T l]
+   // b = - [grad_m phi + Jm^T l]
+   //       [          c        ]
+   BlockVector gradphi(block_offsetsx); gradphi = 0.0;
+   BlockVector JTl(block_offsetsx); JTl = 0.0;
+   Dxphi(x, mu, gradphi);
+   
+   (A.GetBlock(0,2)).Mult(l, JTl.GetBlock(0));
+   (A.GetBlock(1,2)).Mult(l, JTl.GetBlock(1));
+
+   for(int ii = 0; ii < 2; ii++)
+   {
+      b.GetBlock(ii).Set(1.0, gradphi.GetBlock(ii));
+      b.GetBlock(ii).Add(1.0, JTl.GetBlock(ii));
+   }
+   if(!socSolve) 
+   {
+      problem->c(x, b.GetBlock(2));
+   }
+   else
+   {
+      b.GetBlock(2).Set(1.0, ckSoc);
+   }
+   b *= -1.0; 
+   Xhat = 0.0;
+
+   // Direct solver (default)
+   if(linSolver == 0)
+   {
+      Array2D<HypreParMatrix *> ABlockMatrix(3,3);
+      for(int ii = 0; ii < 3; ii++)
+      {
+         for(int jj = 0; jj < 3; jj++)
+         {
+            if(!A.IsZeroBlock(ii, jj))
+            {
+               ABlockMatrix(ii, jj) = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(ii, jj)));
+	         }
+	         else
+	         {
+	            ABlockMatrix(ii, jj) = nullptr;
+	         }
+         }  
+      }
+      
+      HypreParMatrix * Ah = HypreParMatrixFromBlocks(ABlockMatrix);   
+
+      /* direct solve of the 3x3 IP-Newton linear system */
+#ifdef MFEM_USE_MUMPS
+      MUMPSSolver ASolver(*Ah);;
+      ASolver.SetPrintLevel(0);
+      ASolver.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
+      ASolver.Mult(b, Xhat);
+#else 
+#ifdef MFEM_USE_MKL_CPARDISO
+      CPardisoSolver ASolver(MPI_COMM_WORLD);
+      ASolver.SetOperator(*Ah);
+      ASolver.Mult(b, Xhat);
+#else
+  	   MFEM_VERIFY(false, "linSolver 0 will not work unless compiled with MUMPS or MKL");
+#endif
+#endif
+
+      delete Ah;
+   }
+   else if(linSolver == 1 || linSolver == 2)
+   {
+      // form A = Huu + Ju^T D Ju, Wmm = D for contact
+      HypreParMatrix * Wmmloc = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(1, 1)));
+      HypreParMatrix * Huuloc = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(0, 0)));
+      HypreParMatrix * Juloc  = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(2, 0)));
+      HypreParMatrix * JuTloc = dynamic_cast<HypreParMatrix *>(&(A.GetBlock(0, 2)));
+      HypreParMatrix *JuTDJu   = RAP(Wmmloc, Juloc);     // Ju^T D Ju
+      HypreParMatrix *Areduced = ParAdd(Huuloc, JuTDJu);  // Huu + Ju^T D Ju
+
+      Areduced->DropSmallEntries(1e-16);
+
+      /* prepare the reduced rhs */
+      // breduced = bu + Ju^T (bm + Wmm bl)
+      Vector breduced(dimU); breduced = 0.0;
+      Vector tempVec(dimM); tempVec = 0.0;
+      Wmmloc->Mult(b.GetBlock(2), tempVec);
+      tempVec.Add(1.0, b.GetBlock(1));
+      JuTloc->Mult(tempVec, breduced);
+      breduced.Add(1.0, b.GetBlock(0));
+      
+      if(linSolver == 1)
+      {
+         // setup the solver for the reduced linear system
+#ifdef MFEM_USE_MUMPS
+	      MUMPSSolver AreducedSolver(*Areduced);   
+         AreducedSolver.SetPrintLevel(0);
+         AreducedSolver.SetMatrixSymType(MUMPSSolver::MatType::SYMMETRIC_INDEFINITE);
+	      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+#else 
+#ifdef MFEM_USE_MKL_CPARDISO
+         CPardisoSolver AreducedSolver(MPI_COMM_WORLD);
+	      AreducedSolver.SetOperator(*Areduced);
+	      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+#else
+         MFEM_VERIFY(false, "linSolver 1 will not work unless compiled with MUMPS or MKL");
+#endif
+#endif
+      }
+      else
+      {
+         HypreBoomerAMG amg(*Areduced);
+         amg.SetPrintLevel(0);
+         if (pfes)
+         {
+            amg.SetElasticityOptions(pfes);
+         }
+         else
+         {
+            amg.SetSystemsOptions(3,false);
+         }
+         amg.SetRelaxType(relax_type);
+         int n;
+
+
+         // CGSolver AreducedSolver(MPI_COMM_WORLD);
+	      // AreducedSolver.SetOperator(*Areduced);
+         // AreducedSolver.SetRelTol(linSolveTol);
+         // AreducedSolver.SetMaxIter(1000);
+         // AreducedSolver.SetPreconditioner(amg);
+	      // AreducedSolver.SetPrintLevel(3);
+         // AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+         // n = AreducedSolver.GetNumIterations();
+
+         HyprePCG AreducedSolver(*Areduced);
+         AreducedSolver.SetTol(linSolveTol);
+         AreducedSolver.SetMaxIter(1000);
+         AreducedSolver.SetPreconditioner(amg);
+	      AreducedSolver.SetPrintLevel(2);
+         // AreducedSolver.SetResidualConvergenceOptions();
+         AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+         AreducedSolver.GetNumIterations(n);
+
+         cgnum_iterations.Append(n);
+
+
+      }
+
+      // now propagate solved uhat to obtain mhat and lhat
+      // xm = Ju xu - bl
+      Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+      Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+      // xl = Wmm xm - bm
+      Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+      Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+      delete JuTDJu;
+      delete Areduced;
+   }
+
+   /* backsolve to determine zlhat */
+   for(int ii = 0; ii < dimM; ii++)
+   {
+      zlhat(ii) = -1.*(zl(ii) + (zl(ii) * Xhat(ii + dimU) - mu) / (x(ii + dimU) - ml(ii)) );
+   }
+}
+
+// here Xhat, X will be BlockVectors w.r.t. the 4 partitioning X = (u, m, l, zl)
+
+void ParInteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double mu)
+{
+   double tau  = max(tauMin, 1.0 - mu);
+   Vector u0   = X0.GetBlock(0);
+   Vector m0   = X0.GetBlock(1);
+   Vector l0   = X0.GetBlock(2);
+   Vector z0   = X0.GetBlock(3);
+   Vector uhat = Xhat.GetBlock(0);
+   Vector mhat = Xhat.GetBlock(1);
+   Vector lhat = Xhat.GetBlock(2);
+   Vector zhat = Xhat.GetBlock(3);
+   double alphaMax  = MaxStepSize(m0, ml, mhat, tau);
+   double alphaMaxz = MaxStepSize(z0, zhat, tau);
+   alphaz = alphaMaxz;
+
+   BlockVector x0(block_offsetsx); x0 = 0.0;
+   x0.GetBlock(0).Set(1.0, u0);
+   x0.GetBlock(1).Set(1.0, m0);
+   
+   BlockVector xhat(block_offsetsx); xhat = 0.0;
+   xhat.GetBlock(0).Set(1.0, uhat);
+   xhat.GetBlock(1).Set(1.0, mhat);
+   
+   BlockVector xtrial(block_offsetsx); xtrial = 0.0;
+   BlockVector Dxphi0(block_offsetsx); Dxphi0 = 0.0;
+   int maxBacktrack = 20;
+   alpha = alphaMax;
+
+   Vector ck0(dimC); ck0 = 0.0;
+   Vector zhatsoc(dimM); zhatsoc = 0.0;
+   BlockVector Xhatumlsoc(block_offsetsuml); Xhatumlsoc = 0.0;
+   BlockVector xhatsoc(block_offsetsx); xhatsoc = 0.0;
+   Vector uhatsoc(dimU); uhatsoc = 0.0;
+   Vector mhatsoc(dimM); mhatsoc = 0.0;
+
+   Dxphi(x0, mu, Dxphi0);
+
+   Dxphi0_xhat = InnerProduct(MPI_COMM_WORLD, Dxphi0, xhat);
+   descentDirection = Dxphi0_xhat < 0. ? true : false;
+   
+   
+   if (iAmRoot)
+   {
+      if(descentDirection)
+      {
+         cout << "is a descent direction for the log-barrier objective\n";
+      }
+      else
+      {
+         cout << "is not a descent direction for the log-barrier objective\n";
+      }
+   }
+
+   thx0 = theta(x0);
+   phx0 = phi(x0, mu);
+
+   lineSearchSuccess = false;
+   for(int i = 0; i < maxBacktrack; i++)
+   {
+      if (iAmRoot)
+      {
+         cout << "\n--------- alpha = " << alpha << " ---------\n";
+      }
+      // ----- A-5.2. Compute trial point: xtrial = x0 + alpha_i xhat
+      xtrial.Set(1.0, x0);
+      xtrial.Add(alpha, xhat);
+
+      // ------ A-5.3. if not in filter region go to A.5.4 otherwise go to A-5.5.
+      thxtrial = theta(xtrial);
+      phxtrial = phi(xtrial, mu);
+      filterCheck(thxtrial, phxtrial);    
+      if(!inFilterRegion)
+      {
+         if (iAmRoot)
+         {
+            cout << "not in filter region :)\n";
+         }
+         // ------ A.5.4: Check sufficient decrease
+         if(!descentDirection)
+         {
+            switchCondition = false;
+         }
+         else
+         {
+            switchCondition = (alpha * pow(abs(Dxphi0_xhat), sPhi) > delta * pow(thx0, sTheta)) ? true : false;
+         }
+         if (iAmRoot)
+         {
+            cout << "theta(x0) = "     << thx0     << ", thetaMin = "                  << thetaMin             << endl;
+            cout << "theta(xtrial) = " << thxtrial << ", (1-gTheta) *theta(x0) = "     << (1. - gTheta) * thx0 << endl;
+            cout << "phi(xtrial) = "   << phxtrial << ", phi(x0) - gPhi *theta(x0) = " << phx0 - gPhi * thx0   << endl;
+         }      
+         // Case I      
+         if(thx0 <= thetaMin && switchCondition)
+         {
+            sufficientDecrease = (phxtrial <= phx0 + eta * alpha * Dxphi0_xhat) ? true : false;
+            if(sufficientDecrease)
+            {
+               if(iAmRoot) { cout << "Line search successful: sufficient decrease in log-barrier objective.\n"; } 
+               // accept the trial step
+               lineSearchSuccess = true;
+               break;
+            }
+         }
+         else
+         {
+            if(thxtrial <= (1. - gTheta) * thx0 || phxtrial <= phx0 - gPhi * thx0)
+            {
+               if(iAmRoot) { cout << "Line search successful: infeasibility or log-barrier objective decreased.\n"; } 
+               // accept the trial step
+               lineSearchSuccess = true;
+               break;
+            }
+         }
+         // A-5.5: Initialize the second-order correction
+         if((!(thx0 < thxtrial)) && i == 0)
+         {
+            if (iAmRoot)
+            {
+               cout << "second order correction\n";
+            }
+            problem->c(xtrial, ckSoc);
+            problem->c(x0, ck0);
+            ckSoc.Add(alphaMax, ck0);
+            // A-5.6 Compute the second-order correction.
+            IPNewtonSolve(x0, l0, z0, zhatsoc, Xhatumlsoc, mu, true);
+            mhatsoc.Set(1.0, Xhatumlsoc.GetBlock(1));
+            //WARNING: not complete but currently solver isn't entering this region
+         }
+      }
+      else
+      {
+         if (iAmRoot)
+         {
+            cout << "in filter region :(\n"; 
+         }
+      }
+      // include more if needed
+      alpha *= 0.5;
+
+   } 
+}
+
+
+void ParInteriorPointSolver::projectZ(const Vector &x, Vector &z, double mu)
+{
+   double zi;
+   double mudivmml;
+   for(int i = 0; i < dimM; i++)
+   {
+      zi = z(i);
+      mudivmml = mu / (x(i + dimU) - ml(i));
+      z(i) = max(min(zi, kSig * mudivmml), mudivmml / kSig);
+   }
+}
+
+void ParInteriorPointSolver::filterCheck(double th, double ph)
+{
+   inFilterRegion = false;
+   if(th > thetaMax)
+   {
+      inFilterRegion = true;
+   }
+   else
+   {
+      for(int i = 0; i < F1.Size(); i++)
+      {
+         if(th >= F1[i] && ph >= F2[i])
+         {
+            inFilterRegion = true;
+            break;
+         }
+      }
+   }
+}
+
+double ParInteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, double mu, bool printEeval)
+{
+   double E1, E2, E3;
+   double sc, sd;
+   BlockVector gradL(block_offsetsx); gradL = 0.0; // stationarity grad L = grad f + J^T l - z
+   Vector cx(dimC); cx = 0.0;     // feasibility c = c(x)
+   Vector comp(dimM); comp = 0.0; // complementarity M Z - mu 1
+
+   DxL(x, l, zl, gradL);
+   E1 = GlobalLpNorm(infinity(), gradL.Normlinf(), MPI_COMM_WORLD); 
+
+   problem->c(x, cx);
+   E2 = GlobalLpNorm(infinity(), cx.Normlinf(), MPI_COMM_WORLD); 
+
+
+   for(int ii = 0; ii < dimM; ii++) 
+   { 
+      comp(ii) = x(dimU + ii) * zl(ii) - mu;
+   }
+   E3 = GlobalLpNorm(infinity(), comp.Normlinf(), MPI_COMM_WORLD); 
+
+   double ll1, zl1;
+
+   zl1 = GlobalLpNorm(1, zl.Norml1(), MPI_COMM_WORLD)/ double(gdimC + gdimM);; 
+   ll1 = GlobalLpNorm(1, l.Norml1(), MPI_COMM_WORLD);
+   sc = max(sMax, zl1 / (double(gdimM)) ) / sMax;
+   sd = max(sMax, (ll1 + zl1) / (double(gdimC + gdimM))) / sMax;
+   if(iAmRoot && printEeval)
+   {
+      cout << "evaluating optimality error for mu = " << mu << endl;
+      cout << "stationarity measure = "    << E1 / sd << endl;
+      cout << "feasibility measure  = "    << E2      << endl;
+      cout << "complimentarity measure = " << E3 / sc << endl;
+   }
+   return max(max(E1 / sd, E2), E3 / sc);
+}
+
+double ParInteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, bool printEeval)
+{
+  return E(x, l, zl, 0.0, printEeval);
+}
+
+double ParInteriorPointSolver::theta(const BlockVector &x)
+{
+   Vector cx(dimC);
+   problem->c(x, cx);
+   return sqrt(InnerProduct(MPI_COMM_WORLD,cx, cx));
+}
+
+// log-barrier objective
+double ParInteriorPointSolver::phi(const BlockVector &x, double mu)
+{
+   double fx = problem->CalcObjective(x); 
+   double logBarrierLoc = 0.0;
+   for(int i = 0; i < dimM; i++) 
+   { 
+     logBarrierLoc += log(x(dimU+i)-ml(i));
+   }
+   double logBarrierGlb;
+   MPI_Allreduce(&logBarrierLoc, &logBarrierGlb, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+   return fx - mu * logBarrierGlb;
+}
+
+// gradient of log-barrier objective with respect to x = (u, m)
+void ParInteriorPointSolver::Dxphi(const BlockVector &x, double mu, BlockVector &y)
+{
+   problem->CalcObjectiveGrad(x, y);
+   
+   for(int i = 0; i < dimM; i++) 
+   { 
+      y(dimU + i) -= mu / (x(dimU + i));
+   } 
+}
+
+// Lagrangian function evaluation
+// L(x, l, zl) = f(x) + l^T c(x) - zl^T m
+double ParInteriorPointSolver::L(const BlockVector &x, const Vector &l, const Vector &zl)
+{
+   double fx = problem->CalcObjective(x);
+   Vector cx(dimC); problem->c(x, cx);
+   return (fx + InnerProduct(MPI_COMM_WORLD,cx, l) - InnerProduct(MPI_COMM_WORLD, x.GetBlock(1), zl));
+}
+
+void ParInteriorPointSolver::DxL(const BlockVector &x, const Vector &l, const Vector &zl, BlockVector &y)
+{
+   // evaluate the gradient of the objective with respect to the primal variables x = (u, m)
+   BlockVector gradxf(block_offsetsx); gradxf = 0.0;
+   problem->CalcObjectiveGrad(x, gradxf);
+   
+   HypreParMatrix *Jacu, *Jacm, *JacuT, *JacmT;
+   Jacu = problem->Duc(x); 
+   Jacm = problem->Dmc(x);
+   JacuT = Jacu->Transpose();
+   JacmT = Jacm->Transpose();
+   
+   JacuT->Mult(l, y.GetBlock(0));
+   JacmT->Mult(l, y.GetBlock(1));
+   
+   delete JacuT;
+   delete JacmT;
+   
+   y.Add(1.0, gradxf);
+   (y.GetBlock(1)).Add(-1.0, zl);
+}
+
+bool ParInteriorPointSolver::GetConverged() const
+{
+   return converged;
+}
+
+void ParInteriorPointSolver::SetTol(double Tol)
+{
+   OptTol = Tol;
+}
+
+void ParInteriorPointSolver::SetMaxIter(int max_it)
+{
+   max_iter = max_it;
+}
+
+void ParInteriorPointSolver::SetBarrierParameter(double mu_0)
+{
+   mu_k = mu_0;
+}
+
+void ParInteriorPointSolver::SaveLogBarrierHessianIterates(bool save)
+{
+   MFEM_ASSERT(MyRank == 0 || save == false, "currently can only save logbarrier hessian in serial codes");
+   saveLogBarrierIterates = save;
+}
+
+void ParInteriorPointSolver::SetLinearSolver(int LinSolver)
+{
+   linSolver = LinSolver;
+}
+
+void ParInteriorPointSolver::SetLinearSolveTol(double Tol)
+{
+  linSolveTol = Tol;
+}
+
+void ParInteriorPointSolver::SetLinearSolveRelaxType(int relax_type_)
+{
+   relax_type = relax_type_;
+}
+
+
+ParInteriorPointSolver::~ParInteriorPointSolver() 
+{
+   delete JuT;
+   delete JmT;
+   delete Wmm;
+}

miniapps/contact/ipsolver/ParIPsolver.hpp

@@ -0,0 +1,100 @@
+#include "mfem.hpp"
+#include "../problems/parproblems.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+#ifndef PARIPSOLVER 
+#define PARIPSOLVER
+
+class ParInteriorPointSolver
+{
+protected:
+    QPOptParContactProblem* problem;
+    double OptTol;
+    int  max_iter;
+    double mu_k; // \mu_k
+    Vector lk, zlk;
+
+    double sMax, kSig, tauMin, eta, thetaMin, delta, sTheta, sPhi, kMu, thetaMu;
+    double thetaMax, kSoc, gTheta, gPhi, kEps;
+	
+    // filter
+    Array<double> F1, F2;
+	
+    // quantities computed in lineSearch
+    double alpha, alphaz;
+    double thx0, thxtrial;
+    double phx0, phxtrial;
+    bool descentDirection, switchCondition, sufficientDecrease, lineSearchSuccess, inFilterRegion;
+    double Dxphi0_xhat;
+
+    int dimU, dimM, dimC;
+    int gdimU, gdimM, gdimC;
+    Array<int> block_offsetsumlz, block_offsetsuml, block_offsetsx;
+    Vector ml;
+
+    Vector ckSoc;
+    HypreParMatrix * Huu = nullptr;
+    HypreParMatrix * Hum = nullptr;
+    HypreParMatrix * Hmu = nullptr;
+    HypreParMatrix * Hmm = nullptr;
+    HypreParMatrix * Wmm = nullptr;
+    HypreParMatrix * Ju = nullptr;
+    HypreParMatrix * Jm = nullptr;
+    HypreParMatrix * JuT = nullptr;
+    HypreParMatrix * JmT = nullptr;
+
+    Array<int> cgnum_iterations;
+    ParFiniteElementSpace *pfes = nullptr;
+    
+    int jOpt;
+    bool converged;
+    
+    int MyRank;
+    bool iAmRoot;
+
+    bool saveLogBarrierIterates = false;
+
+    int linSolver;
+    double linSolveTol;
+    int relax_type = 8;
+public:
+    ParInteriorPointSolver(QPOptParContactProblem*);
+    double MaxStepSize(Vector& , Vector& , Vector& , double);
+    double MaxStepSize(Vector& , Vector& , double);
+    void Mult(const BlockVector& , BlockVector&);
+    void Mult(const Vector&, Vector &); 
+    void FormIPNewtonMat(BlockVector& , Vector& , Vector& , BlockOperator &);
+    void IPNewtonSolve(BlockVector& , Vector& , Vector& , Vector&, BlockVector& , double, bool);
+    void lineSearch(BlockVector& , BlockVector& , double);
+    void projectZ(const Vector & , Vector &, double);
+    void filterCheck(double, double);
+    double E(const BlockVector &, const Vector &, const Vector &, double, bool);
+    double E(const BlockVector &, const Vector &, const Vector &, bool);
+    bool GetConverged() const;
+    Array<int> & GetCGIterNumbers() {return cgnum_iterations;}
+    // TO DO: include Hessian of Lagrangian
+    double theta(const BlockVector &);
+    double phi(const BlockVector &, double);
+    void Dxphi(const BlockVector &, double, BlockVector &);
+    double L(const BlockVector &, const Vector &, const Vector &);
+    void DxL(const BlockVector &, const Vector &, const Vector &, BlockVector &);
+    void SetTol(double);
+    void SetMaxIter(int);
+    void SetBarrierParameter(double);    
+    void SaveLogBarrierHessianIterates(bool);
+    void SetLinearSolver(int);
+    void SetLinearSolveTol(double);
+    void SetLinearSolveRelaxType(int);
+    void SetFiniteElementSpace(ParFiniteElementSpace * pfes_)
+    {
+        pfes = pfes_;
+    }
+    virtual ~ParInteriorPointSolver();
+};
+
+#endif

miniapps/contact/makefile

@@ -0,0 +1,110 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/contact/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+# Include defaults.mk to get XLINKER
+DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk
+include $(DEFAULTS_MK)
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+CONTACT_SEQ_SRC = problems/problems.cpp problems/problems_util.cpp util/util.cpp ipsolver/IPsolver.cpp
+CONTACT_SEC_OBJ = $(CONTACT_PAR_SRC:.cpp=.o)
+CONTACT_PAR_SRC = $(CONTACT_SEQ_SRC) ipsolver/ParIPsolver.cpp problems/parproblems.cpp problems/parproblems_util.cpp util/mpicomm.cpp 
+CONTACT_PAR_OBJ = $(CONTACT_PAR_SRC:.cpp=.o)
+
+CONTACT_SRC = contact_driver.cpp $(CONTACT_SEQ_SRC)
+CONTACT_OBJ = $(CONTACT_SRC:.cpp=.o)
+
+PCONTACT_SRC = pcontact_driver.cpp $(CONTACT_PAR_SRC)
+PCONTACT_OBJ = $(PCONTACT_SRC:.cpp=.o)
+
+SEQ_MINIAPPS = contact_driver
+PAR_MINIAPPS = pcontact_driver
+
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common
+
+# If MFEM_SHARED is set, add the ../common rpath
+COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
+   $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\
+   $(MFEM_BUILD_DIR)/miniapps/common))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all lib-common clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+%.o: %.cpp
+
+%.o: $(SRC)%.cpp $(wildcard $(SRC)%.hpp) $(MFEM_LIB_FILE)\
+ $(CONFIG_MK) | lib-common
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
+
+util/%.o: $(SRC)util/%.cpp $(wildcard $(SRC)util/%.hpp) $(MFEM_LIB_FILE)\
+ $(CONFIG_MK) | lib-common
+	mkdir -p $(@D)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
+
+problems/%.o: $(SRC)problems/%.cpp $(wildcard $(SRC)problems/%.hpp) $(MFEM_LIB_FILE)\
+ $(CONFIG_MK) | lib-common
+	mkdir -p $(@D)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@
+
+
+all: $(MINIAPPS)
+
+contact_driver: $(CONTACT_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(CONTACT_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+pcontact_driver: $(PCONTACT_OBJ)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $(PCONTACT_OBJ) $(COMMON_LIB) $(MFEM_LIBS)
+
+
+# Rule for building lib-common
+lib-common:
+	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Testing: Specific execution options
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+contact-test-seq: diffusion
+	@$(call mfem-test,$<,, contact miniapp,)
+pcontact-test-par: pcontact
+	@$(call mfem-test,$<, $(RUN_MPI), pcontact miniapp,)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+	rm -f $(CONTACT_OBJ) $(PCONTACT_OBJ)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf ParaView 

miniapps/contact/meshes/block1.mesh

@@ -0,0 +1,103 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+9
+1   5     0      1     3     2     8    9     11    10
+1   5     2      3     5     4    10    11    13    12
+1   5     4      5     7     6    12    13    15    14
+1   5     8      9    11    10    16    17    19    18
+1   5     10    11    13    12    18    19    21    20
+1   5     12    13    15    14    20    21    23    22
+1   5     16    17    19    18    24    25    27    26
+1   5     18    19    21    20    26    27    29    28
+1   5     20    21    23    22    28    29    31    30
+
+
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+30
+1   3    1     0     2     3
+1   3    3     2     4     5
+1   3    5     4     6     7
+1   3    24    25    27    26
+1   3    26    27    29    28
+1   3    28    29    31    30
+2   3    2     0     8    10
+2   3    4     2    10    12
+2   3    6     4    12    14
+2  3    10    8    16    18
+2  3    12    10    18    20
+2  3    14    12    20    22
+2  3    18    16    24    26
+2  3    20    18    26    28
+2  3    22    20    28    30
+3  3    1     3    11    9
+3  3     3     5    13    11
+3  3     5     7    15    13
+3  3    9    11    19    17
+3  3    11    13    21    19
+3  3    13    15    23    21
+3  3    17    19    27    25
+3  3    19    21    29    27
+3  3    21    23    31    29
+1  3     8     0     1    9
+1  3    16     8    9    17
+1  3    24    16    17    25
+1  3     6    14    15     7
+1  3    14    22    23    15
+1  3    22    30    31    23
+
+
+vertices
+32
+3
+   -1.0000         0         0
+         0         0         0
+   -1.0000    0.3000         0
+         0    0.3000         0
+   -1.0000    0.6500         0
+         0    0.6500         0
+   -1.0000    1.0000         0
+         0    1.0000         0
+   -1.0000         0    0.3000
+         0         0    0.3000
+   -1.0000    0.3000    0.3500
+         0    0.3000    0.3500
+   -1.0000    0.6500    0.3000
+         0    0.6500    0.3000
+   -1.0000    1.0000    0.3000
+         0    1.0000    0.3000
+   -1.0000         0    0.6500
+         0         0    0.6500
+   -1.0000    0.3000    0.6500
+         0    0.3000    0.6500
+   -1.0000    0.6500    0.6500
+         0    0.6500    0.6500
+   -1.0000    1.0000    0.6500
+         0    1.0000    0.6500
+   -1.0000         0    1.0000
+         0         0    1.0000
+   -1.0000    0.3000    1.0000
+         0    0.3000    1.0000
+   -1.0000    0.6500    1.0000
+         0    0.6500    1.0000
+   -1.0000    1.0000    1.0000
+         0    1.0000    1.0000

miniapps/contact/meshes/block2.mesh

@@ -0,0 +1,68 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+# 1 nothing
+elements
+4
+1   5     0     1     3     2     6     7    9     8
+1   5     2     3     5     4     8     9    11    10
+1   5     6     7     9     8    12    13    15    14
+1   5     8     9    11    10    14    15    17    16
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+16
+1   3     1     0     2     3
+1   3     3     2     4     5
+1   3    12    13    15    14
+1   3    14    15    17    16
+3   3     2     0     6     8
+3   3     4     2     8    10
+3   3     8     6    12    14
+3   3    10     8    14    16
+2   3     1     3     9     7
+2   3     3     5    11    9
+2   3     7    9    15    13
+2   3     9    11    17    15
+1   3     6     0     1     7
+1   3    12     6     7    13
+1   3     4    10    11     5
+1   3    10    16    17    11
+
+vertices
+18
+3
+         0    0.2464    0.2464
+    0.5071    0.2464    0.2464
+         0    0.5000    0.2464
+    0.5071    0.5000    0.2464
+         0    0.7536    0.2464
+    0.5071    0.7536    0.2464
+         0    0.2464    0.5000
+    0.5071    0.2464    0.5000
+         0    0.5000    0.5000
+    0.5071    0.5000    0.5000
+         0    0.7536    0.5000
+    0.5071    0.7536    0.5000
+         0    0.2464    0.7536
+    0.5071    0.2464    0.7536
+         0    0.5000    0.7536
+    0.5071    0.5000    0.7536
+         0    0.7536    0.7536
+    0.5071    0.7536    0.7536

miniapps/contact/meshes/rotatedblock2.mesh

@@ -0,0 +1,70 @@
+
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+# 1 nothing
+elements
+4
+1   5     0     1     3     2     6     7    9     8
+1   5     2     3     5     4     8     9    11    10
+1   5     6     7     9     8    12    13    15    14
+1   5     8     9    11    10    14    15    17    16
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+16
+1   3     1     0     2     3
+1   3     3     2     4     5
+1   3    12    13    15    14
+1   3    14    15    17    16
+3   3     2     0     6     8
+3   3     4     2     8    10
+3   3     8     6    12    14
+3   3    10     8    14    16
+2   3     1     3     9     7
+2   3     3     5    11    9
+2   3     7    9    15    13
+2   3     9    11    17    15
+1   3     6     0     1     7
+1   3    12     6     7    13
+1   3     4    10    11     5
+1   3    10    16    17    11
+
+vertices
+18
+3
+
+0.000000000000 0.145770950245 0.443895630208
+0.507100000000 0.145770950245 0.443895630208
+0.000000000000 0.350937660019 0.294833290227
+0.507100000000 0.350937660019 0.294833290227
+0.000000000000 0.556104369792 0.145770950245
+0.507100000000 0.556104369792 0.145770950245
+0.000000000000 0.294833290227 0.649062339981
+0.507100000000 0.294833290227 0.649062339981
+0.000000000000 0.500000000000 0.500000000000
+0.507100000000 0.500000000000 0.500000000000
+0.000000000000 0.705166709773 0.350937660019
+0.507100000000 0.705166709773 0.350937660019
+0.000000000000 0.443895630208 0.854229049755
+0.507100000000 0.443895630208 0.854229049755
+0.000000000000 0.649062339981 0.705166709773
+0.507100000000 0.649062339981 0.705166709773
+0.000000000000 0.854229049755 0.556104369792
+0.507100000000 0.854229049755 0.556104369792

miniapps/contact/pcontact_driver.cpp

@@ -0,0 +1,254 @@
+//                               Parallel contact example
+//
+// Compile with: make pcontact_driver
+// sample run
+// mpirun -np 6 ./pcontact_driver -sr 2 -pr 2
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include "ipsolver/ParIPsolver.hpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int myid = Mpi::WorldRank();
+   int num_procs = Mpi::WorldSize();
+   Hypre::Init();
+   // 1. Parse command-line options.
+   const char *mesh_file = "meshes/merged.mesh";
+   int order = 1;
+   int sref = 0;
+   int pref = 0;
+   Array<int> attr;
+   Array<int> m_attr;
+   bool visualization = true;
+   bool paraview = false;
+   double linsolvertol = 1e-6;
+   int relax_type = 8;
+   double optimizer_tol = 1e-6;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&attr, "-at", "--attributes-surf",
+                  "Attributes of boundary faces on contact surface for mesh 2.");
+   args.AddOption(&sref, "-sr", "--serial-refinements",
+                  "Number of uniform refinements.");
+   args.AddOption(&pref, "-pr", "--parallel-refinements",
+                  "Number of uniform refinements.");
+   args.AddOption(&linsolvertol, "-stol", "--solver-tol",
+                  "Linear Solver Tolerance.");
+   args.AddOption(&optimizer_tol, "-otol", "--optimizer-tol",
+                  "Interior Point Solver Tolerance.");
+   args.AddOption(&relax_type, "-rt", "--relax-type",
+                  "Selection of Smoother for AMG");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview", "-no-paraview",
+                  "--no-paraview",
+                  "Enable or disable ParaView visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh * merged_mesh = new Mesh(mesh_file,1);
+
+
+   Array<int> attr1; attr1.Append(1);
+   Array<int> attr2; attr2.Append(2);
+   Mesh * mesh1 = new Mesh(SubMesh::CreateFromDomain(*merged_mesh,attr1));
+   Mesh * mesh2 = new Mesh(SubMesh::CreateFromDomain(*merged_mesh,attr2));
+
+   for (int i = 0; i<sref; i++)
+   {
+      mesh1->UniformRefinement();
+      mesh2->UniformRefinement();
+   }
+   for (int i = 0; i<mesh1->GetNE(); i++)
+   {
+      mesh1->SetAttribute(i,1);
+   }
+   mesh1->SetAttributes();
+   for (int i = 0; i<mesh2->GetNE(); i++)
+   {
+      mesh2->SetAttribute(i,2);
+   }
+   mesh2->SetAttributes();
+
+   ParMesh * pmesh1 = new ParMesh(MPI_COMM_WORLD,*mesh1);
+   ParMesh * pmesh2 = new ParMesh(MPI_COMM_WORLD,*mesh2);
+
+   for (int i = 0; i<pref; i++)
+   {
+      pmesh1->UniformRefinement();
+      pmesh2->UniformRefinement();
+   }
+
+   MFEM_VERIFY(pmesh1->GetNE(), "Empty partition mesh1");
+   MFEM_VERIFY(pmesh2->GetNE(), "Empty partition mesh2");
+
+   ParElasticityProblem * prob1 = new ParElasticityProblem(pmesh1,order);
+   ParElasticityProblem * prob2 = new ParElasticityProblem(pmesh2,order);
+
+
+   Vector lambda1(prob1->GetMesh()->attributes.Max()); lambda1 = 57.6923076923;
+   Vector mu1(prob1->GetMesh()->attributes.Max()); mu1 = 38.4615384615;
+   Vector lambda2(prob2->GetMesh()->attributes.Max()); lambda2 = 57.6923076923;
+   Vector mu2(prob2->GetMesh()->attributes.Max()); mu2 = 38.4615384615;
+
+   prob1->SetLambda(lambda1); prob1->SetMu(mu1);
+   prob2->SetLambda(lambda2); prob2->SetMu(mu2);
+
+   ParContactProblem contact(prob1,prob2);
+   QPOptParContactProblem qpopt(&contact);
+   int numconstr = contact.GetGlobalNumConstraints();
+
+   ParInteriorPointSolver optimizer(&qpopt);
+
+   optimizer.SetTol(optimizer_tol);
+   optimizer.SetMaxIter(50);
+
+   int linsolver = 2;
+   optimizer.SetLinearSolver(linsolver);
+   optimizer.SetLinearSolveTol(linsolvertol);
+   optimizer.SetLinearSolveRelaxType(relax_type);
+
+   ParGridFunction x1 = prob1->GetDisplacementGridFunction();
+   ParGridFunction x2 = prob2->GetDisplacementGridFunction();
+
+   int ndofs1 = prob1->GetNumTDofs();
+   int ndofs2 = prob2->GetNumTDofs();
+   int gndofs1 = prob1->GetGlobalNumDofs();
+   int gndofs2 = prob2->GetGlobalNumDofs();
+   int ndofs = ndofs1 + ndofs2;
+
+   Vector X1 = x1.GetTrueVector();
+   Vector X2 = x2.GetTrueVector();
+
+   Vector x0(ndofs); x0 = 0.0;
+   x0.SetVector(X1,0);
+   x0.SetVector(X2,X1.Size());
+
+   Vector xf(ndofs); xf = 0.0;
+   optimizer.Mult(x0, xf);
+
+   double Einitial = contact.E(x0);
+   double Efinal = contact.E(xf);
+   Array<int> & CGiterations = optimizer.GetCGIterNumbers();
+   if (Mpi::Root())
+   {
+      mfem::out << endl;
+      mfem::out << " Initial Energy objective     = " << Einitial << endl;
+      mfem::out << " Final Energy objective       = " << Efinal << endl;
+      mfem::out << " Global number of dofs        = " << gndofs1 + gndofs2 << endl;
+      mfem::out << " Global number of constraints = " << numconstr << endl;
+      mfem::out << " CG iteration numbers         = " ;
+      CGiterations.Print(mfem::out, CGiterations.Size());
+   }
+
+   MFEM_VERIFY(optimizer.GetConverged(),
+               "Interior point solver did not converge.");
+
+
+   if (visualization || paraview)
+   {
+      ParFiniteElementSpace * fes1 = prob1->GetFESpace();
+      ParFiniteElementSpace * fes2 = prob2->GetFESpace();
+
+      ParMesh * pmesh_1 = fes1->GetParMesh();
+      ParMesh * pmesh_2 = fes2->GetParMesh();
+
+      Vector X1_new(xf.GetData(),fes1->GetTrueVSize());
+      Vector X2_new(&xf.GetData()[fes1->GetTrueVSize()],fes2->GetTrueVSize());
+
+      ParGridFunction x1_gf(fes1);
+      ParGridFunction x2_gf(fes2);
+
+      x1_gf.SetFromTrueDofs(X1_new);
+      x2_gf.SetFromTrueDofs(X2_new);
+
+      pmesh_1->MoveNodes(x1_gf);
+      pmesh_2->MoveNodes(x2_gf);
+
+      if (paraview)
+      {
+         ParaViewDataCollection paraview_dc1("QPContactBody1", pmesh_1);
+         paraview_dc1.SetPrefixPath("ParaView");
+         paraview_dc1.SetLevelsOfDetail(1);
+         paraview_dc1.SetDataFormat(VTKFormat::BINARY);
+         paraview_dc1.SetHighOrderOutput(true);
+         paraview_dc1.SetCycle(0);
+         paraview_dc1.SetTime(0.0);
+         paraview_dc1.RegisterField("Body1", &x1_gf);
+         paraview_dc1.Save();
+
+         ParaViewDataCollection paraview_dc2("QPContactBody2", pmesh_2);
+         paraview_dc2.SetPrefixPath("ParaView");
+         paraview_dc2.SetLevelsOfDetail(1);
+         paraview_dc2.SetDataFormat(VTKFormat::BINARY);
+         paraview_dc2.SetHighOrderOutput(true);
+         paraview_dc2.SetCycle(0);
+         paraview_dc2.SetTime(0.0);
+         paraview_dc2.RegisterField("Body2", &x2_gf);
+         paraview_dc2.Save();
+      }
+
+
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int visport = 19916;
+
+         {
+            socketstream sol_sock1(vishost, visport);
+            sol_sock1.precision(8);
+            sol_sock1 << "parallel " << num_procs << " " << myid << "\n"
+                      << "solution\n" << *pmesh_1 << x1_gf << flush;
+         }
+         {
+            socketstream sol_sock2(vishost, visport);
+            sol_sock2.precision(8);
+            sol_sock2 << "parallel " << num_procs << " " << myid << "\n"
+                      << "solution\n" << *pmesh_2 << x2_gf << flush;
+         }
+
+         // {
+         //    socketstream sol_sock(vishost, visport);
+         //    sol_sock.precision(8);
+         //    sol_sock << "parallel " << 2*num_procs << " " << myid << "\n"
+         //             << "solution\n" << *pmesh_1 << x1_gf << flush;
+         // }
+         // {
+         //    socketstream sol_sock(vishost, visport);
+         //    sol_sock.precision(8);
+         //    sol_sock << "parallel " << 2*num_procs << " " << myid+num_procs << "\n"
+         //             << "solution\n" << *pmesh_2 << x2_gf << flush;
+         // }
+      }
+   }
+
+   delete prob2;
+   delete prob1;
+   delete pmesh2;
+   delete pmesh1;
+   // delete mesh1;
+   // delete mesh2;
+
+   return 0;
+}

miniapps/contact/problems/parproblems.cpp

@@ -0,0 +1,516 @@
+#include "parproblems.hpp"
+
+void ParElasticityProblem::Init()
+{
+   int dim = pmesh->Dimension();
+   fec = new H1_FECollection(order,dim);
+   fes = new ParFiniteElementSpace(pmesh,fec,dim,Ordering::byVDIM);
+   ndofs = fes->GetVSize();
+   ntdofs = fes->GetTrueVSize();
+   gndofs = fes->GlobalTrueVSize();
+   pmesh->SetNodalFESpace(fes);
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+   }
+   ess_bdr = 0; ess_bdr[1] = 1;
+   fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
+   // Solution GridFunction
+   x.SetSpace(fes);  x = 0.0;
+   // RHS
+   b.Update(fes);
+
+   // Elasticity operator
+   lambda.SetSize(pmesh->attributes.Max()); lambda = 57.6923076923;
+   mu.SetSize(pmesh->attributes.Max()); mu = 38.4615384615;
+
+   lambda_cf.UpdateConstants(lambda);
+   mu_cf.UpdateConstants(mu);
+
+   a = new ParBilinearForm(fes);
+   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_cf,mu_cf));
+}
+
+void ParElasticityProblem::FormLinearSystem()
+{
+   if (!formsystem) 
+   {
+      formsystem = true;
+      b.Assemble();
+      a->Assemble();
+      a->FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   }
+}
+
+void ParElasticityProblem::UpdateLinearSystem()
+{
+   if (formsystem)
+   {
+      b.Update();
+      a->Update();
+      formsystem = false;
+   }
+   FormLinearSystem();
+}
+
+ParContactProblem::ParContactProblem(ParElasticityProblem * prob1_, ParElasticityProblem * prob2_)
+: prob1(prob1_), prob2(prob2_)
+{
+   ParMesh* pmesh1 = prob1->GetMesh();
+   comm = pmesh1->GetComm();
+   MPI_Comm_rank(comm, &myid);
+   MPI_Comm_size(comm, &numprocs);
+ 
+   dim = pmesh1->Dimension();
+   nodes0.SetSpace(pmesh1->GetNodes()->FESpace());
+   nodes0 = *pmesh1->GetNodes();
+   nodes1 = pmesh1->GetNodes();
+   Vector delta1(dim);
+   delta1 = 0.0; delta1[0] = 0.1;
+   prob1->SetDisplacementDirichletData(delta1);
+   prob1->FormLinearSystem();
+
+   Vector delta2(dim);
+   delta2 = 0.0; 
+   prob2->SetDisplacementDirichletData(delta2);
+   prob2->FormLinearSystem();
+
+   int ndof1 = prob1->GetNumTDofs();
+   int ndof2 = prob2->GetNumTDofs();
+
+   tdof_offsets.SetSize(3);
+   tdof_offsets[0] = 0;
+   tdof_offsets[1] = ndof1;
+   tdof_offsets[2] = ndof2;
+   tdof_offsets.PartialSum();
+
+   Array2D<HypreParMatrix*> A(2,2);
+   A(0,0) = &prob1->GetOperator();
+   A(1,1) = &prob2->GetOperator();
+   A(1,0) = nullptr;
+   A(0,1) = nullptr;
+   K = HypreParMatrixFromBlocks(A);
+
+   B = new BlockVector(tdof_offsets);
+   B->GetBlock(0).Set(1.0, prob1->GetRHS());
+   B->GetBlock(1).Set(1.0, prob2->GetRHS());
+
+   ComputeContactVertices();
+}
+
+void ParContactProblem::ComputeContactVertices()
+{
+   if (gnpoints>0) return;
+
+   ParMesh * pmesh1 = prob1->GetMesh();
+   ParMesh * pmesh2 = prob2->GetMesh();
+   dim = pmesh1->Dimension();
+
+   vfes1 = new ParFiniteElementSpace(pmesh1, prob1->GetFECol());
+   vfes2 = new ParFiniteElementSpace(pmesh2, prob2->GetFECol());
+
+   int gnv1 = vfes1->GlobalTrueVSize();
+   int gnv2 = vfes2->GlobalTrueVSize();
+   gnv = gnv1+gnv2;
+   int nv1 = vfes1->GetTrueVSize();
+   int nv2 = vfes2->GetTrueVSize();
+   nv = nv1+nv2;
+
+   vertices1.SetSize(pmesh1->GetNV());
+   vertices2.SetSize(pmesh2->GetNV());
+
+   for (int i = 0; i<pmesh1->GetNV(); i++)
+   {
+      vertices1[i] = i;
+   }
+   pmesh1->GetGlobalVertexIndices(vertices1);
+
+   for (int i = 0; i<pmesh2->GetNV(); i++)
+   {
+      vertices2[i] = i;
+   }
+   pmesh2->GetGlobalVertexIndices(vertices2);
+
+   int voffset2 = vfes2->GetMyTDofOffset();
+
+   std::vector<int> vertex2_offsets;
+   ComputeTdofOffsets(comm,voffset2, vertex2_offsets);
+
+   Array<int> vert;
+   for (int b=0; b<pmesh2->GetNBE(); b++)
+   {
+      if (pmesh2->GetBdrAttribute(b) == 3)
+      {
+         pmesh2->GetBdrElementVertices(b, vert);
+         for (auto v : vert)
+         {
+            if (myid != get_rank(vertices2[v],vertex2_offsets)) { continue; }
+            contact_vertices.insert(v);
+         }
+      }
+   }
+
+   npoints = contact_vertices.size();
+
+   MPI_Allreduce(&npoints, &gnpoints,1,MPI_INT,MPI_SUM,pmesh1->GetComm());
+   int constrains_offset;
+   MPI_Scan(&npoints,&constrains_offset,1,MPI_INT,MPI_SUM,pmesh1->GetComm());
+
+   constrains_offset-=npoints;
+   constraints_starts.SetSize(2);
+   constraints_starts[0] = constrains_offset;
+   constraints_starts[1] = constrains_offset+npoints;
+
+   ComputeTdofOffsets(comm,constrains_offset, constraints_offsets);
+}
+
+void ParContactProblem::ComputeGapFunctionAndDerivatives(const Vector & displ1, const Vector &displ2)
+{
+   ComputeContactVertices();
+   ParMesh * pmesh1 = prob1->GetMesh();
+   ParMesh * pmesh2 = prob2->GetMesh();
+
+   ParGridFunction displ1_gf(prob1->GetFESpace());
+   ParGridFunction displ2_gf(prob2->GetFESpace());
+
+   displ1_gf.SetFromTrueDofs(displ1);
+   displ2_gf.SetFromTrueDofs(displ2);
+
+   Array<int> conn2(npoints); 
+   Vector xyz(dim * npoints);
+
+   int cnt = 0;
+   for (auto v : contact_vertices)
+   {
+      for (int d = 0; d<dim; d++)
+      {
+         xyz(cnt*dim + d) = pmesh2->GetVertex(v)[d]+displ2_gf[v*dim+d];
+      }
+      conn2[cnt] = vertices2[v];
+      cnt++;
+   }
+
+   MFEM_VERIFY(cnt == npoints, "");
+   gapv.SetSize(npoints*dim); gapv = 0.0;
+   // segment reference coordinates of the closest point
+   Vector xi1(npoints*(dim-1));
+   Array<int> conn1(npoints*4);
+   DenseMatrix coordsm(npoints*4, dim);
+   // add(nodes0, displ1_gf, *nodes1);
+   FindPointsInMesh(*pmesh1, vertices1, conn2, displ1_gf, xyz, conn1, xi1, coordsm);
+   if (M)
+   {
+      delete M;
+      for (int i = 0; i<dM.Size(); i++)
+      {
+         delete dM[i];
+      }
+      dM.SetSize(0);
+   }
+
+   int ndofs1 = prob1->GetFESpace()->GetTrueVSize();
+   int ndofs2 = prob2->GetFESpace()->GetTrueVSize();
+   int gndofs1 = prob1->GetFESpace()->GlobalTrueVSize();
+   int gndofs2 = prob2->GetFESpace()->GlobalTrueVSize();
+   
+   Array<int> npts(numprocs);
+   MPI_Allgather(&npoints,1,MPI_INT,&npts[0],1,MPI_INT,comm);
+   npts.PartialSum(); npts.Prepend(0);
+
+   SparseMatrix S1(gnpoints,gndofs1);
+   SparseMatrix S2(gnpoints,gndofs2);
+   Array<SparseMatrix *> dS11;
+   Array<SparseMatrix *> dS12;
+   Array<SparseMatrix *> dS21;
+   Array<SparseMatrix *> dS22;
+
+   // local to global map for constraints
+   Array<int> points_map(npoints);
+   cnt = 0;
+   for (int i = 0; i<gnpoints; i++)
+   {
+      if (i >= npts[myid] && i< npts[myid+1])
+      {
+         points_map[cnt++] = i;
+      }
+   }
+   if (compute_hessians)
+   {
+      dS11.SetSize(gnpoints);
+      dS12.SetSize(gnpoints);
+      dS21.SetSize(gnpoints);
+      dS22.SetSize(gnpoints);
+      for (int i = 0; i<gnpoints; i++)
+      {
+         if (i >= npts[myid] && i< npts[myid+1])
+         {
+            dS11[i] = new SparseMatrix(gndofs1,gndofs1);
+            dS12[i] = new SparseMatrix(gndofs1,gndofs2);
+            dS21[i] = new SparseMatrix(gndofs2,gndofs1);
+            dS22[i] = new SparseMatrix(gndofs2,gndofs2);
+         }
+         else
+         {
+            dS11[i] = nullptr;
+            dS12[i] = nullptr;
+            dS21[i] = nullptr;
+            dS22[i] = nullptr;
+         }
+      }
+      Assemble_Contact(xyz, xi1, coordsm, conn2, conn1, gapv, S1,S2,
+                      dS11,dS12,dS21,dS22);
+   }
+   else
+   {
+      Assemble_Contact(xyz, xi1, coordsm, conn2, conn1, gapv, S1,S2, points_map);
+   }                   
+                
+   // --------------------------------------------------------------------
+   // Redistribute the M block matrix [M1 M2]
+   // --------------------------------------------------------------------
+   int offset = constraints_offsets[myid];
+   MPICommunicator Mcomm1(comm,offset,gnpoints);
+   SparseMatrix localS1(npoints,gndofs1);
+   Mcomm1.Communicate(S1,localS1);
+   MPICommunicator Mcomm2(comm,offset,gnpoints);
+   SparseMatrix localS2(npoints,gndofs2);
+   Mcomm2.Communicate(S2,localS2);
+   
+   MFEM_VERIFY(HYPRE_AssumedPartitionCheck(), "Hypre_AssumedPartitionCheck is False");
+
+   // Construct M row and col starts to construct HypreParMatrix
+   int M1rows[2], M2rows[2]; 
+   int M1cols[2], M2cols[2];
+   M1rows[0] = constraints_starts[0];
+   M1rows[1] = constraints_starts[1];
+
+   M2rows[0] = constraints_starts[0];
+   M2rows[1] = constraints_starts[1];
+
+   M1cols[0] = prob1->GetFESpace()->GetTrueDofOffsets()[0];
+   M1cols[1] = prob1->GetFESpace()->GetTrueDofOffsets()[1];
+
+   M2cols[0] = prob2->GetFESpace()->GetTrueDofOffsets()[0];
+   M2cols[1] = prob2->GetFESpace()->GetTrueDofOffsets()[1];
+
+   Array2D<HypreParMatrix*> blockM(1,2);
+   blockM(0,0) = new HypreParMatrix(comm,npoints,gnpoints,gndofs1,
+                          localS1.GetI(), localS1.GetJ(),localS1.GetData(),
+                          M1rows,M1cols);
+
+   blockM(0,1) = new HypreParMatrix(comm,npoints,gnpoints,gndofs2,
+                          localS2.GetI(), localS2.GetJ(),localS2.GetData(),
+                          M2rows,M2cols);
+
+   M = HypreParMatrixFromBlocks(blockM);
+   delete blockM(0,0);
+   delete blockM(0,1);
+   blockM.DeleteAll();
+
+   if (compute_hessians)
+   {
+      Array<SparseMatrix*> localdS11(gnpoints);
+      Array<SparseMatrix*> localdS12(gnpoints);
+      Array<SparseMatrix*> localdS21(gnpoints);
+      Array<SparseMatrix*> localdS22(gnpoints);
+      for (int k = 0; k<gnpoints; k++)
+      {
+         localdS11[k] = new SparseMatrix(ndofs1,gndofs1); 
+         localdS12[k] = new SparseMatrix(ndofs1,gndofs2); 
+         localdS21[k] = new SparseMatrix(ndofs2,gndofs1); 
+         localdS22[k] = new SparseMatrix(ndofs2,gndofs2); 
+      }
+
+      int offset1 = prob1->GetFESpace()->GetMyTDofOffset();
+      int offset2 = prob2->GetFESpace()->GetMyTDofOffset();
+
+      MPICommunicator dmcomm11(comm, offset1, gndofs1);
+      dmcomm11.Communicate(dS11,localdS11);
+      for (int k = 0; k<gnpoints; k++) { delete dS11[k]; }
+
+      MPICommunicator dmcomm12(comm, offset1, gndofs1);
+      dmcomm12.Communicate(dS12,localdS12);
+      for (int k = 0; k<gnpoints; k++) { delete dS12[k]; }
+
+      MPICommunicator dmcomm21(comm, offset2, gndofs2);
+      dmcomm21.Communicate(dS21,localdS21);
+      for (int k = 0; k<gnpoints; k++) { delete dS21[k]; }
+
+      MPICommunicator dmcomm22(comm, offset2, gndofs2);
+      dmcomm22.Communicate(dS22,localdS22);
+      for (int k = 0; k<gnpoints; k++) { delete dS22[k]; }
+
+      // --------------------------------------------------------------------
+      // Redistribute the block dM matrices [dM11 dM12; dM21 dM22]
+      // --------------------------------------------------------------------
+
+      // Construct dMi HypreParMatrix
+      Array2D<HypreParMatrix *> dMs(2,2);
+      dM.SetSize(gnpoints);
+      int * offs1 = prob1->GetFESpace()->GetTrueDofOffsets();
+      int * offs2 = prob2->GetFESpace()->GetTrueDofOffsets();
+      for (int i = 0; i<gnpoints; i++)
+      {
+         dMs(0,0) = new HypreParMatrix(comm, ndofs1, gndofs1, gndofs1, 
+                                      localdS11[i]->GetI(), localdS11[i]->GetJ(), 
+                                      localdS11[i]->GetData(),
+                                      offs1,offs1);
+         delete localdS11[i];                                 
+         dMs(0,1) = new HypreParMatrix(comm, ndofs1, gndofs1, gndofs2, 
+                                      localdS12[i]->GetI(), localdS12[i]->GetJ(), 
+                                      localdS12[i]->GetData(),
+                                      offs1,offs2);   
+         delete localdS12[i];                                 
+         dMs(1,0) = new HypreParMatrix(comm, ndofs2, gndofs2, gndofs1, 
+                                      localdS21[i]->GetI(), localdS21[i]->GetJ(), 
+                                      localdS21[i]->GetData(),
+                                      offs2,offs1);
+         delete localdS21[i];                                 
+         dMs(1,1) = new HypreParMatrix(comm, ndofs2, gndofs2, gndofs2, 
+                                      localdS22[i]->GetI(), localdS22[i]->GetJ(), 
+                                      localdS22[i]->GetData(),
+                                      offs2,offs2);    
+         delete localdS22[i];                                 
+
+         dM[i] = HypreParMatrixFromBlocks(dMs);
+         delete dMs(0,0);
+         delete dMs(0,1);
+         delete dMs(1,0);
+         delete dMs(1,1);
+      }
+      dMs.DeleteAll();
+   }
+}
+
+double ParContactProblem::E(const Vector & d)
+{
+   Vector kd(K->Height());
+   K->Mult(d,kd);
+   return 0.5 * InnerProduct(comm,d, kd) - InnerProduct(comm,d, *B);
+}
+
+void ParContactProblem::DdE(const Vector &d, Vector &gradE)
+{
+   gradE.SetSize(K->Height());
+   K->Mult(d, gradE);
+   gradE.Add(-1.0, *B); 
+}
+
+HypreParMatrix* ParContactProblem::DddE(const Vector &d)
+{
+   return K; 
+}
+
+void ParContactProblem::g(const Vector &d, Vector &gd, bool compute_hessians_)
+{
+   compute_hessians = compute_hessians_;
+   int ndof1 = prob1->GetNumTDofs();
+   int ndof2 = prob2->GetNumTDofs();
+   double * data = d.GetData();
+   Vector displ1(data,ndof1);
+   Vector displ2(&data[ndof1],ndof2);
+
+   if (recompute)
+   {
+      ComputeGapFunctionAndDerivatives(displ1, displ2);
+      recompute = false;
+   }
+
+   gd = GetGapFunction();
+}
+
+HypreParMatrix* ParContactProblem::Ddg(const Vector &d)
+{
+  return GetJacobian();
+}
+
+HypreParMatrix* ParContactProblem::lDddg(const Vector &d, const Vector &l)
+{
+   return nullptr; // for now
+}
+
+
+QPOptParContactProblem::QPOptParContactProblem(ParContactProblem * problem_)
+: problem(problem_)
+{
+   dimU = problem->GetNumDofs();
+   dimM = problem->GetNumContraints();
+   dimC = problem->GetNumContraints();
+   ml.SetSize(dimM); ml = 0.0;
+   Vector negone(dimM); negone = -1.0;
+   SparseMatrix diag(negone);
+
+   int gsize = problem->GetGlobalNumConstraints();
+   int * rows = problem->GetConstraintsStarts().GetData();
+
+   NegId = new HypreParMatrix(problem->GetComm(),gsize, rows,&diag);
+   HypreStealOwnership(*NegId, diag);
+}
+
+int QPOptParContactProblem::GetDimU() { return dimU; }
+
+int QPOptParContactProblem::GetDimM() { return dimM; }
+
+int QPOptParContactProblem::GetDimC() { return dimC; }
+
+Vector & QPOptParContactProblem::Getml() { return ml; }
+
+HypreParMatrix * QPOptParContactProblem::Duuf(const BlockVector & x)
+{
+   return problem->DddE(x.GetBlock(0));
+}
+
+HypreParMatrix * QPOptParContactProblem::Dumf(const BlockVector & x)
+{
+   return nullptr;
+}
+
+HypreParMatrix * QPOptParContactProblem::Dmuf(const BlockVector & x)
+{
+   return nullptr;
+}
+
+HypreParMatrix * QPOptParContactProblem::Dmmf(const BlockVector & x)
+{
+   return nullptr;
+}
+
+HypreParMatrix * QPOptParContactProblem::Duc(const BlockVector & x)
+{
+   return problem->Ddg(x.GetBlock(0));
+}
+
+HypreParMatrix * QPOptParContactProblem::Dmc(const BlockVector & x)
+{
+   return NegId;
+}
+
+HypreParMatrix * QPOptParContactProblem::lDuuc(const BlockVector & x, const Vector & l)
+{
+   return nullptr;
+}
+
+void QPOptParContactProblem::c(const BlockVector &x, Vector & y)
+{
+   Vector g0;
+   problem->g(x.GetBlock(0),g0, false); // gap function
+   g0.Add(-1.0, x.GetBlock(1));  
+   problem->GetJacobian()->Mult(x.GetBlock(0),y);
+   y.Add(1.0, g0);
+}
+
+double QPOptParContactProblem::CalcObjective(const BlockVector & x)
+{
+   return problem->E(x.GetBlock(0));
+}
+
+void QPOptParContactProblem::CalcObjectiveGrad(const BlockVector & x, BlockVector & y)
+{
+   problem->DdE(x.GetBlock(0), y.GetBlock(0));
+   y.GetBlock(1) = 0.0;
+}
+
+QPOptParContactProblem::~QPOptParContactProblem()
+{
+   delete NegId;
+}

miniapps/contact/problems/parproblems.hpp

@@ -0,0 +1,218 @@
+
+#include "parproblems_util.hpp"
+
+class ParElasticityProblem
+{
+private:
+   MPI_Comm comm;
+   bool formsystem = false;
+   ParMesh * pmesh = nullptr;
+   int order;
+   int ndofs;
+   int ntdofs;
+   int gndofs;
+   FiniteElementCollection * fec = nullptr;
+   ParFiniteElementSpace * fes = nullptr;
+   Vector lambda, mu;
+   PWConstCoefficient lambda_cf, mu_cf;
+   Array<int> ess_bdr, ess_tdof_list;
+   ParBilinearForm *a=nullptr;
+   ParLinearForm b;
+   ParGridFunction x;
+   HypreParMatrix A;
+   Vector B,X;
+   void Init();
+   bool own_mesh;
+public:
+   ParElasticityProblem(MPI_Comm comm_, const char *mesh_file , int sref, int pref, int order_ = 1) : comm(comm_), order(order_) 
+   {
+      own_mesh = true;
+      Mesh * mesh = new Mesh(mesh_file,1,1);
+      for (int i = 0; i<sref; i++)
+      {
+         mesh->UniformRefinement();
+      }
+      pmesh = new ParMesh(comm,*mesh);
+      MFEM_VERIFY(pmesh->GetNE(), "ParElasticityProblem::Empty partition");
+      delete mesh;
+      for (int i = 0; i<pref; i++)
+      {
+         pmesh->UniformRefinement();
+      }
+      Init();
+   }
+
+   ParElasticityProblem(ParMesh * pmesh_, int order_ = 1) :  pmesh(pmesh_), order(order_)
+   {
+      own_mesh = false;
+      comm = pmesh->GetComm();
+      Init();
+   }
+
+   ParMesh * GetMesh() { return pmesh; }
+   ParFiniteElementSpace * GetFESpace() { return fes; }
+   FiniteElementCollection * GetFECol() { return fec; }
+   int GetNumDofs() { return ndofs; }
+   int GetNumTDofs() { return ntdofs; }
+   int GetGlobalNumDofs() { return gndofs; }
+   HypreParMatrix & GetOperator() 
+   { 
+      MFEM_VERIFY(formsystem, "System not formed yet. Call FormLinearSystem()"); 
+      return A; 
+   }
+   Vector & GetRHS() 
+   { 
+      MFEM_VERIFY(formsystem, "System not formed yet. Call FormLinearSystem()"); 
+      return B; 
+   }
+
+   void SetLambda(const Vector & lambda_) 
+   { 
+      lambda = lambda_; 
+      lambda_cf.UpdateConstants(lambda);
+   }
+   void SetMu(const Vector & mu_) 
+   { 
+      mu = mu_; 
+      mu_cf.UpdateConstants(mu);
+   }
+
+   void FormLinearSystem();
+   void UpdateLinearSystem();
+
+   void SetDisplacementDirichletData(const Vector & delta) 
+   {
+      VectorConstantCoefficient delta_cf(delta);
+      x.ProjectBdrCoefficient(delta_cf,ess_bdr);
+   };
+
+   ParGridFunction & GetDisplacementGridFunction() {return x;};
+   Array<int> & GetEssentialDofs() {return ess_tdof_list;};
+
+   ~ParElasticityProblem()
+   {
+      delete a;
+      delete fes;
+      delete fec;
+      if (own_mesh)
+      {
+         delete pmesh;
+      }
+   }
+};
+
+
+class ParContactProblem
+{
+private:
+   MPI_Comm comm;
+   int numprocs;
+   int myid;
+   ParElasticityProblem * prob1 = nullptr;
+   ParElasticityProblem * prob2 = nullptr;
+   ParFiniteElementSpace * vfes1 = nullptr;
+   ParFiniteElementSpace * vfes2 = nullptr;
+   int dim;
+   GridFunction nodes0;
+   GridFunction *nodes1 = nullptr;
+   std::set<int> contact_vertices;
+   bool recompute = true;
+   bool compute_hessians = true;
+   std::vector<int> dof_offsets;
+   std::vector<int> vertex_offsets; 
+   std::vector<int> constraints_offsets; 
+   Array<int> tdof_offsets;
+   Array<int> constraints_starts;
+   Array<int> globalvertices1;
+   Array<int> globalvertices2;
+   Array<int> vertices2;
+   Array<int> vertices1;
+
+protected:
+   int npoints=0;
+   int gnpoints=0;
+   int nv, gnv;
+   HypreParMatrix * K = nullptr;
+   BlockVector *B = nullptr;
+   Vector gapv;
+   HypreParMatrix * M=nullptr;
+   Array<HypreParMatrix*> dM;
+   void ComputeContactVertices();
+
+public:
+   ParContactProblem(ParElasticityProblem * prob1_, ParElasticityProblem * prob2_);
+
+   ParElasticityProblem * GetElasticityProblem1() {return prob1;}
+   ParElasticityProblem * GetElasticityProblem2() {return prob2;}
+   MPI_Comm GetComm() {return comm;}
+   int GetNumDofs() {return K->Height();}
+   int GetGlobalNumDofs() {return K->GetGlobalNumRows();}
+   int GetNumContraints() {return npoints;}
+   int GetGlobalNumConstraints() {return gnpoints;}
+   
+   std::vector<int> & GetDofOffets() { return dof_offsets; }
+   std::vector<int> & GetVertexOffsets() { return vertex_offsets; }
+   std::vector<int> & GetConstraintsOffsets() { return constraints_offsets; }
+   Array<int> & GetConstraintsStarts() { return constraints_starts; }
+   
+   Vector & GetGapFunction() {return gapv;}
+
+   HypreParMatrix * GetJacobian() {return M;}
+   Array<HypreParMatrix*> & GetHessian() {return dM;}
+   void ComputeGapFunctionAndDerivatives(const Vector & displ1, const Vector &displ2);
+
+   double E(const Vector & d);
+   void DdE(const Vector &d, Vector &gradE);
+   HypreParMatrix* DddE(const Vector &d);
+   void g(const Vector &d, Vector &gd, bool compute_hessians_ = true);
+   HypreParMatrix* Ddg(const Vector &d);
+   HypreParMatrix* lDddg(const Vector &d, const Vector &l);
+
+   ~ParContactProblem()
+   {
+      delete B;
+      delete K;
+      delete M;
+      for (int i = 0; i<dM.Size(); i++)
+      {
+         delete dM[i];
+      }
+      delete vfes1;
+      delete vfes2;
+   }
+};
+
+
+class QPOptParContactProblem
+{
+private:
+   ParContactProblem * problem = nullptr;
+   int dimU, dimM, dimC;
+   // Array<int> block_offsets;
+   Vector ml;
+   HypreParMatrix * NegId = nullptr;
+public:
+   QPOptParContactProblem(ParContactProblem * problem_);
+   int GetDimU();
+   int GetDimM();
+   int GetDimC();
+   Vector & Getml();
+   MPI_Comm GetComm() {return problem->GetComm();}
+   int * GetConstraintsStarts() {return problem->GetConstraintsStarts().GetData();}
+   int GetGlobalNumConstraints() {return problem->GetGlobalNumConstraints();}
+
+   ParElasticityProblem * GetElasticityProblem1() {return problem->GetElasticityProblem1();}
+   ParElasticityProblem * GetElasticityProblem2() {return problem->GetElasticityProblem2();}
+
+   HypreParMatrix * Duuf(const BlockVector &);
+   HypreParMatrix * Dumf(const BlockVector &);
+   HypreParMatrix * Dmuf(const BlockVector &);
+   HypreParMatrix * Dmmf(const BlockVector &);
+   HypreParMatrix * Duc(const BlockVector &);
+   HypreParMatrix * Dmc(const BlockVector &);
+   HypreParMatrix * lDuuc(const BlockVector &, const Vector &);
+   void c(const BlockVector &, Vector &);
+   double CalcObjective(const BlockVector &);
+   void CalcObjectiveGrad(const BlockVector &, BlockVector &);
+   ~QPOptParContactProblem();
+};

miniapps/contact/problems/parproblems_util.cpp

@@ -0,0 +1,554 @@
+#include "parproblems_util.hpp"
+
+void FindPointsInMesh(Mesh & mesh, const Array<int> & gvert, const Vector & xyz, const Array<int> & s_conn, Array<int>& conn,
+                      Vector & xyz2, Array<int> & s_conn2, Vector& xi, DenseMatrix & coords)
+{
+   const int dim = mesh.Dimension();
+   const int np = xyz.Size() / dim;
+
+   MFEM_VERIFY(np * dim == xyz.Size(), "");
+
+   mesh.EnsureNodes();
+
+   FindPointsGSLIB finder(MPI_COMM_WORLD);
+
+   finder.SetDistanceToleranceForPointsFoundOnBoundary(0.5);
+
+   const double bb_t = 0.5;
+   finder.Setup(mesh, bb_t);
+
+   finder.FindPoints(xyz,mfem::Ordering::byVDIM);
+
+   Array<unsigned int> procs = finder.GetProc();
+
+   /// Return code for each point searched by FindPoints: inside element (0), on
+   /// element boundary (1), or not found (2).
+   Array<unsigned int> codes = finder.GetCode();
+
+   /// Return element number for each point found by FindPoints.
+   Array<unsigned int> elems = finder.GetElem();
+
+   /// Return reference coordinates for each point found by FindPoints.
+   Vector refcrd = finder.GetReferencePosition();
+
+   /// Return distance between the sought and the found point in physical space,
+   /// for each point found by FindPoints.
+   Vector dist = finder.GetDist();
+
+   finder.FreeData();
+
+   MFEM_VERIFY(dist.Size() == np, "");
+   MFEM_VERIFY(refcrd.Size() == np * dim, "");
+   MFEM_VERIFY(elems.Size() == np, "");
+   MFEM_VERIFY(codes.Size() == np, "");
+
+   bool allfound = true;
+   for (auto code : codes)
+      if (code == 2) { allfound = false; }
+
+   MFEM_VERIFY(allfound, "A point was not found");
+
+   // cout << "Maximum distance of projected points: " << dist.Max() << endl;
+
+
+   Array<unsigned int> elems_recv, proc_recv;
+   Vector ref_recv;
+   Vector xyz_recv;
+   Array<int> s_conn_recv;
+
+   MPICommunicator mycomm(MPI_COMM_WORLD, procs);
+   mycomm.Communicate(xyz,xyz_recv,3,mfem::Ordering::byNODES);
+   mycomm.Communicate(elems,elems_recv,1,mfem::Ordering::byVDIM);
+   mycomm.Communicate(refcrd,ref_recv,3,mfem::Ordering::byVDIM);
+   mycomm.Communicate(s_conn,s_conn_recv,1,mfem::Ordering::byVDIM);
+
+   proc_recv = mycomm.GetOriginProcs();
+
+   int np_loc = elems_recv.Size();
+   Array<int> conn_loc(np_loc*4);
+   Vector xi_send(np_loc*(dim-1));
+   for (int i=0; i<np_loc; ++i)
+   {
+      int refFace, refNormal;
+      // int refNormalSide;
+      bool is_interior = -1;
+
+      Vector normal = GetNormalVector(mesh, elems_recv[i],
+                                      ref_recv.GetData() + (i*dim),
+                                      refFace, refNormal, is_interior);
+
+      // continue;
+      int phyFace;
+      if (is_interior)
+      {
+         phyFace = -1; // the id of the face that has the closest point
+         FindSurfaceToProject(mesh, elems_recv[i], phyFace); // seems that this works
+
+         Array<int> cbdrVert;
+         mesh.GetFaceVertices(phyFace, cbdrVert);
+         Vector xs(dim);
+         xs[0] = xyz_recv[i + 0*np_loc];
+         xs[1] = xyz_recv[i + 1*np_loc];
+         xs[2] = xyz_recv[i + 2*np_loc];
+
+         Vector xi_tmp(dim-1);
+         // get nodes!
+
+         GridFunction *nodes = mesh.GetNodes();
+         DenseMatrix coord(4,3);
+         for (int j=0; j<4; j++)
+         {
+            for (int k=0; k<3; k++)
+            {
+               coord(j,k) = (*nodes)[cbdrVert[j]*3+k];
+            }
+         }
+         SlaveToMaster(coord, xs, xi_tmp);
+
+         for (int j=0; j<dim-1; ++j)
+         {
+            xi_send[i*(dim-1)+j] = xi_tmp[j];
+         }
+         // now get get the projection to the surface
+      }
+      else
+      {
+         Vector faceRefCrd(dim-1);
+         {
+            int fd = 0;
+            for (int j=0; j<dim; ++j)
+            {
+               if (j == refNormal)
+               {
+                  // refNormalSide = (ref_recv[(i*dim) + j] > 0.5); // not used
+               }
+               else
+               {
+                  faceRefCrd[fd] = ref_recv[(i*dim) + j];
+                  fd++;
+               }
+            }
+            MFEM_VERIFY(fd == dim-1, "");
+         }
+
+         for (int j=0; j<dim-1; ++j)
+         {
+            xi_send[i*(dim-1)+j] = faceRefCrd[j]*2.0 - 1.0;
+         }
+      }
+      // Get the element face
+      Array<int> faces;
+      Array<int> ori;
+      int face;
+
+      if (is_interior)
+      {
+         face = phyFace;
+      }
+      else
+      {
+         mesh.GetElementFaces(elems_recv[i], faces, ori);
+         face = faces[refFace];
+      }
+
+      Array<int> faceVert;
+      mesh.GetFaceVertices(face, faceVert);
+
+      for (int p=0; p<4; p++)
+      {
+         conn_loc[4*i+p] = faceVert[p];
+      }
+   }
+
+   if (0) // for debugging
+   {
+      int sz = xi_send.Size()/2;
+
+      for (int i = 0; i<sz; i++)
+      {
+         mfem::out << "("<<xi_send[i*(dim-1)]<<","<<xi_send[i*(dim-1)+1]<<"): -> ";
+         for (int j = 0; j<4; j++)
+         {
+            double * vc = mesh.GetVertex(conn_loc[4*i+j]);
+            if (j<3)
+            {
+               mfem::out << "("<<vc[0]<<","<<vc[1]<<","<<vc[2]<<"), ";
+            }
+            else
+            {
+               mfem::out << "("<<vc[0]<<","<<vc[1]<<","<<vc[2]<<") \n " << endl;
+            }
+         }
+      }
+   }
+
+   int sz = xi_send.Size()/2;
+   DenseMatrix coordsm(sz*4, dim);
+   for (int i = 0; i<sz; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         for (int k=0; k<dim; k++)
+         {
+            coordsm(i*4+j,k) = mesh.GetVertex(conn_loc[i*4+j])[k];
+         }
+      }
+   }
+
+   // pass global indices for conn_loc
+   for (int i = 0; i<conn_loc.Size(); i++)
+   {
+      conn_loc[i] = gvert[conn_loc[i]];
+   }
+
+   mycomm.UpdateDestinationProcs();
+   mycomm.Communicate(xyz_recv,xyz2,3,mfem::Ordering::byNODES);
+   mycomm.Communicate(xi_send,xi,2,mfem::Ordering::byVDIM);
+   mycomm.Communicate(s_conn_recv,s_conn2,1,mfem::Ordering::byVDIM);
+   mycomm.Communicate(conn_loc,conn,4,mfem::Ordering::byVDIM);
+   mycomm.Communicate(coordsm,coords,4,mfem::Ordering::byVDIM);
+}
+
+
+void FindPointsInMesh(Mesh & mesh, const Array<int> & gvert, Array<int> & s_conn, const Vector &x1, Vector & xyz, Array<int>& conn,
+                      Vector& xi, DenseMatrix & coords)
+{
+   const int dim = mesh.Dimension();
+   const int np = xyz.Size() / dim;
+   MFEM_VERIFY(np * dim == xyz.Size(), "");
+
+   mesh.EnsureNodes();
+
+   FindPointsGSLIB finder(MPI_COMM_WORLD);
+
+   finder.SetDistanceToleranceForPointsFoundOnBoundary(0.5);
+
+   const double bb_t = 0.5;
+   finder.Setup(mesh, bb_t);
+
+   finder.FindPoints(xyz,mfem::Ordering::byVDIM);
+
+   Array<unsigned int> procs = finder.GetProc();
+
+   /// Return code for each point searched by FindPoints: inside element (0), on
+   /// element boundary (1), or not found (2).
+   Array<unsigned int> codes = finder.GetCode();
+
+   /// Return element number for each point found by FindPoints.
+   Array<unsigned int> elems = finder.GetElem();
+
+   /// Return reference coordinates for each point found by FindPoints.
+   Vector refcrd = finder.GetReferencePosition();
+
+   /// Return distance between the sought and the found point in physical space,
+   /// for each point found by FindPoints.
+   Vector dist = finder.GetDist();
+
+   finder.FreeData();
+
+   MFEM_VERIFY(dist.Size() == np, "");
+   MFEM_VERIFY(refcrd.Size() == np * dim, "");
+   MFEM_VERIFY(elems.Size() == np, "");
+   MFEM_VERIFY(codes.Size() == np, "");
+
+   bool allfound = true;
+   for (auto code : codes)
+      if (code == 2) { allfound = false; }
+
+   MFEM_VERIFY(allfound, "A point was not found");
+
+   // reorder data so that the procs are in ascending order
+   // sort procs and save the permutation
+   std::vector<unsigned int> procs_index(np);
+   std::iota(procs_index.begin(),procs_index.end(),0); //Initializing
+   sort( procs_index.begin(),procs_index.end(), [&](int i,int j){return procs[i]<procs[j];} );
+
+   // map to sorted
+   Array<unsigned int> procs_sorted(np);
+   Array<unsigned int> elems_sorted(np);
+   Vector xyz_sorted(np*dim);
+   Vector refcrd_sorted(np*dim);
+   Array<int> s_conn_sorted(np);
+   for (int i = 0; i<np; i++)
+   {
+      int j = procs_index[i];
+      procs_sorted[i] = procs[j];
+      elems_sorted[i] = elems[j];
+      s_conn_sorted[i] = s_conn[j];
+      for (int d = 0; d<dim; d++)
+      {
+         xyz_sorted(i*dim+d) = xyz(j*dim+d);
+         refcrd_sorted(i*dim+d) = refcrd(j*dim+d);
+      }
+   }
+
+   Array<unsigned int> elems_recv, proc_recv;
+   xyz = xyz_sorted;
+   s_conn = s_conn_sorted;
+   Vector ref_recv;
+   Vector xyz_recv;
+
+   MPICommunicator mycomm(MPI_COMM_WORLD, procs_sorted);
+   mycomm.Communicate(xyz_sorted,xyz_recv,3,mfem::Ordering::byVDIM);
+   mycomm.Communicate(elems_sorted,elems_recv,1,mfem::Ordering::byVDIM);
+   mycomm.Communicate(refcrd_sorted,ref_recv,3,mfem::Ordering::byVDIM);
+
+
+   proc_recv = mycomm.GetOriginProcs();
+
+   int np_loc = elems_recv.Size();
+   Array<int> conn_loc(np_loc*4);
+   Vector xi_send(np_loc*(dim-1));
+   for (int i=0; i<np_loc; ++i)
+   {
+      int refFace, refNormal; 
+      // int refNormalSide;
+      bool is_interior = -1;
+
+      Vector normal = GetNormalVector(mesh, elems_recv[i],
+                                      ref_recv.GetData() + (i*dim),
+                                      refFace, refNormal, is_interior);
+
+      // continue;
+      int phyFace;
+      if (is_interior)
+      {
+         phyFace = -1; // the id of the face that has the closest point
+         FindSurfaceToProject(mesh, elems_recv[i], phyFace); // seems that this works
+
+         Array<int> cbdrVert;
+         mesh.GetFaceVertices(phyFace, cbdrVert);
+         Vector xs(dim);
+         xs[0] = xyz_recv[i*dim + 0];
+         xs[1] = xyz_recv[i*dim + 1];
+         xs[2] = xyz_recv[i*dim + 2];
+
+         Vector xi_tmp(dim-1);
+         // get nodes!
+
+         GridFunction *nodes = mesh.GetNodes();
+         DenseMatrix coord(4,3);
+         for (int j=0; j<4; j++)
+         {
+            for (int k=0; k<3; k++)
+            {
+               coord(j,k) = (*nodes)[cbdrVert[j]*3+k];
+            }
+         }
+         SlaveToMaster(coord, xs, xi_tmp);
+
+         for (int j=0; j<dim-1; ++j)
+         {
+            xi_send[i*(dim-1)+j] = xi_tmp[j];
+         }
+         // now get the projection to the surface
+      }
+      else
+      {
+         Vector faceRefCrd(dim-1);
+         {
+            int fd = 0;
+            for (int j=0; j<dim; ++j)
+            {
+               if (j == refNormal)
+               {
+                  // refNormalSide = (ref_recv[(i*dim) + j] > 0.5); // not used
+               }
+               else
+               {
+                  faceRefCrd[fd] = ref_recv[(i*dim) + j];
+                  fd++;
+               }
+            }
+            MFEM_VERIFY(fd == dim-1, "");
+         }
+
+         for (int j=0; j<dim-1; ++j)
+         {
+            xi_send[i*(dim-1)+j] = faceRefCrd[j]*2.0 - 1.0;
+         }
+      }
+      // Get the element face
+      Array<int> faces;
+      Array<int> ori;
+      int face;
+
+      if (is_interior)
+      {
+         face = phyFace;
+      }
+      else
+      {
+         mesh.GetElementFaces(elems_recv[i], faces, ori);
+         face = faces[refFace];
+      }
+
+      Array<int> faceVert;
+      mesh.GetFaceVertices(face, faceVert);
+
+      for (int p=0; p<4; p++)
+      {
+         conn_loc[4*i+p] = faceVert[p];
+      }
+   }
+
+   if (0) // for debugging
+   {
+      int sz = xi_send.Size()/2;
+
+      for (int i = 0; i<sz; i++)
+      {
+         mfem::out << "("<<xi_send[i*(dim-1)]<<","<<xi_send[i*(dim-1)+1]<<"): -> ";
+         for (int j = 0; j<4; j++)
+         {
+            double * vc = mesh.GetVertex(conn_loc[4*i+j]);
+            if (j<3)
+            {
+               mfem::out << "("<<vc[0]<<","<<vc[1]<<","<<vc[2]<<"), ";
+            }
+            else
+            {
+               mfem::out << "("<<vc[0]<<","<<vc[1]<<","<<vc[2]<<") \n " << endl;
+            }
+         }
+      }
+   }
+
+   int sz = xi_send.Size()/2;
+   DenseMatrix coordsm(sz*4, dim);
+   for (int i = 0; i<sz; i++)
+   {
+      for (int j = 0; j<4; j++)
+      {
+         for (int k=0; k<dim; k++)
+         {
+            coordsm(i*4+j,k) = mesh.GetVertex(conn_loc[i*4+j])[k]+x1[dim*conn_loc[i*4+j]+k];
+         }
+      }
+   }
+
+   // pass global indices for conn_loc
+   for (int i = 0; i<conn_loc.Size(); i++)
+   {
+      conn_loc[i] = gvert[conn_loc[i]];
+   }
+
+   mycomm.UpdateDestinationProcs();
+   mycomm.Communicate(xi_send,xi,2,mfem::Ordering::byVDIM);
+   mycomm.Communicate(conn_loc,conn,4,mfem::Ordering::byVDIM);
+   mycomm.Communicate(coordsm,coords,4,mfem::Ordering::byVDIM);
+}
+
+int get_rank(int tdof, std::vector<int> & tdof_offsets)
+{
+   int size = tdof_offsets.size();
+   if (size == 1) { return 0; }
+   std::vector<int>::iterator up;
+   up=std::upper_bound(tdof_offsets.begin(), tdof_offsets.end(),tdof); //
+   return std::distance(tdof_offsets.begin(),up)-1;
+}
+
+void ComputeTdofOffsets(const ParFiniteElementSpace * pfes,
+                        std::vector<int> & tdof_offsets)
+{
+   MPI_Comm comm = pfes->GetComm();
+   int num_procs;
+   MPI_Comm_size(comm, &num_procs);
+   tdof_offsets.resize(num_procs);
+   int mytoffset = pfes->GetMyTDofOffset();
+   MPI_Allgather(&mytoffset,1,MPI_INT,&tdof_offsets[0],1,MPI_INT,comm);
+}
+
+void ComputeTdofOffsets(MPI_Comm comm, int mytoffset, std::vector<int> & tdof_offsets)
+{
+   int num_procs;
+   MPI_Comm_size(comm,&num_procs);
+   tdof_offsets.resize(num_procs);
+   MPI_Allgather(&mytoffset,1,MPI_INT,&tdof_offsets[0],1,MPI_INT,comm);
+}
+
+void ComputeTdofs(MPI_Comm comm, int mytoffs, std::vector<int> & tdofs)
+{
+   int num_procs;
+   MPI_Comm_size(comm,&num_procs);
+   tdofs.resize(num_procs);
+   MPI_Allgather(&mytoffs,1,MPI_INT,&tdofs,1,MPI_INT,comm);
+}         
+
+
+// Performs Pᵀ * A * P for BlockOperator  P (with blocks as HypreParMatrices)
+// and A a HypreParMatrix, i.e., this handles the special case 
+// where P = [P₁ P₂ ⋅⋅⋅ Pₙ] 
+// C = Pᵀ * A * P 
+void RAP(const HypreParMatrix & A, const BlockOperator & P, 
+         BlockOperator & C)
+{
+   int nblocks = P.NumColBlocks();
+
+   const HypreParMatrix * Pi = nullptr;
+   const HypreParMatrix * Pj = nullptr;
+   HypreParMatrix * PitAPj = nullptr; 
+
+   for (int i = 0; i< nblocks; i++)
+   {
+      if (P.IsZeroBlock(0,i)) continue;
+      Pi = dynamic_cast<const HypreParMatrix*>(&P.GetBlock(0,i));
+      for (int j = 0; j<nblocks; j++)
+      {
+         if (P.IsZeroBlock(0,j)) continue;
+         Pj = dynamic_cast<const HypreParMatrix*>(&P.GetBlock(0,j));
+         if (i == j) 
+         {
+            PitAPj = RAP(&A, Pj); 
+         }
+         else
+         {
+            PitAPj = RAP(Pi, &A, Pj); 
+         }
+         C.SetBlock(i,j,PitAPj);
+      }
+   }
+}
+
+void ParAdd(const BlockOperator & A, const BlockOperator & B, BlockOperator & C)
+{
+   int n = A.NumRowBlocks();
+   int m = A.NumColBlocks();
+   MFEM_VERIFY(B.NumRowBlocks() == n, "Inconsistent number of row blocks");
+   MFEM_VERIFY(B.NumColBlocks() == m, "Inconsistent number of column blocks");
+
+   const HypreParMatrix * a;
+   const HypreParMatrix * b;
+   for (int i = 0; i<n; i++)
+   {
+      for (int j = 0; j<m; j++)
+      {
+         a = nullptr;
+         b = nullptr;
+         if (!A.IsZeroBlock(i,j))
+         {
+            a = dynamic_cast<const HypreParMatrix*>(&A.GetBlock(i,j));
+         }
+         if (!B.IsZeroBlock(i,j))
+         {
+            b = dynamic_cast<const HypreParMatrix*>(&B.GetBlock(i,j));
+         }
+         if (a && b)
+         {
+            C.SetBlock(i,j,ParAdd(a,b));
+         }
+         else if (a)
+         {
+            C.SetBlock(i,j,new HypreParMatrix(*a));
+         }
+         else if (b)
+         {
+            C.SetBlock(i,j,new HypreParMatrix(*b));
+         }
+         else
+         {
+            // do nothing
+         }
+      }
+   }
+}

miniapps/contact/problems/parproblems_util.hpp

@@ -0,0 +1,28 @@
+
+#include "mfem.hpp"
+#include "problems_util.hpp"
+#include "../util/mpicomm.hpp"
+
+// Coordinates in xyz are assumed to be ordered as [X, Y, Z]
+// where X is the list of x-coordinates for all points and so on.
+// conn: connectivity of the target surface elements
+// xi: surface reference cooridnates for the cloest point, involves a linear transformation from [0,1] to [-1,1]
+void FindPointsInMesh(Mesh & mesh, const Array<int> & gvert, const Vector & xyz, const Array<int> & s_conn, Array<int>& conn,
+                      Vector & xyz2, Array<int> & s_conn2, Vector& xi, DenseMatrix & coords);
+
+// somewhat simplified version of the above
+void FindPointsInMesh(Mesh & mesh, const Array<int> & gvert, Array<int> & s_conn, const Vector &x1, Vector & xyz, Array<int>& conn,
+                      Vector& xi, DenseMatrix & coords);                   
+
+int get_rank(int tdof, std::vector<int> & tdof_offsets);
+void ComputeTdofOffsets(const ParFiniteElementSpace * pfes,
+                        std::vector<int> & tdof_offsets);
+void ComputeTdofOffsets(MPI_Comm comm, int mytoffset, std::vector<int> & tdof_offsets);
+void ComputeTdofs(MPI_Comm comm, int mytoffs, std::vector<int> & tdofs);
+
+
+// Performs Pᵀ * A * P for BlockOperator  P (with blocks as HypreParMatrices)
+// and A a HypreParMatrix, i.e., this handles the special case 
+// where P = [P₁ P₂ ⋅⋅⋅ Pₙ] 
+void RAP(const HypreParMatrix & A, const BlockOperator & P, BlockOperator & C);
+void ParAdd(const BlockOperator & A, const BlockOperator & B, BlockOperator & C);

miniapps/contact/problems/problems.cpp

@@ -0,0 +1,367 @@
+#include "problems.hpp"
+
+
+void ElasticityProblem::Init()
+{
+   int dim = mesh->Dimension();
+   fec = new H1_FECollection(order,dim);
+   fes = new FiniteElementSpace(mesh,fec,dim,Ordering::byVDIM);
+   ndofs = fes->GetTrueVSize();
+   mesh->SetNodalFESpace(fes);
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+   }
+   ess_bdr = 0; ess_bdr[1] = 1;
+   fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
+   // Solution GridFunction
+   x.SetSpace(fes);  x = 0.0;
+   // RHS
+   b.Update(fes); 
+   // Elasticity operator
+   lambda.SetSize(mesh->attributes.Max()); lambda = 57.6923076923;
+   mu.SetSize(mesh->attributes.Max()); mu = 38.4615384615;
+
+   lambda_cf.UpdateConstants(lambda);
+   mu_cf.UpdateConstants(mu);
+   a = new BilinearForm(fes);
+   a->SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_cf,mu_cf));
+}
+
+void ElasticityProblem::FormLinearSystem()
+{
+   if (!formsystem) 
+   {
+      formsystem = true;
+      b.Assemble();
+      a->Assemble();
+      a->FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   }
+}
+void ElasticityProblem::UpdateLinearSystem()
+{
+   if (formsystem)
+   {
+      b.Update();
+      a->Update();
+      formsystem = false;
+   }
+   FormLinearSystem();
+}
+
+ContactProblem::ContactProblem(ElasticityProblem * prob1_, ElasticityProblem * prob2_)
+: prob1(prob1_), prob2(prob2_)
+{
+   // 1. Set up block system
+   Mesh* mesh1 = prob1->GetMesh();
+   int dim = mesh1->Dimension();
+
+   nodes0.SetSpace(mesh1->GetNodes()->FESpace());
+   nodes0 = *mesh1->GetNodes();
+   nodes1 = mesh1->GetNodes();
+
+   Vector delta1(dim);
+   delta1 = 0.0; delta1[0] = 0.1;
+   prob1->SetDisplacementDirichletData(delta1);
+   prob1->FormLinearSystem();
+   
+   Vector delta2(dim);
+   delta2 = 0.0; 
+   prob2->SetDisplacementDirichletData(delta2);
+   prob2->FormLinearSystem();
+
+   int ndof1 = prob1->GetNumDofs();
+   int ndof2 = prob2->GetNumDofs();
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = ndof1;
+   offsets[2] = ndof2;
+   offsets.PartialSum();
+
+   BlockMatrix Kb(offsets);
+   SparseMatrix A1 = prob1->GetOperator();
+   SparseMatrix A2 = prob2->GetOperator();
+
+   Kb.SetBlock(0,0,&A1);
+   Kb.SetBlock(1,1,&A2);
+
+   K = Kb.CreateMonolithic();
+   K->Threshold(0.0);
+   K->SortColumnIndices();
+
+   B = new BlockVector(offsets);
+   B->GetBlock(0).Set(1.0, prob1->GetRHS());
+   B->GetBlock(1).Set(1.0, prob2->GetRHS());
+
+   ComputeContactVertrices();
+}
+
+void ContactProblem::ComputeContactVertrices()
+{
+   if (npoints>0) return;
+   Mesh * mesh2 = prob2->GetMesh();
+   Array<int> vert;
+   for (int b=0; b<mesh2->GetNBE(); b++)
+   {
+      if (mesh2->GetBdrAttribute(b) == 3)
+      {
+         mesh2->GetBdrElementVertices(b, vert);
+         for (auto v : vert)
+         {
+            contact_vertices.insert(v);
+         }
+      }
+   }
+   npoints = contact_vertices.size();
+}
+
+void ContactProblem::ComputeGapFunctionAndDerivatives(const Vector &displ1, 
+                                                      const Vector & displ2)
+{
+   ComputeContactVertrices();
+
+   Mesh * mesh1 = prob1->GetMesh();
+   int dim = mesh1->Dimension();
+   Mesh * mesh2 = prob2->GetMesh();
+
+   int ndof1 = prob1->GetNumDofs();
+   int ndof2 = prob2->GetNumDofs();
+   int ndofs = ndof1 + ndof2;
+
+   int nv1 = mesh1->GetNV();
+   // connectivity of the second mesh
+
+   Array<int> conn2(npoints); 
+   // mesh2->MoveNodes(displ2);
+   Vector xyz(dim * npoints);
+
+   int cnt = 0;
+   for (auto v : contact_vertices)
+   {
+      for (int d = 0; d<dim; d++)
+      {
+         xyz(cnt*dim + d) = mesh2->GetVertex(v)[d]+displ2[v*dim+d];
+      }
+      conn2[cnt] = v + nv1; 
+      cnt++;
+   }
+   
+   MFEM_VERIFY(cnt == npoints, "");
+   gapv.SetSize(npoints*dim);
+
+   // segment reference coordinates of the closest point
+   Vector xi1(npoints*(dim-1));
+   Array<int> conn1(npoints*4);
+   
+   // add(nodes0, displ1, *nodes1);
+   FindPointsInMesh(*mesh1, xyz, conn1, xi1);
+
+   DenseMatrix coordsm(npoints*4, dim);
+   for (int i=0; i<npoints; i++)
+   {
+      for (int j=0; j<4; j++)
+      {
+         for (int k=0; k<dim; k++)
+         {
+            coordsm(i*4+j,k) = mesh1->GetVertex(conn1[i*4+j])[k]+displ1[dim*conn1[i*4+j]+k];
+         }
+      }
+   }
+
+   if (M)
+   {
+      delete M;
+      for (int i = 0; i<dM.Size(); i++)
+      {
+         delete dM[i];
+      }
+      dM.SetSize(0);
+   }
+
+   int h = npoints;
+   M = new SparseMatrix(h,ndofs);
+   dM.SetSize(npoints);
+   for (int i = 0; i<npoints; i++)
+   {
+      dM[i] = new SparseMatrix(ndofs,ndofs);
+   }
+   Assemble_Contact(xyz, xi1, coordsm, conn2, conn1, gapv, *M, dM);
+}
+
+
+double ContactProblem::E(const Vector & d)
+{
+   return 0.5 * K->InnerProduct(d, d) - InnerProduct(d, *B);
+}
+
+void ContactProblem::DdE(const Vector &d, Vector &gradE)
+{
+   gradE.SetSize(K->Height());
+   K->Mult(d, gradE);
+   gradE.Add(-1.0, *B); 
+}
+
+SparseMatrix* ContactProblem::DddE(const Vector &d)
+{
+  return K; 
+}
+
+void ContactProblem::g(const Vector &d, Vector &gd)
+{
+   int ndof1 = prob1->GetNumDofs();
+   int ndof2 = prob2->GetNumDofs();
+   double * data = d.GetData();
+   Vector displ1(data,ndof1);
+   Vector displ2(&data[ndof1],ndof2);
+   if (recompute)
+   {
+      ComputeGapFunctionAndDerivatives(displ1, displ2);
+      recompute = false;
+   }
+
+   gd = GetGapFunction();
+}
+
+SparseMatrix* ContactProblem::Ddg(const Vector &d)
+{
+  return GetJacobian();
+}
+
+SparseMatrix* ContactProblem::lDddg(const Vector &d, const Vector &l)
+{
+   return nullptr; // for now
+}
+
+QPContactProblem::QPContactProblem(ElasticityProblem * prob1_, ElasticityProblem * prob2_)
+: ContactProblem(prob1_,prob2_) 
+{ 
+   ContactProblem::ComputeContactVertrices();
+   dimS = npoints;
+   dimD = K->Height();
+}
+
+// E(d) = 1 / 2 d^T K d + f^T d
+double QPContactProblem::E(const Vector &d)
+{
+   return ContactProblem::E(d);
+}
+
+// gradient(E) = K d + f
+void QPContactProblem::DdE(const Vector &d, Vector &gradE)
+{
+   ContactProblem::DdE(d,gradE);
+}
+
+// Hessian(E) = K
+SparseMatrix* QPContactProblem::DddE(const Vector &d)
+{
+  return ContactProblem::DddE(d);
+}
+
+// g(d) = J * d + g0 >= 0
+void QPContactProblem::g(const Vector &d, Vector &gd)
+{
+  Vector g0;
+  ContactProblem::g(d,g0);
+  M->Mult(d, gd);
+  gd.Add(1.0, g0);
+}
+
+// Jacobian(g) = J
+SparseMatrix* QPContactProblem::Ddg(const Vector &d)
+{
+  return M;
+}
+
+SparseMatrix* QPContactProblem::lDddg(const Vector &d, const Vector &l)
+{
+   return ContactProblem::lDddg(d,l);
+}
+
+
+QPOptContactProblem::QPOptContactProblem(ContactProblem * problem_)
+: problem(problem_)
+{
+   dimU = problem->GetNumDofs();
+   dimM = problem->GetNumConstraints();
+   dimC = problem->GetNumConstraints();
+   block_offsets.SetSize(3);
+   block_offsets[0] = 0;
+   block_offsets[1] = dimU;
+   block_offsets[2] = dimM;
+   block_offsets.PartialSum();
+   ml.SetSize(dimM); ml = 0.0;
+   Vector negone(dimM); negone = -1.0;
+   NegId = new SparseMatrix(negone);
+}
+
+int QPOptContactProblem::GetDimU() { return dimU; }
+
+int QPOptContactProblem::GetDimM() { return dimM; }
+
+int QPOptContactProblem::GetDimC() { return dimC; }
+
+Vector & QPOptContactProblem::Getml() { return ml; }
+
+SparseMatrix * QPOptContactProblem::Duuf(const BlockVector & x)
+{
+   return problem->DddE(x.GetBlock(0));
+}
+
+SparseMatrix * QPOptContactProblem::Dumf(const BlockVector & x)
+{
+   return nullptr;
+}
+
+SparseMatrix * QPOptContactProblem::Dmuf(const BlockVector & x)
+{
+   return nullptr;
+}
+
+SparseMatrix * QPOptContactProblem::Dmmf(const BlockVector & x)
+{
+   return nullptr;
+}
+
+SparseMatrix * QPOptContactProblem::Duc(const BlockVector & x)
+{
+   return problem->Ddg(x.GetBlock(0));
+}
+
+SparseMatrix * QPOptContactProblem::Dmc(const BlockVector & x)
+{
+   return NegId;
+}
+
+SparseMatrix * QPOptContactProblem::lDuuc(const BlockVector & x, const Vector & l)
+{
+   return nullptr;
+}
+
+void QPOptContactProblem::c(const BlockVector &x, Vector & y)
+{
+   Vector g0;
+   problem->g(x.GetBlock(0),g0); // gap function
+   g0.Add(-1.0, x.GetBlock(1));  
+
+   problem->GetJacobian()->Mult(x.GetBlock(0),y);
+   y.Add(1.0, g0);
+}
+
+double QPOptContactProblem::CalcObjective(const BlockVector & x)
+{
+   return problem->E(x.GetBlock(0));
+}
+
+void QPOptContactProblem::CalcObjectiveGrad(const BlockVector & x, BlockVector & y)
+{
+   problem->DdE(x.GetBlock(0), y.GetBlock(0));
+   y.GetBlock(1) = 0.0;
+}
+
+QPOptContactProblem::~QPOptContactProblem()
+{
+   delete NegId;
+}

miniapps/contact/problems/problems.hpp

@@ -0,0 +1,169 @@
+#include "problems_util.hpp"
+
+
+class ElasticityProblem
+{
+private:
+   bool formsystem = false;
+   Mesh * mesh = nullptr;
+   int order;
+   int ndofs;
+   FiniteElementCollection * fec = nullptr;
+   FiniteElementSpace * fes = nullptr;
+   Vector lambda, mu;
+   PWConstCoefficient lambda_cf, mu_cf;
+   Array<int> ess_bdr, ess_tdof_list;
+   BilinearForm *a=nullptr;
+   LinearForm b;
+   GridFunction x;
+   SparseMatrix A;
+   Vector B,X;
+   void Init();
+public:
+   ElasticityProblem(const char *mesh_file , int ref, int order_ = 1) : order(order_) 
+   {
+      mesh = new Mesh(mesh_file,1,1);
+      for (int i = 0; i<ref; i++)
+      {
+         mesh->UniformRefinement();
+      }
+      Init();
+   }
+
+   Mesh * GetMesh() { return mesh; }
+   FiniteElementSpace * GetFESpace() { return fes; }
+   int GetNumDofs() { return ndofs; }
+   SparseMatrix & GetOperator() 
+   {
+      MFEM_VERIFY(formsystem, "System not formed yet. Call FormLinearSystem()"); 
+      return A; 
+   }
+
+   Vector & GetRHS() 
+   {
+      MFEM_VERIFY(formsystem, "System not formed yet. Call FormLinearSystem()"); 
+      return B; 
+   }
+
+   void FormLinearSystem();
+   void UpdateLinearSystem();
+
+   void SetDisplacementDirichletData(const Vector & delta) 
+   {
+      VectorConstantCoefficient delta_cf(delta);
+      x.ProjectBdrCoefficient(delta_cf,ess_bdr);
+   };
+
+   void UpdateDisplacement(const Vector & x_) 
+   {
+      // x = x_;
+      // mesh->MoveVertices(x);
+      // mesh->NodesUpdated();
+   };
+
+   GridFunction & GetDisplacementGridFunction() {return x;};
+   Array<int> & GetEssentialDofs() {return ess_tdof_list;};
+
+   ~ElasticityProblem()
+   {
+      delete a;
+      delete fes;
+      delete fec;
+      delete mesh;
+   }
+};
+
+
+class ContactProblem
+{
+private:
+   ElasticityProblem * prob1 = nullptr;
+   ElasticityProblem * prob2 = nullptr;
+   GridFunction nodes0;
+   GridFunction *nodes1 = nullptr;
+   std::set<int> contact_vertices;
+   bool recompute = true;
+
+protected:
+   int npoints=0;
+   SparseMatrix *K =nullptr;
+   BlockVector *B = nullptr;
+   Vector gapv;
+   Array<SparseMatrix*> dM;
+   SparseMatrix * M=nullptr;
+   void ComputeContactVertrices();
+public:
+   ContactProblem(ElasticityProblem * prob1_, ElasticityProblem * prob2_);
+
+   ElasticityProblem * GetElasticityProblem1() {return prob1;}
+   ElasticityProblem * GetElasticityProblem2() {return prob2;}
+
+   int GetNumDofs() {return K->Height();}
+   int GetNumConstraints() {return npoints;}
+   Vector & GetGapFunction() {return gapv;}
+   SparseMatrix * GetJacobian() {return M;}
+   Array<SparseMatrix*> & GetHessian() {return dM;}
+   void ComputeGapFunctionAndDerivatives(const Vector & displ1, const Vector &displ2);
+
+   virtual double E(const Vector & d);
+   virtual void DdE(const Vector &d, Vector &gradE);
+   virtual SparseMatrix* DddE(const Vector &d);
+   void g(const Vector &d, Vector &gd);
+   virtual SparseMatrix* Ddg(const Vector &d);
+   virtual SparseMatrix* lDddg(const Vector &d, const Vector &l);
+
+   ~ContactProblem()
+   {
+      delete B;
+      delete K;
+      delete M;
+      for (int i = 0; i<dM.Size(); i++)
+      {
+         delete dM[i];
+      }
+   }
+};
+
+
+class QPContactProblem : public ContactProblem
+{
+private:
+   int dimD, dimS;
+public:
+   QPContactProblem(ElasticityProblem * prob1_, ElasticityProblem * prob2_);
+
+   double E(const Vector & d);
+   void DdE(const Vector &d, Vector &gradE);
+   SparseMatrix* DddE(const Vector &d);
+   void g(const Vector &d, Vector &gd);
+   SparseMatrix* Ddg(const Vector &d);
+   SparseMatrix* lDddg(const Vector &d, const Vector &l);
+};
+
+
+class QPOptContactProblem
+{
+private:
+   ContactProblem * problem = nullptr;
+   int dimU, dimM, dimC;
+   Array<int> block_offsets;
+   Vector ml;
+   SparseMatrix * NegId = nullptr;
+public:
+   QPOptContactProblem(ContactProblem * problem_);
+   int GetDimU();
+   int GetDimM();
+   int GetDimC();
+   Vector & Getml();
+   SparseMatrix * Duuf(const BlockVector &);
+   SparseMatrix * Dumf(const BlockVector &);
+   SparseMatrix * Dmuf(const BlockVector &);
+   SparseMatrix * Dmmf(const BlockVector &);
+   SparseMatrix * Duc(const BlockVector &);
+   SparseMatrix * Dmc(const BlockVector &);
+   SparseMatrix * lDuuc(const BlockVector &, const Vector &);
+   void c(const BlockVector &, Vector &);
+   double CalcObjective(const BlockVector &);
+   void CalcObjectiveGrad(const BlockVector &, BlockVector &);
+   ~QPOptContactProblem();
+};

miniapps/contact/problems/problems_util.hpp

@@ -0,0 +1,61 @@
+
+#include "mfem.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi); // dNdxi is 2*4
+void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
+                     DenseMatrix& dN2dxi);
+// returns the vector and matrix form of the shape functions and its derivative
+void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
+                       DenseMatrix& ddNdxi);
+void cross(const Vector a, const Vector b, Vector& c);
+// a outer b
+void outer(const Vector a, const Vector b, DenseMatrix& c);
+// dphidxi 2*4
+// coords 4*3
+void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
+                   Vector& normal, double& nnorm);
+void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi);
+
+// m_coords is expected to be 4 * 3
+void  ComputeGapJacobian(const Vector x_s, const Vector xi,
+                         const DenseMatrix m_coords,
+                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs);
+
+void ComputeGapHessian(const Vector x_s, const Vector xi,
+                       const DenseMatrix m_coords,
+                       DenseMatrix& dg2dx);
+void NodeSegConPairs(const Vector x1, const Vector xi2,
+                     const DenseMatrix coords2,
+                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2);
+// coordsm : (npoints*4, 3) use what class?
+// m_conn: (npoints*4)
+void Assemble_Contact(const Vector x_s,
+                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
+                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
+                      Array<SparseMatrix *> & dM);
+
+void Assemble_Contact(const Vector x_s,
+                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
+                      const Array<int> m_conn, Vector & g, SparseMatrix & M1, SparseMatrix & M2, 
+                      Array<SparseMatrix *> & dM11,
+                      Array<SparseMatrix *> & dM12,
+                      Array<SparseMatrix *> & dM21,
+                      Array<SparseMatrix *> & dM22);
+void Assemble_Contact(const Vector x_s,
+                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
+                      const Array<int> m_conn, Vector & g, SparseMatrix & M1, SparseMatrix & M2,const Array<int> & points_map); 
+
+void FindSurfaceToProject(Mesh& mesh, const int elem, int& cbdrface);
+
+Vector GetNormalVector(Mesh & mesh, const int elem, const double *ref,
+                       int & refFace, int & refNormal, bool & interior);
+int GetHexVertex(int cdim, int c, int fa, int fb, Vector & refCrd);
+
+// Coordinates in xyz are assumed to be ordered as [X, Y, Z]
+// where X is the list of x-coordinates for all points and so on.
+// conn: connectivity of the target surface elements
+// xi: surface reference cooridnates for the cloest point, involves a linear transformation from [0,1] to [-1,1]
+void FindPointsInMesh(Mesh & mesh, Vector const& xyz, Array<int>& conn, Vector& xi);

miniapps/contact/util/mpicomm.cpp

@@ -0,0 +1,530 @@
+#include "mpicomm.hpp"
+#include "util.hpp"
+
+
+MPICommunicator::MPICommunicator(MPI_Comm comm_, int offset_, int gsize)
+: comm(comm_), offset(offset_)
+{
+   MPI_Comm_size(comm,&num_procs);
+   MPI_Comm_rank(comm,&myid);
+   offsets.resize(num_procs);
+   MPI_Allgather(&offset,1,MPI_INT,&offsets[0],1,MPI_INT,comm);
+   lsize = (myid == num_procs-1) ? gsize - offsets[myid]
+                                : offsets[myid+1]-offsets[myid];
+
+   send_count.SetSize(num_procs); send_count = 0;
+   send_displ.SetSize(num_procs); send_displ = 0;
+   recv_count.SetSize(num_procs); recv_count = 0;
+   recv_displ.SetSize(num_procs); recv_displ = 0;
+} 
+
+MPICommunicator::MPICommunicator(MPI_Comm comm_, Array<unsigned int> & destination_procs_)
+: comm(comm_), destination_procs(destination_procs_)
+{
+   MPI_Comm_size(comm,&num_procs);
+   MPI_Comm_rank(comm,&myid);   
+   send_count.SetSize(num_procs); 
+   send_displ.SetSize(num_procs); 
+   recv_count.SetSize(num_procs); 
+   recv_displ.SetSize(num_procs); 
+   resetcounts();
+}
+
+
+int MPICommunicator::get_rank(int dof)
+{
+   if (num_procs == 1) { return 0; }
+   std::vector<int>::iterator up;
+   up=std::upper_bound(offsets.begin(), offsets.end(),dof);
+   return std::distance(offsets.begin(),up)-1;
+}
+
+
+void MPICommunicator::Communicate(const Vector & x_s, Vector & x_r, int vdim, int ordering)
+{
+   int npts = x_s.Size()/vdim;
+   MFEM_VERIFY(npts == destination_procs.Size(), "Inconsistent number of points to be send");
+
+   // construct send count
+   for (int i = 0; i<npts; i++)
+   {
+      int rank = destination_procs[i];
+      send_count[rank] += vdim + 1; // including the sending processor id
+   }
+
+   // 2. Compute recv_count
+   MPI_Alltoall(&send_count[0],1,MPI_INT,&recv_count[0],1,MPI_INT,comm);
+
+   // 3. Compute displacements
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   int sbuff_size = send_count.Sum();
+   int rbuff_size = recv_count.Sum();
+
+   // 4. Allocate memory and fill in send buffers
+   Array<double> sendvals(sbuff_size);  sendvals = 0.0;
+   Array<int> sendoffs(num_procs); sendoffs = 0;
+   for (int i = 0; i<npts; i++)
+   {
+      int rank = destination_procs[i];
+      int j = send_displ[rank] + sendoffs[rank];
+      sendoffs[rank] += vdim+1;
+      sendvals[j] = (double)myid;
+      for (int k = 0; k<vdim; k++)
+      {
+         int kk = (ordering == mfem::Ordering::byNODES) ? k*npts+i : i*vdim + k;
+         sendvals[j+k+1] = x_s(kk);
+      }
+   }
+
+   // 5. Communication
+   Array<double> recvvals(rbuff_size);
+
+   double * sendvals_ptr = nullptr;
+   double * recvvals_ptr = nullptr;
+   if (sbuff_size !=0 ) { sendvals_ptr = &sendvals[0]; }   
+   if (rbuff_size !=0 ) { recvvals_ptr = &recvvals[0]; }
+
+   MPI_Alltoallv(sendvals_ptr, send_count, send_displ, MPI_DOUBLE, recvvals_ptr,
+                 recv_count, recv_displ, MPI_DOUBLE, comm);
+
+   // 6. Unpack
+   int n = rbuff_size/(vdim+1);
+   origin_procs.SetSize(n);
+   x_r.SetSize(vdim*n);
+   for (int i = 0; i<n; i++)
+   {
+      origin_procs[i] = (unsigned int)recvvals[(vdim+1)*i];
+      for (int j=0; j<vdim; j++)
+      {
+         int kk = (ordering == mfem::Ordering::byNODES) ? j*n+i : i*vdim + j;
+         x_r(kk) = recvvals[(vdim+1)*i + j+1];
+      }
+   }
+   resetcounts();
+}
+
+void MPICommunicator::Communicate(const Array<unsigned int> & x_s, Array<unsigned int> & x_r, int vdim, int ordering)
+{
+   int npts = x_s.Size()/vdim;
+   MFEM_VERIFY(npts == destination_procs.Size(), "Inconsistent number of points to be send");
+
+   // construct send count
+   for (int i = 0; i<npts; i++)
+   {
+      int rank = destination_procs[i];
+      send_count[rank] += vdim + 1; // including the sending processor id
+   }
+
+   // 2. Compute recv_count
+   MPI_Alltoall(&send_count[0],1,MPI_INT,&recv_count[0],1,MPI_INT,comm);
+
+   // 3. Compute displacements
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   int sbuff_size = send_count.Sum();
+   int rbuff_size = recv_count.Sum();
+
+   // 4. Allocate memory and fill in send buffers
+   Array<unsigned int> sendvals(sbuff_size);  sendvals = 0.0;
+   Array<int> sendoffs(num_procs); sendoffs = 0;
+   for (int i = 0; i<npts; i++)
+   {
+      int rank = destination_procs[i];
+      int j = send_displ[rank] + sendoffs[rank];
+      sendoffs[rank] += vdim+1;
+      sendvals[j] = myid;
+      for (int k = 0; k<vdim; k++)
+      {
+         int kk = (ordering == mfem::Ordering::byNODES) ? k*npts+i : i*vdim + k;
+         sendvals[j+k+1] = x_s[kk];
+      }
+   }
+
+   // 5. Communication
+   Array<unsigned int> recvvals(rbuff_size);
+
+   unsigned int * sendvals_ptr = nullptr;
+   unsigned int * recvvals_ptr = nullptr;
+   if (sbuff_size !=0 ) { sendvals_ptr = &sendvals[0]; }   
+   if (rbuff_size !=0 ) { recvvals_ptr = &recvvals[0]; }
+
+   MPI_Alltoallv(sendvals_ptr, send_count, send_displ, MPI_UNSIGNED, recvvals_ptr,
+                 recv_count, recv_displ, MPI_UNSIGNED, comm);
+
+   // 6. Unpack
+   int n = rbuff_size/(vdim+1);
+   origin_procs.SetSize(n);
+   x_r.SetSize(vdim*n);
+   for (int i = 0; i<n; i++)
+   {
+      origin_procs[i] = recvvals[(vdim+1)*i];
+      for (int j=0; j<vdim; j++)
+      {
+         int kk = (ordering == mfem::Ordering::byNODES) ? j*n+i : i*vdim + j;
+         x_r[kk] = recvvals[(vdim+1)*i + j+1];
+      }
+   }
+   resetcounts();
+}
+
+void MPICommunicator::Communicate(const Array<int> & x_s, Array<int> & x_r, int vdim, int ordering)
+{
+   int npts = x_s.Size()/vdim;
+   MFEM_VERIFY(npts == destination_procs.Size(), "Inconsistent number of points to be send");
+
+   // construct send count
+   for (int i = 0; i<npts; i++)
+   {
+      int rank = destination_procs[i];
+      send_count[rank] += vdim + 1; // including the sending processor id
+   }
+
+   // 2. Compute recv_count
+   MPI_Alltoall(&send_count[0],1,MPI_INT,&recv_count[0],1,MPI_INT,comm);
+
+   // 3. Compute displacements
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   int sbuff_size = send_count.Sum();
+   int rbuff_size = recv_count.Sum();
+
+   // 4. Allocate memory and fill in send buffers
+   Array<int> sendvals(sbuff_size);  sendvals = 0.0;
+   Array<int> sendoffs(num_procs); sendoffs = 0;
+   for (int i = 0; i<npts; i++)
+   {
+      int rank = destination_procs[i];
+      int j = send_displ[rank] + sendoffs[rank];
+      sendoffs[rank] += vdim+1;
+      sendvals[j] = myid;
+      for (int k = 0; k<vdim; k++)
+      {
+         int kk = (ordering == mfem::Ordering::byNODES) ? k*npts+i : i*vdim + k;
+         sendvals[j+k+1] = x_s[kk];
+      }
+   }
+
+   // 5. Communication
+   Array<int> recvvals(rbuff_size);
+
+   int * sendvals_ptr = nullptr;
+   int * recvvals_ptr = nullptr;
+   if (sbuff_size !=0 ) { sendvals_ptr = &sendvals[0]; }   
+   if (rbuff_size !=0 ) { recvvals_ptr = &recvvals[0]; }
+
+   MPI_Alltoallv(sendvals_ptr, send_count, send_displ, MPI_INT, recvvals_ptr,
+                 recv_count, recv_displ, MPI_INT, comm);
+
+   // 6. Unpack
+   int n = rbuff_size/(vdim+1);
+   origin_procs.SetSize(n);
+   x_r.SetSize(vdim*n);
+   for (int i = 0; i<n; i++)
+   {
+      origin_procs[i] = (unsigned int)recvvals[(vdim+1)*i];
+      for (int j=0; j<vdim; j++)
+      {
+         int kk = (ordering == mfem::Ordering::byNODES) ? j*n+i : i*vdim + j;
+         x_r[kk] = recvvals[(vdim+1)*i + j+1];
+      }
+   }
+   resetcounts();
+}
+
+void MPICommunicator::Communicate(const DenseMatrix & A_s, DenseMatrix & A_r, int vdim, int ordering)
+{
+   // matrix width corresponds to dim coordinates
+   // matrix rows might include vdim copies
+   int npts = A_s.Height()/vdim;
+   int dim = A_s.Width();
+   MFEM_VERIFY(npts == destination_procs.Size(), "Inconsistent number of points to be send");
+
+   // construct send count
+   for (int i = 0; i<npts; i++)
+   {
+      int rank = destination_procs[i];
+      send_count[rank] += dim*vdim + 1; // including the sending processor id
+   }
+
+   // 2. Compute recv_count
+   MPI_Alltoall(&send_count[0],1,MPI_INT,&recv_count[0],1,MPI_INT,comm);
+
+   // 3. Compute displacements
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   int sbuff_size = send_count.Sum();
+   int rbuff_size = recv_count.Sum();
+
+   // 4. Allocate memory and fill in send buffers
+   Array<double> sendvals(sbuff_size);  sendvals = 0.0;
+   Array<int> sendoffs(num_procs); sendoffs = 0;
+   for (int i = 0; i<npts; i++)
+   {
+      int rank = destination_procs[i];
+      int j = send_displ[rank] + sendoffs[rank];
+      sendoffs[rank] += dim*vdim+1;
+      sendvals[j] = myid;
+      for (int k = 0; k<vdim; k++)
+      {
+         int kk = (ordering == mfem::Ordering::byNODES) ? k*npts+i : i*vdim + k;
+         for (int d=0; d<dim; d++)
+         {
+            sendvals[j+k*dim+d+1] = A_s(kk,d);
+         }
+      }
+   }
+   // 5. Communication
+   Array<double> recvvals(rbuff_size);
+
+   double * sendvals_ptr = nullptr;
+   double * recvvals_ptr = nullptr;
+   if (sbuff_size !=0 ) { sendvals_ptr = &sendvals[0]; }   
+   if (rbuff_size !=0 ) { recvvals_ptr = &recvvals[0]; }
+
+   MPI_Alltoallv(sendvals_ptr, send_count, send_displ, MPI_DOUBLE, recvvals_ptr,
+                 recv_count, recv_displ, MPI_DOUBLE, comm);
+
+   // 6. Unpack
+   int n = rbuff_size/(dim*vdim+1);
+   origin_procs.SetSize(n);
+   A_r.SetSize(vdim*n,dim);
+
+   for (int i = 0; i<n; i++)
+   {
+      origin_procs[i] = (unsigned int)recvvals[(dim*vdim+1)*i];
+      for (int j=0; j<vdim; j++)
+      {
+         int kk = (ordering == mfem::Ordering::byNODES) ? j*n+i : i*vdim + j;
+         for (int d=0; d<dim; d++)
+         {
+            A_r(kk,d) = recvvals[(dim*vdim+1)*i + j*dim + d+1];
+         }
+      }
+   }
+   resetcounts();
+
+}
+
+
+void MPICommunicator::Communicate(const SparseMatrix & mat_s , SparseMatrix & mat_r)
+{
+   // 1. Compute send_count 
+   int n = mat_s.NumRows();
+   for (int i = 0; i<n; i++)
+   {
+      int rsize = mat_s.RowSize(i);
+      if (rsize == 0) continue;
+      int rank = get_rank(i);
+      send_count[rank] += rsize+2;
+   }
+   // 2. Compute recv_count
+   MPI_Alltoall(&send_count[0],1,MPI_INT,&recv_count[0],1,MPI_INT,comm);
+
+   // 3. Compute displacements
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   int sbuff_size = send_count.Sum();
+   int rbuff_size = recv_count.Sum();
+
+   // 4. Allocate memory and fill in send buffers
+   Array<double> sendvals(sbuff_size);  sendvals = 0.0;
+   Array<int> sendcols(sbuff_size);  sendcols = 0;
+   Array<int> sendoffs(num_procs); sendoffs = 0;
+   Array<int> cols;
+   Vector vals;
+   for (int i = 0; i<n; i++)
+   {
+      int rsize = mat_s.RowSize(i);
+      if (rsize == 0) continue;
+      int rank = get_rank(i);
+      int j = send_displ[rank] + sendoffs[rank];
+      mat_s.GetRow(i,cols,vals);
+      sendoffs[rank] += rsize+2;
+      sendvals[j] = (double)i;
+      sendvals[j+1] = (double)rsize;
+      sendcols[j] = i;
+      sendcols[j+1] = rsize;
+      for (int l=0; l<rsize ; l++)
+      {
+         sendvals[j+l+2] = vals[l];
+         sendcols[j+l+2] = cols[l];
+      }
+   }
+
+   // 5. Communication
+   Array<double> recvvals(rbuff_size);
+   Array<int> recvcols(rbuff_size);
+
+   double * sendvals_ptr = nullptr;
+   double * recvvals_ptr = nullptr;
+   int * sendcols_ptr = nullptr;
+   int * recvcols_ptr = nullptr;
+   if (sbuff_size !=0 ) 
+   {
+      sendvals_ptr = &sendvals[0]; 
+      sendcols_ptr = &sendcols[0]; 
+   }   
+   if (rbuff_size !=0 ) 
+   {
+      recvvals_ptr = &recvvals[0]; 
+      recvcols_ptr = &recvcols[0]; 
+   }
+
+   MPI_Alltoallv(sendvals_ptr, send_count, send_displ, MPI_DOUBLE, recvvals_ptr,
+                 recv_count, recv_displ, MPI_DOUBLE, comm);
+
+   MPI_Alltoallv(sendcols_ptr, send_count, send_displ, MPI_INT, recvcols_ptr,
+                 recv_count, recv_displ, MPI_INT, comm);
+
+   // 6. Unpack and store to the output SparseMatrix
+   MFEM_VERIFY(mat_r.Height() == lsize, "Inconsistent row size of output SparseMatrix");
+   MFEM_VERIFY(mat_r.Width() == mat_s.Width(), "Inconsistent column size of output SparseMatrix");
+
+   int counter = 0;
+   while (counter < rbuff_size)
+   {
+      int row = recvcols[counter] - offset;
+      int size = recvcols[counter+1];
+      vals.SetSize(size);
+      cols.SetSize(size);
+      for (int i = 0; i<size; i++)
+      {
+         vals[i] = recvvals[counter+2 + i];
+         cols[i] = recvcols[counter+2 + i];
+      }
+      mat_r.AddRow(row,cols,vals);
+      counter += size+2; 
+   }
+   MFEM_VERIFY(counter == rbuff_size, "inconsistent rbuff size");
+   mat_r.Finalize();
+   mat_r.SortColumnIndices();
+   resetcounts();
+}
+
+void MPICommunicator::Communicate(const Array<SparseMatrix*> & vmat_s, Array<SparseMatrix*> & vmat_r)
+{
+   // 1. Compute send_count 
+   for (int k = 0; k<vmat_s.Size(); k++)
+   {
+      if (!vmat_s[k]) continue;
+      if (vmat_s[k]->NumNonZeroElems() == 0) continue;
+      int nrows = vmat_s[k]->NumRows();
+      for (int i = 0; i<nrows; i++)
+      {
+         int rsize = vmat_s[k]->RowSize(i);
+         if (rsize == 0) continue;
+         int rank = get_rank(i);
+         send_count[rank] += rsize+3;
+      }
+   } 
+
+   // 2. Compute recv_count
+   MPI_Alltoall(&send_count[0],1,MPI_INT,&recv_count[0],1,MPI_INT,comm);
+   
+   // 3. Compute displacements
+   for (int k=0; k<num_procs-1; k++)
+   {
+      send_displ[k+1] = send_displ[k] + send_count[k];
+      recv_displ[k+1] = recv_displ[k] + recv_count[k];
+   }
+   int sbuff_size = send_count.Sum();
+   int rbuff_size = recv_count.Sum();
+
+   // 4. Allocate memory and fill in send buffers
+   Array<double> sendvals(sbuff_size);  sendvals = 0.0;
+   Array<int> sendcols(sbuff_size);  sendcols = 0;
+   Array<int> sendoffs(num_procs); sendoffs = 0;
+   for (int k = 0; k<vmat_s.Size(); k++)
+   {
+      if (!vmat_s[k]) continue;
+      if (vmat_s[k]->NumNonZeroElems() == 0) continue;
+      int nrows = vmat_s[k]->NumRows();
+      for (int i = 0; i<nrows; i++)
+      {
+         int rsize = vmat_s[k]->RowSize(i);
+         if (rsize == 0) continue;
+         int rank = get_rank(i);
+         int j = send_displ[rank] + sendoffs[rank];
+         Array<int> cols;
+         Vector vals;
+         vmat_s[k]->GetRow(i,cols,vals);
+         sendoffs[rank] += rsize+3;
+         sendvals[j] = (double)k;
+         sendvals[j+1] = (double)i;
+         sendvals[j+2] = (double)rsize;
+         sendcols[j] = k;
+         sendcols[j+1] = i;
+         sendcols[j+2] = rsize;
+         for (int l=0; l<rsize ; l++)
+         {
+            sendvals[j+l+3] = vals[l];
+            sendcols[j+l+3] = cols[l];
+         }
+      }
+   }
+   // 5. Communication
+   Array<double> recvvals(rbuff_size);
+   Array<int> recvcols(rbuff_size);
+   double * sendvals_ptr = nullptr;
+   double * recvvals_ptr = nullptr;
+   int * sendcols_ptr = nullptr;
+   int * recvcols_ptr = nullptr;
+   if (sbuff_size !=0 ) 
+   {
+      sendvals_ptr = &sendvals[0]; 
+      sendcols_ptr = &sendcols[0]; 
+   }   
+   if (rbuff_size !=0 ) 
+   {
+      recvvals_ptr = &recvvals[0]; 
+      recvcols_ptr = &recvcols[0]; 
+   }
+
+   MPI_Alltoallv(sendvals_ptr, send_count, send_displ, MPI_DOUBLE, recvvals_ptr,
+                 recv_count, recv_displ, MPI_DOUBLE,comm);
+
+   MPI_Alltoallv(sendcols_ptr, send_count, send_displ, MPI_INT, recvcols_ptr,
+                 recv_count, recv_displ, MPI_INT,comm);
+
+   // 6. Unpack and store to the output SparseMatrix
+   int counter = 0;
+   while (counter < rbuff_size)
+   {
+      int npt = recvcols[counter];
+      int row = recvcols[counter+1] - offset;
+      int size = recvcols[counter+2];
+      Vector vals(size);
+      Array<int> cols(size);
+      for (int i = 0; i<size; i++)
+      {
+         vals[i] = recvvals[counter+3 + i];
+         cols[i] = recvcols[counter+3 + i];
+      }
+      vmat_r[npt]->AddRow(row,cols,vals);
+      counter += size+3; 
+   }
+   MFEM_VERIFY(counter == rbuff_size, "inconsistent size");
+
+   for (int i = 0; i<vmat_r.Size(); i++)
+   {
+      vmat_r[i]->Finalize();
+      vmat_r[i]->SortColumnIndices();
+   }
+   resetcounts();
+}

miniapps/contact/util/mpicomm.hpp

@@ -0,0 +1,47 @@
+#include "mfem.hpp"
+
+using namespace std;
+using namespace mfem;
+
+
+class MPICommunicator
+{
+private:
+    MPI_Comm comm;
+    int myid, num_procs;
+    Array<unsigned int > origin_procs;
+    Array<unsigned int > destination_procs;
+    int offset, lsize;
+    std::vector<int> offsets; 
+    Array<int> send_count;
+    Array<int> send_displ;
+    Array<int> recv_count;
+    Array<int> recv_displ;
+    void resetcounts()
+    {
+        send_count = 0;
+        send_displ = 0;
+        recv_count = 0;
+        recv_displ = 0;
+    }
+
+public:
+    MPICommunicator(MPI_Comm comm_, int offset_, int gsize);
+    MPICommunicator(MPI_Comm comm_, Array<unsigned int> & destination_procs_);
+
+    int get_rank(int dof);
+
+    Array<unsigned int> & GetOriginProcs() {return origin_procs;}
+    void UpdateDestinationProcs() 
+    {   
+        destination_procs.SetSize(origin_procs.Size());
+        destination_procs = origin_procs;
+        resetcounts();
+    }
+    void Communicate(const Vector & x_s, Vector & x_r, int vdim, int ordering);
+    void Communicate(const Array<int> & x_s, Array<int> & x_r, int vdim, int ordering);
+    void Communicate(const DenseMatrix & A_s, DenseMatrix & A_r, int vdim, int ordering);
+    void Communicate(const Array<unsigned int> & x_s, Array<unsigned int> & x_r, int vdim, int ordering);
+    void Communicate(const SparseMatrix & mat_s , SparseMatrix & mat_r);
+    void Communicate(const Array<SparseMatrix*> & vmat_s, Array<SparseMatrix*> & vmat_r);
+};

miniapps/contact/util/util.cpp

@@ -0,0 +1,171 @@
+#include "util.hpp"
+
+
+void PrintVertex(Mesh * mesh, int vertex)
+{
+   Array<int> vertices;
+   mfem::out << "vertex: " << vertex << ":  ";
+   double *coords = mesh->GetVertex(vertex);
+   mfem::out << "(" << coords[0] << ", " << coords[1] << ", " << coords[2] << ") \n";
+}
+
+void PrintElementVertices(Mesh * mesh, int elem)
+{
+   Array<int> vertices;
+   mfem::out <<  "elem: " << elem << ". Vertices = \n" ;
+   mesh->GetElementVertices(elem,vertices);
+   for (int i = 0; i<vertices.Size(); i++)
+   {
+      PrintVertex(mesh,vertices[i]);
+   }
+   mfem::out << endl;
+}
+
+void PrintFaceVertices(Mesh * mesh, int face)
+{
+   Array<int> vertices;
+   mfem::out << "face: " << face << ". Vertices = \n" ;
+   mesh->GetFaceVertices(face,vertices);
+   for (int i = 0; i<vertices.Size(); i++)
+   {
+      PrintVertex(mesh,vertices[i]);
+   }
+   mfem::out << endl;
+}
+
+void PrintSet(const std::set<int> & a, const char *aname)
+{
+   mfem::out << aname << " = " ;
+   for (std::set<int>::iterator it = a.begin(); it!= a.end(); it++)
+   {
+      mfem::out << *it << "  ";
+   }
+   mfem::out << endl;
+}
+
+void PrintVector(const Vector & a, const char *aname)
+{
+   int sz = a.Size();
+   mfem::out << aname << " = " ;
+   for (int i = 0; i<sz; i++)
+   {
+      mfem::out << a[i] << "  ";
+   }
+   mfem::out << endl;
+}
+
+void PrintVertex(Mesh * mesh, int vertex,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   if (myid == printid)
+   {
+      mfem::out << "myid = " << myid <<":   " <<  "vertex: " << vertex << ":  ";
+      double *coords = mesh->GetVertex(vertex);
+      mfem::out << "(" << coords[0] << ", " << coords[1] << ", " << coords[2] << ")\n";
+   }
+}
+
+void PrintElementVertices(Mesh * mesh, int elem,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   Array<int> vertices;
+   if (myid == printid)
+   {
+      mfem::out << "myid = " << myid <<":   " <<  "elem: " << elem <<
+                ". Vertices = \n" ;
+      mesh->GetElementVertices(elem,vertices);
+      for (int i = 0; i<vertices.Size(); i++)
+      {
+         PrintVertex(mesh,vertices[i],printid);
+      }
+      mfem::out << endl;
+   }
+}
+
+void PrintFaceVertices(Mesh * mesh, int face,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   Array<int> vertices;
+   if (myid == printid)
+   {
+      mfem::out << "myid = " << myid <<":   " <<  "face: " << face <<
+                ". Vertices = \n" ;
+      mesh->GetFaceVertices(face,vertices);
+      for (int i = 0; i<vertices.Size(); i++)
+      {
+         PrintVertex(mesh,vertices[i],printid);
+      }
+      mfem::out << endl;
+   }
+}
+
+
+void PrintSet(const std::set<int> & a, const char *aname,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   if (myid == printid)
+   {
+      mfem::out << "myid = " << myid <<":   " << aname << " = " ;
+      for (std::set<int>::iterator it = a.begin(); it!= a.end(); it++)
+      {
+         mfem::out << *it << "  ";
+      }
+      mfem::out << endl;
+   }
+}
+
+void PrintVector(const Vector & a, const char *aname,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   if (myid == printid)
+   {
+      int sz = a.Size();
+      mfem::out << "myid = " << myid <<":   " << aname << " = " ;
+      for (int i = 0; i<sz; i++)
+      {
+         mfem::out << a[i] << "  ";
+      }
+      mfem::out << endl;
+   }
+}
+
+void PrintVector(const std::vector<int> & a, const char *aname,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   if (myid == printid)
+   {
+      int sz = a.size();
+      mfem::out << "myid = " << myid <<":   " << aname << " = " ;
+      for (int i = 0; i<sz; i++)
+      {
+         mfem::out << a[i] << "  ";
+      }
+      mfem::out << endl;
+   }
+}
+
+void PrintVector(const std::vector<unsigned int> & a, const char *aname,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   if (myid == printid)
+   {
+      int sz = a.size();
+      mfem::out << "myid = " << myid <<":   " << aname << " = " ;
+      for (int i = 0; i<sz; i++)
+      {
+         mfem::out << a[i] << "  ";
+      }
+      mfem::out << endl;
+   }
+}
+
+void PrintSparseMatrix(const SparseMatrix & a, const char *aname,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   if (myid == printid)
+   {
+      mfem::out << "myid = " << myid <<":   " << aname << " = " ;
+      a.PrintMatlab(mfem::out);
+   }
+   mfem::out << endl;
+}

miniapps/contact/util/util.hpp

@@ -0,0 +1,46 @@
+#include "mfem.hpp"
+
+using namespace std;
+using namespace mfem;
+
+void PrintVertex(Mesh * mesh, int vertex);
+void PrintElementVertices(Mesh * mesh, int elem);
+void PrintFaceVertices(Mesh * mesh, int face);
+template <class T>
+void PrintArray(const Array<T> & a, const char *aname)
+{
+   int sz = a.Size();
+   mfem::out << aname << " = " ;
+   for (int i = 0; i<sz; i++)
+   {
+      mfem::out << a[i] << "  ";
+   }
+   mfem::out << endl;
+}
+void PrintSet(const std::set<int> & a, const char *aname);
+void PrintVector(const Vector & a, const char *aname);
+
+// for parallel 
+void PrintVertex(Mesh * mesh, int vertex,  int printid);
+void PrintElementVertices(Mesh * mesh, int elem,  int printid);
+void PrintFaceVertices(Mesh * mesh, int face,  int printid);
+template <class T>
+void PrintArray(const Array<T> & a, const char *aname,  int printid)
+{
+   int myid = Mpi::WorldRank();
+   if (myid == printid)
+   {
+      int sz = a.Size();
+      mfem::out << "myid = " << myid <<":   " << aname << " = " ;
+      for (int i = 0; i<sz; i++)
+      {
+         mfem::out << a[i] << "  ";
+      }
+      mfem::out << endl;
+   }
+}
+void PrintSet(const std::set<int> & a, const char *aname,  int printid);
+void PrintVector(const Vector & a, const char *aname,  int printid);
+void PrintVector(const std::vector<int> & a, const char *aname,  int printid);
+void PrintVector(const std::vector<unsigned int> & a, const char *aname,  int printid);
+void PrintSparseMatrix(const SparseMatrix & a, const char *aname,  int printid);

tests/unit/CMakeLists.txt

@@ -29,6 +29,7 @@ set(UNIT_TESTS_SRCS
   linalg/test_constrainedsolver.cpp
   linalg/test_direct_solvers.cpp
   linalg/test_hypre_ilu.cpp
+  linalg/test_hypre_prec.cpp
   linalg/test_hypre_vector.cpp
   linalg/test_ilu.cpp
   linalg/test_matrix_block.cpp
@@ -41,6 +42,7 @@ set(UNIT_TESTS_SRCS
   linalg/test_ode2.cpp
   linalg/test_operator.cpp
   linalg/test_vector.cpp
+  mesh/test_face_orientations.cpp
   mesh/test_fms.cpp
   mesh/test_mesh.cpp
   mesh/test_ncmesh.cpp

tests/unit/linalg/test_hypre_prec.cpp

@@ -0,0 +1,288 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "unit_tests.hpp"
+#include "mfem.hpp"
+
+namespace mfem
+{
+
+#ifdef MFEM_USE_MPI
+
+enum PartType {ALL, FIRST, LAST, ALL_BUT_LAST, ALL_BUT_FIRST};
+
+double sin3d(const Vector &x)
+{
+   return sin(x[0]) * sin(x[1]) * sin(x[2]);
+}
+
+void sin2d_vec(const Vector &x, Vector &v)
+{
+   v.SetSize(2);
+   v[0] = cos(x[0]) * sin(x[1]);
+   v[1] = sin(x[0]) * cos(x[1]);
+}
+
+void sin3d_vec(const Vector &x, Vector &v)
+{
+   v.SetSize(3);
+   v[0] = cos(x[0]) * sin(x[1]) * sin(x[2]);
+   v[1] = sin(x[0]) * cos(x[1]) * sin(x[2]);
+   v[2] = sin(x[0]) * sin(x[1]) * cos(x[2]);
+}
+
+void GeneratePart(PartType part_type, int nelems, int world_size,
+                  int *partitioning)
+{
+   if (world_size == 1)
+   {
+      for (int i=0; i<nelems; i++)
+      {
+         partitioning[i] = 0;
+      }
+      return;
+   }
+   switch (part_type)
+   {
+      case ALL:
+         for (int i=0; i<nelems; i++)
+         {
+            partitioning[i] = i % world_size;
+         }
+         break;
+      case FIRST:
+         for (int i=0; i<nelems; i++)
+         {
+            partitioning[i] = 0;
+         }
+         break;
+      case LAST:
+         for (int i=0; i<nelems; i++)
+         {
+            partitioning[i] = world_size - 1;
+         }
+         break;
+      case ALL_BUT_LAST:
+         for (int i=0; i<nelems; i++)
+         {
+            partitioning[i] = i % (world_size-1);
+         }
+         break;
+      case ALL_BUT_FIRST:
+         for (int i=0; i<nelems; i++)
+         {
+            partitioning[i] = i % (world_size-1) + 1;
+         }
+         break;
+   }
+}
+
+TEST_CASE("HypreBoomerAMG", "[Parallel], [HypreBoomerAMG]")
+{
+   int world_size, rank;
+   MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+   int n = 3;
+   int dim = 3;
+   int order = 2;
+
+   Mesh mesh = Mesh::MakeCartesian3D(n, n, n, Element::HEXAHEDRON);
+
+   int nelems = mesh.GetNE();
+   int *partitioning = new int[nelems];
+
+   PartType last_type = (world_size == 1) ? ALL : ALL_BUT_FIRST;
+   for (int part_type = ALL; part_type <= last_type; part_type++)
+   {
+      GeneratePart((PartType)part_type, nelems, world_size, partitioning);
+
+      ParMesh pmesh(MPI_COMM_WORLD, mesh, partitioning);
+
+      H1_FECollection fec(order, dim);
+      ParFiniteElementSpace fespace(&pmesh, &fec);
+
+      ParBilinearForm a(&fespace);
+      a.AddDomainIntegrator(new DiffusionIntegrator);
+      a.AddDomainIntegrator(new MassIntegrator);
+      a.Assemble();
+
+      ParGridFunction x(&fespace);
+      FunctionCoefficient sin3dCoef(sin3d);
+      x.ProjectCoefficient(sin3dCoef);
+      double err0 = x.ComputeL2Error(sin3dCoef);
+
+      ParLinearForm b(&fespace);
+      a.Mult(x, b);
+      x = 0.0;
+
+      OperatorPtr A;
+      Vector B, X;
+      Array<int> ess_tdof_list;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+      HypreBoomerAMG amg;
+      amg.SetPrintLevel(0);
+
+      HyprePCG pcg(MPI_COMM_WORLD);
+      pcg.SetTol(1e-10);
+      pcg.SetMaxIter(2000);
+      pcg.SetPrintLevel(3);
+      pcg.SetPreconditioner(amg);
+      pcg.SetOperator(*A);
+      pcg.Mult(B, X);
+
+      int its = -1;
+      pcg.GetNumIterations(its);
+
+      a.RecoverFEMSolution(X, b, x);
+
+      double err1 = x.ComputeL2Error(sin3dCoef);
+      REQUIRE(fabs(err1 - err0) < 1e-6 * err0);
+   }
+}
+
+TEST_CASE("HypreAMS", "[Parallel], [HypreAMS]")
+{
+   int world_size, rank;
+   MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+   int n = 3;
+   int dim = GENERATE(2, 3);
+   int order = 2;
+
+   Mesh mesh = (dim == 2) ?
+               Mesh::MakeCartesian2D(n, n, Element::QUADRILATERAL):
+               Mesh::MakeCartesian3D(n, n, n, Element::HEXAHEDRON);
+
+   int nelems = mesh.GetNE();
+   int *partitioning = new int[nelems];
+
+   PartType last_type = (world_size == 1) ? ALL : ALL_BUT_FIRST;
+   for (int part_type = ALL; part_type <= last_type; part_type++)
+   {
+      GeneratePart((PartType)part_type, nelems, world_size, partitioning);
+
+      ParMesh pmesh(MPI_COMM_WORLD, mesh, partitioning);
+
+      ND_FECollection fec(order, dim);
+      ParFiniteElementSpace fespace(&pmesh, &fec);
+
+      ParBilinearForm a(&fespace);
+      a.AddDomainIntegrator(new CurlCurlIntegrator);
+      a.AddDomainIntegrator(new VectorFEMassIntegrator);
+      a.Assemble();
+
+      ParGridFunction x(&fespace);
+      VectorFunctionCoefficient sinCoef(dim,
+                                        (dim == 2) ? sin2d_vec : sin3d_vec);
+      x.ProjectCoefficient(sinCoef);
+      double err0 = x.ComputeL2Error(sinCoef);
+
+      ParLinearForm b(&fespace);
+      a.Mult(x, b);
+      x = 0.0;
+
+      OperatorPtr A;
+      Vector B, X;
+      Array<int> ess_tdof_list;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+      HypreAMS ams(*A.As<HypreParMatrix>(), &fespace);
+      ams.SetPrintLevel(0);
+
+      HyprePCG pcg(MPI_COMM_WORLD);
+      pcg.SetTol(1e-10);
+      pcg.SetMaxIter(2000);
+      pcg.SetPrintLevel(3);
+      pcg.SetPreconditioner(ams);
+      pcg.SetOperator(*A);
+      pcg.Mult(B, X);
+
+      int its = -1;
+      pcg.GetNumIterations(its);
+
+      a.RecoverFEMSolution(X, b, x);
+
+      double err1 = x.ComputeL2Error(sinCoef);
+      REQUIRE(fabs(err1 - err0) < 1e-6 * err0);
+   }
+}
+
+TEST_CASE("HypreADS", "[Parallel], [HypreADS]")
+{
+   int world_size, rank;
+   MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+   int n = 3;
+   int dim = 3;
+   int order = 2;
+
+   Mesh mesh = Mesh::MakeCartesian3D(n, n, n, Element::HEXAHEDRON);
+
+   int nelems = mesh.GetNE();
+   int *partitioning = new int[nelems];
+
+   PartType last_type = (world_size == 1) ? ALL : ALL_BUT_FIRST;
+   for (int part_type = ALL; part_type <= last_type; part_type++)
+   {
+      GeneratePart((PartType)part_type, nelems, world_size, partitioning);
+
+      ParMesh pmesh(MPI_COMM_WORLD, mesh, partitioning);
+
+      RT_FECollection fec(order, dim);
+      ParFiniteElementSpace fespace(&pmesh, &fec);
+
+      ParBilinearForm a(&fespace);
+      a.AddDomainIntegrator(new DivDivIntegrator);
+      a.AddDomainIntegrator(new VectorFEMassIntegrator);
+      a.Assemble();
+
+      ParGridFunction x(&fespace);
+      VectorFunctionCoefficient sin3dCoef(3, sin3d_vec);
+      x.ProjectCoefficient(sin3dCoef);
+      double err0 = x.ComputeL2Error(sin3dCoef);
+
+      ParLinearForm b(&fespace);
+      a.Mult(x, b);
+      x = 0.0;
+
+      OperatorPtr A;
+      Vector B, X;
+      Array<int> ess_tdof_list;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+      HypreADS ads(*A.As<HypreParMatrix>(), &fespace);
+      ads.SetPrintLevel(0);
+
+      HyprePCG pcg(MPI_COMM_WORLD);
+      pcg.SetTol(1e-10);
+      pcg.SetMaxIter(2000);
+      pcg.SetPrintLevel(3);
+      pcg.SetPreconditioner(ads);
+      pcg.SetOperator(*A);
+      pcg.Mult(B, X);
+
+      int its = -1;
+      pcg.GetNumIterations(its);
+
+      a.RecoverFEMSolution(X, b, x);
+
+      double err1 = x.ComputeL2Error(sin3dCoef);
+      REQUIRE(fabs(err1 - err0) < 1e-6 * err0);
+   }
+}
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem