Cleanup pass

NURBS orientation bug fix

.gitignore

@@ -313,6 +313,8 @@ miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex3
 miniapps/nurbs/nurbs_ex5
+miniapps/nurbs/nurbs_ex10
+miniapps/nurbs/nurbs_ex10p
 miniapps/nurbs/nurbs_ex11p
 miniapps/nurbs/nurbs_ex24
 miniapps/nurbs/nurbs_solenoidal
@@ -338,7 +340,14 @@ miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
 miniapps/nurbs/glvis_naca-cmesh.mesh
 miniapps/nurbs/Naca_cmesh
+miniapps/nurbs/nurbs_mesh_info
+miniapps/nurbs/k*_*.dat
 miniapps/nurbs/*-Surface.mesh
+miniapps/nurbs/*.mesh
+miniapps/nurbs/*.sol
+miniapps/nurbs/deformed.*
+miniapps/nurbs/elastic_energy.*
+miniapps/nurbs/velocity.*
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -360,6 +369,7 @@ miniapps/shifted/lsf_integral
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
+miniapps/tools/compare-dc
 miniapps/tools/gridfunction-bounds
 miniapps/tools/lor-transfer
 miniapps/tools/plor-transfer

CHANGELOG

@@ -11,6 +11,25 @@
 Version 4.9.1 (development)
 ===========================
 
+Discretization improvements
+---------------------------
+- Improved the gridfunction projection routines. Projections work for Scalar,
+  Vector and VectorFE, also NURBS versions. Optionally different types of
+  projections can be selected, default behaviour has not changed.
+
+- Added methods to estimate function extremum using piecewise linear bounds +
+  recursive subdivision.
+
+Meshing improvements
+--------------------
+- Improved support for 1D NURBS meshes with variable order, including using
+  the patches construct for 1D NURBS meshes.
+
+New and updated examples and miniapps
+-------------------------------------
+- Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
+  capability.
+
 
 Version 4.9, released on Dec 11, 2025
 =====================================
@@ -95,6 +114,23 @@ Linear and nonlinear solvers
   Filtering (AMGF), providing robust preconditioning for linear systems arising
   in constrained optimization problems such as frictionless contact.
 
+  Added 'GetResiduals' and 'GetFinalAbsResidualNorm' to 'HyprePCG',
+  'HypreGMRES', and 'HypreFGMRES' to get 'r' and '|r|_p'. Note that the latter
+  computes '|r|_p' from 'r' instead of returning a cached value like the
+  relative 'GetFinalResidualNorm'. These require Hypre >= 2.15.0.
+
+  Changed the default solver parameters for 'HyprePCG' to 'tol=1e-6' and
+  'max_iter=1000'. This matches the default parameters in Hypre 3.0.
+
+  Added various helper functions for querying/modifying Hypre solvers:
+  'HypreSmoother::GetType', 'HypreSmoother::GetSOROptions',
+  'HypreSmoother::GetPolyOptions', 'HypreSmoother::GetWindowParameters',
+  'HypreSmoother::IsOperatorSymmetric', 'HyprePCG::GetTol',
+  'HyprePCG::GetAbsTol', 'HyprePCG::GetMaxIter', 'HyprePCG::SetUseTwoNorm',
+  'HypreGMRES::GetTol', 'HypreGMRES::GetAbsTol', 'HypreGMRES::GetMaxIter',
+  'HypreGMRES::GetKDim', 'HypreFGMRES::GetTol', 'HypreFGMRES::GetMaxIter',
+  'HypreFGMRES::GetKDim', and 'HypreBoomerAMG::GetMaxIter'.
+
 GPU computing
 -------------
 - Added the 'gpu', 'raja-gpu', and 'ceed-gpu' backend aliases/shortcuts which

CMakeLists.txt

@@ -723,6 +723,7 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
 
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
+target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
 if (TPL_TARGETS)
@@ -869,11 +870,12 @@ add_dependencies(exec
 #  - https://cmake.org/Bug/view.php?id=8438
 
 # Add a target to copy the mfem data directory to the build directory
-add_custom_command(OUTPUT data_is_copied
-  COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/data data
-  COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
-  COMMENT "Copying the data directory ...")
-add_custom_target(copy_data DEPENDS data_is_copied)
+# Implementable as a single copy_directory_if_different command w/ CMake >= 3.26
+file(GLOB DATA_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/data/*)
+add_custom_target(copy_data
+  COMMAND ${CMAKE_COMMAND} -E make_directory data
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${DATA_FILES} data
+  COMMENT "Syncing the data directory ...")
 # Add 'copy_data' as a prerequisite for all executables, if the source and the
 # build directories are not the same.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
@@ -1005,6 +1007,10 @@ install(FILES
 install(EXPORT ${PROJECT_NAME_UC}Targets
     DESTINATION ${INSTALL_CMAKE_DIR})
 
+# Install the data directory if present, i.e. if the copy_data target is built
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
+  DESTINATION ${MFEM_INSTALL_DIR} OPTIONAL)
+
 #-------------------------------------------------------------------------------
 # Create 'config.mk' from 'config.mk.in' for the build and install locations and
 # define install rules for 'config.mk' and 'test.mk'

INSTALL

@@ -725,7 +725,9 @@ The specific libraries and their options are:
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
            Debug).
-  Versions: Ginkgo >= 1.9.0.
+  Versions: Ginkgo >= 1.9.0. When building Ginkgo with distributed support, a
+            recent version of the "develop" branch is required (1.11 as defined
+            in include/ginkgo/config.hpp).
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
   URL: https://github.com/NVIDIA/AMGX

config/defaults.mk

@@ -18,6 +18,7 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))
 
+ASTYLE_BIN = astyle
 ETAGS_BIN = $(shell command -v etags 2> /dev/null)
 EGREP_BIN = $(shell command -v egrep 2> /dev/null)
 
@@ -407,7 +408,7 @@ AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt
 # MAGMA library configuration
 MAGMA_DIR = @MFEM_DIR@/../magma
 MAGMA_OPT = -I$(MAGMA_DIR)/include
-MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a -lcublas -lcusparse $(LAPACK_LIB)
+MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a $(LAPACK_LIB)
 
 # GnuTLS library configuration
 GNUTLS_OPT =

data/nurbs-segments2d-patches-multispan.mesh

@@ -0,0 +1,86 @@
+MFEM NURBS mesh v1.0
+
+dimension
+1
+
+# Four segments with different NURBS orders, described via patches.
+elements
+4
+1 1 0 1
+2 1 2 3
+3 1 4 5
+4 1 6 7
+
+boundary
+0
+
+edges
+4
+0 0 1
+1 2 3
+2 4 5
+3 6 7
+
+vertices
+8
+
+patches
+
+# Patch 0: linear (order 1, 3 spans)
+knotvectors
+1
+1 4  0 0  .4 .6 1 1
+
+dimension
+2
+
+controlpoints
+0.0  0.0  1.0
+0.6  0.4  1.0
+0.4  0.6  1.0
+1.0  1.0  1.0
+
+# Patch 1: quadratic (order 2, 2 spans)
+knotvectors
+1
+2 4  0 0 0  .5 1 1 1
+
+dimension
+2
+
+controlpoints
+1.0  0.0  1.0
+1.9  0.0  1.21
+2.0  0.9  1.22
+2.0  1.0  1.0
+
+# Patch 2: cubic (order 3, 3 spans)
+knotvectors
+1
+3 6  0 0 0 0 .33 .66 1 1 1 1
+
+dimension
+2
+
+controlpoints
+2.0   0.0  1.0
+2.1   0.2  1.31
+3.5   0.4  1.32
+2.5   0.6  1.33
+2.9   1.0  1.34
+3.0   1.0  1.0
+
+# Patch 3: quartic (order 4, 1 span)
+knotvectors
+1
+4 5  0 0 0 0 0 1 1 1 1 1
+
+dimension
+2
+
+controlpoints
+3.0    0.0  1.0
+3.45   0.5  1.41
+3.50   1.0  1.42
+3.75   0.8  1.43
+4.0    0.0  1.0

data/nurbs-segments2d-patches.mesh

@@ -0,0 +1,79 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+# Three segments with different NURBS orders, described via patches.
+elements
+3
+1 1 0 1
+2 1 2 3
+3 1 4 5
+
+boundary
+6
+1 0 0
+1 0 1
+1 0 2
+1 0 3
+1 0 4
+1 0 5
+
+edges
+3
+0 0 1
+1 2 3
+2 4 5
+
+vertices
+6
+
+patches
+
+# Patch 0: linear (order 1, 2 control points)
+knotvectors
+1
+1 2  0 0  1 1
+
+dimension
+2
+
+controlpoints
+0.0  0.0  1.0
+1.0  1.0  1.0
+
+# Patch 1: quadratic (order 2, 3 control points)
+knotvectors
+1
+2 3  0 0 0  1 1 1
+
+dimension
+2
+
+controlpoints
+1.0  0.0   1.0
+1.02 1.02  1.2
+2.0  1.0   1.0
+
+# Patch 2: cubic (order 3, 4 control points)
+knotvectors
+1
+3 4  0 0 0 0  1 1 1 1
+
+dimension
+2
+
+controlpoints
+2.0   0.0   1.0
+2.03  0.83  1.31
+2.33  1.03  1.32
+3.0   1.0   1.0
+

data/nurbs-segments2d.mesh

@@ -0,0 +1,72 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+elements
+3
+1 1 0 1
+2 1 2 3
+3 1 4 5
+
+boundary
+6
+1 0 0
+1 0 1
+1 0 2
+1 0 3
+1 0 4
+1 0 5
+
+edges
+3
+0 0 1
+1 2 3
+2 4 5
+
+vertices
+6
+
+# Edge 0: linear (order 1, 2 control points)
+# Edge 1: quadratic (order 2, 3 control points)
+# Edge 2: cubic (order 3, 4 control points)
+knotvectors
+3
+1 2  0 0  1 1
+2 3  0 0 0  1 1 1
+3 4  0 0 0 0  1 1 1 1
+
+# One weight per control point, in the same order as the control points; (2 + 3 + 4) = 9 weights total
+weights
+1
+1
+1
+1
+1
+1
+1.2
+1.31
+1.32
+
+FiniteElementSpace
+FiniteElementCollection: NURBS
+VDim: 2
+Ordering: 1
+
+0.0  0.0
+1.0  1.0
+1.0  0.0
+2.0  1.0
+2.0  0.0
+3.0  1.0
+1.02 1.02
+2.03 0.83
+2.33 1.03

data/nurbs-segments3d-patches.mesh

@@ -0,0 +1,79 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+# Three segments with different NURBS orders, described via patches.
+elements
+3
+1 1 0 1
+2 1 2 3
+3 1 4 5
+
+boundary
+6
+1 0 0
+1 0 1
+1 0 2
+1 0 3
+1 0 4
+1 0 5
+
+edges
+3
+0 0 1
+1 2 3
+2 4 5
+
+vertices
+6
+
+patches
+
+# Patch 0: linear (order 1, 2 control points)
+knotvectors
+1
+1 2  0 0  1 1
+
+dimension
+3
+
+controlpoints
+0.0  0.0  0.01 1.0
+1.0  1.0  1.01 1.0
+
+# Patch 1: quadratic (order 2, 3 control points)
+knotvectors
+1
+2 3  0 0 0  1 1 1
+
+dimension
+3
+
+controlpoints
+1.0  0.0   0.02 1.0
+1.02 1.02  0.52 1.2
+2.0  1.0   1.02 1.0
+
+# Patch 2: cubic (order 3, 4 control points)
+knotvectors
+1
+3 4  0 0 0 0  1 1 1 1
+
+dimension
+3
+
+controlpoints
+2.0   0.0   0.03 1.0
+2.03  0.83  0.33 1.31
+2.33  1.03  0.63 1.32
+3.0   1.0   1.03 1.0
+

data/nurbs-segments3d.mesh

@@ -0,0 +1,72 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+elements
+3
+1 1 0 1
+2 1 2 3
+3 1 4 5
+
+boundary
+6
+1 0 0
+1 0 1
+1 0 2
+1 0 3
+1 0 4
+1 0 5
+
+edges
+3
+0 0 1
+1 2 3
+2 4 5
+
+vertices
+6
+
+# Edge 0: linear (order 1, 2 control points)
+# Edge 1: quadratic (order 2, 3 control points)
+# Edge 2: cubic (order 3, 4 control points)
+knotvectors
+3
+1 2  0 0  1 1
+2 3  0 0 0  1 1 1
+3 4  0 0 0 0  1 1 1 1
+
+# One weight per control point, in the same order as the control points; (2 + 3 + 4) = 9 weights total
+weights
+1
+1
+1
+1
+1
+1
+1.2
+1.31
+1.32
+
+FiniteElementSpace
+FiniteElementCollection: NURBS
+VDim: 3
+Ordering: 1
+
+0.0  0.0   0.01
+1.0  1.0   1.01
+1.0  0.0   0.02
+2.0  1.0   1.02
+2.0  0.0   0.03
+3.0  1.0   1.03
+1.02 1.02  0.52
+2.03 0.83  0.33
+2.33 1.03  0.63

doc/CodeDocumentation.dox

@@ -117,6 +117,8 @@ namespace mfem {
  * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
  * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
  * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
+ * - <a class="el" href="ex41_8cpp_source.html">Example 41</a>: DG/CG IMEX time-dependent advection-diffusion
+ * - <a class="el" href="ex41p_8cpp_source.html">Example 41p</a>: parallel DG/CG IMEX time-dependent advection-diffusion
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples
@@ -188,6 +190,8 @@ namespace mfem {
  *  <a class="el" href="nurbs__ex1p_8cpp_source.html">1p</a>,
  *  <a class="el" href="nurbs__ex3_8cpp_source.html">3</a>,
  *  <a class="el" href="nurbs__ex5_8cpp_source.html">5</a>,
+ *  <a class="el" href="nurbs__ex10_8cpp_source.html">10</a>,
+ *  <a class="el" href="nurbs__ex10p_8cpp_source.html">10p</a>,
  *  <a class="el" href="nurbs__ex11p_8cpp_source.html">11p</a>, and
  *  <a class="el" href="nurbs__ex24_8cpp_source.html">24</a>,
  *   demonstrating howto perform NURBS-based Isogeometric Analysis.
@@ -196,6 +200,7 @@ namespace mfem {
  * - <a class="el" href="nurbs__curveint_8cpp_source.html">NURBS Interpolation</a>: NURBS interpolation of given geometry
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
+ * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -234,7 +239,8 @@ namespace mfem {
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Poisson problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Poisson problem
  * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
- * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Tribol</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="contact_8cpp_source.html">Contact</a>: Frictionless contact examples using <a class="el" href="classmfem_1_1IPSolver.html#details">IP optimization</a> and the <a class="el" href="classmfem_1_1AMGFSolver.html#details">AMGF solver</a>
  * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem

examples/ex16.cpp

@@ -105,6 +105,7 @@ int main(int argc, char *argv[])
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
+   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -126,6 +127,9 @@ int main(int argc, char *argv[])
                   "Alpha coefficient.");
    args.AddOption(&kappa, "-k", "--kappa",
                   "Kappa coefficient offset.");
+   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
+                  "-imp-slope", "--implicit-slope",
+                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -179,6 +183,11 @@ int main(int argc, char *argv[])
 
    // 7. Initialize the conduction operator and the visualization.
    ConductionOperator oper(fespace, alpha, kappa, u);
+   using ImplicitVariableType = ConductionOperator::ImplicitVariableType;
+   ImplicitVariableType imp_var = solve_implicit_state ?
+                                  ImplicitVariableType::STATE
+                                  : ImplicitVariableType::SLOPE;
+   oper.SetImplicitVariableType(imp_var);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -316,11 +325,14 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 }
 
 void ConductionOperator::ImplicitSolve(const real_t dt,
-                                       const Vector &u, Vector &du_dt)
+                                       const Vector &u, Vector &k)
 {
    // Solve the equation:
-   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    M*k = -K(u + dt*k) for k = du/dt, if solving for stage-slope
+   // or
+   //    M*k = -dt*K(k) + M*u for k = u_s, if solving for stage-state
+   // where K is linearized by using u from the previous timestep, and
+   // the stage-state and slope relation: du/dt = (u_s - u)/dt.
    if (!T)
    {
       T = Add(1.0, Mmat, dt, Kmat);
@@ -328,9 +340,20 @@ void ConductionOperator::ImplicitSolve(const real_t dt,
       T_solver.SetOperator(*T);
    }
    MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-   Kmat.Mult(u, z);
-   z.Neg();
-   T_solver.Mult(z, du_dt);
+
+   // Construct current right-hand side for stage state vs. slope solve
+   if (ImplicitVarTypeIsState())
+   {
+      // k, on return, is the stage value u_s
+      Mmat.Mult(u, z);
+   }
+   else
+   {
+      // k, on return, is the stage slope du/dt
+      Kmat.Mult(u, z);
+      z.Neg();
+   }
+   T_solver.Mult(z, k);
 }
 
 void ConductionOperator::SetParameters(const Vector &u)

examples/ex16p.cpp

@@ -115,6 +115,7 @@ int main(int argc, char *argv[])
    bool visit = false;
    int vis_steps = 5;
    bool adios2 = false;
+   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -138,6 +139,9 @@ int main(int argc, char *argv[])
                   "Alpha coefficient.");
    args.AddOption(&kappa, "-k", "--kappa",
                   "Kappa coefficient offset.");
+   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
+                  "-imp-slope", "--implicit-slope",
+                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -212,6 +216,11 @@ int main(int argc, char *argv[])
 
    // 9. Initialize the conduction operator and the VisIt visualization.
    ConductionOperator oper(fespace, alpha, kappa, u);
+   using ImplicitVariableType = ConductionOperator::ImplicitVariableType;
+   ImplicitVariableType imp_var = solve_implicit_state ?
+                                  ImplicitVariableType::STATE
+                                  : ImplicitVariableType::SLOPE;
+   oper.SetImplicitVariableType(imp_var);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -407,11 +416,14 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 }
 
 void ConductionOperator::ImplicitSolve(const real_t dt,
-                                       const Vector &u, Vector &du_dt)
+                                       const Vector &u, Vector &k)
 {
    // Solve the equation:
-   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    M*k = -K(u + dt*k) for k = du/dt, if solving for stage-slope
+   // or
+   //    M*k = -dt*K(k) + M*u for k = u_s, if solving for stage-state
+   // where K is linearized by using u from the previous timestep, and
+   // the stage-state and slope relation: du/dt = (u_s - u)/dt.
    if (!T)
    {
       T = Add(1.0, Mmat, dt, Kmat);
@@ -419,9 +431,20 @@ void ConductionOperator::ImplicitSolve(const real_t dt,
       T_solver.SetOperator(*T);
    }
    MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-   Kmat.Mult(u, z);
-   z.Neg();
-   T_solver.Mult(z, du_dt);
+
+   // Construct current right-hand side for stage state vs. slope solve
+   if (ImplicitVarTypeIsState())
+   {
+      // k, on return, is the stage value u
+      Mmat.Mult(u, z);
+   }
+   else
+   {
+      // k, on return, is the stage slope du/dt
+      Kmat.Mult(u, z);
+      z.Neg();
+   }
+   T_solver.Mult(z, k);
 }
 
 void ConductionOperator::SetParameters(const Vector &u)

examples/ex21.cpp

@@ -119,7 +119,7 @@ int main(int argc, char *argv[])
    }
 
    LinearForm b(&fespace);
-   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
+   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
 
    // 6. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the linear elasticity integrator with piece-wise

examples/ex21p.cpp

@@ -140,7 +140,7 @@ int main(int argc, char *argv[])
    }
 
    ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
+   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
 
    // 6. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the linear elasticity integrator with piece-wise

examples/ex4.cpp

@@ -9,6 +9,7 @@
 //               ex4 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex4 -m ../data/escher.mesh
 //               ex4 -m ../data/fichera.mesh -o 2 -hb
+//               ex4 -m ../data/fichera.mesh -o 2 -hb -ea
 //               ex4 -m ../data/fichera-q2.vtk
 //               ex4 -m ../data/fichera-q3.mesh -o 2 -sc
 //               ex4 -m ../data/square-disc-nurbs.mesh
@@ -18,6 +19,7 @@
 //               ex4 -m ../data/amr-quad.mesh
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
+//               ex4 -m ../data/amr-hex.mesh -o 2 -hb -ea
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
 //               ex4 -m ../data/ref-prism.mesh -o 1
 //               ex4 -m ../data/octahedron.mesh -o 1
@@ -25,6 +27,8 @@
 //
 // Device sample runs:
 //               ex4 -m ../data/star.mesh -pa -d cuda
+//               ex4 -m ../data/star.mesh -hb -ea -d cuda
+//               ex4 -m ../data/amr-quad.mesh -hb -ea -d cuda
 //               ex4 -m ../data/star.mesh -pa -d raja-cuda
 //               ex4 -m ../data/star.mesh -pa -d raja-omp
 //               ex4 -m ../data/beam-hex.mesh -pa -d cuda
@@ -193,7 +197,7 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
-   if (!pa)
+   if (!pa && (!ea || hybridization))
    {
 #ifndef MFEM_USE_SUITESPARSE
       // Use a simple symmetric Gauss-Seidel preconditioner with PCG.

examples/ex4p.cpp

@@ -9,6 +9,7 @@
 //               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -o 2 -pa
 //               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
 //               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
@@ -17,14 +18,18 @@
 //               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
+//               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d cuda
+//               mpirun -np 4 ex4p -m ../data/star.mesh -ea -hb -d cuda
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -ea -hb -d cuda
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-cuda
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-omp
 //               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -pa -d cuda
@@ -230,7 +235,7 @@ int main(int argc, char *argv[])
    pcg->SetMaxIter(2000);
    pcg->SetPrintLevel(1);
    if (hybridization) { prec = new HypreBoomerAMG(*A.As<HypreParMatrix>()); }
-   else if (pa) { prec = new OperatorJacobiSmoother(*a, ess_tdof_list); }
+   else if (pa || ea) { prec = new OperatorJacobiSmoother(*a, ess_tdof_list); }
    else
    {
       ParFiniteElementSpace *prec_fespace =

examples/ex9.cpp

@@ -160,6 +160,7 @@ int main(int argc, char *argv[])
    bool paraview = false;
    bool binary = false;
    int vis_steps = 5;
+   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -187,6 +188,9 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
+   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
+                  "-imp-slope", "--implicit-slope",
+                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -366,6 +370,11 @@ int main(int argc, char *argv[])
    //    right-hand side, and perform time-integration (looping over the time
    //    iterations, ti, with a time-step dt).
    FE_Evolution adv(m, k, b);
+   using ImplicitVariableType = FE_Evolution::ImplicitVariableType;
+   ImplicitVariableType imp_var = solve_implicit_state ?
+                                  ImplicitVariableType::STATE
+                                  : ImplicitVariableType::SLOPE;
+   adv.SetImplicitVariableType(imp_var);
 
    real_t t = 0.0;
    adv.SetTime(t);
@@ -459,7 +468,17 @@ void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
    MFEM_VERIFY(dg_solver != NULL,
                "Implicit time integration is not supported with partial assembly");
-   K.Mult(x, z);
+   // Construct current right-hand side for stage state vs. slope solve
+   if (ImplicitVarTypeIsState())
+   {
+      // k, on return, is the stage value u
+      M.Mult(x, z);
+   }
+   else
+   {
+      // k, on return, is the stage slope du/dt
+      K.Mult(x, z);
+   }
    z += b;
    dg_solver->SetTimeStep(dt);
    dg_solver->Mult(z, k);

examples/ex9p.cpp

@@ -257,6 +257,7 @@ int main(int argc, char *argv[])
    bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
+   bool solve_implicit_state = false;
 #if MFEM_HYPRE_VERSION >= 21800
    PrecType prec_type = PrecType::AIR;
 #else
@@ -290,6 +291,9 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
+   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
+                  "-imp-slope", "--implicit-slope",
+                  "Implicitly solve for stage state or slope.");
    args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
                   "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -536,6 +540,11 @@ int main(int argc, char *argv[])
    //     right-hand side, and perform time-integration (looping over the time
    //     iterations, ti, with a time-step dt).
    FE_Evolution adv(*m, *k, *B, prec_type);
+   using ImplicitVariableType = FE_Evolution::ImplicitVariableType;
+   ImplicitVariableType imp_var = solve_implicit_state ?
+                                  ImplicitVariableType::STATE
+                                  : ImplicitVariableType::SLOPE;
+   adv.SetImplicitVariableType(imp_var);
 
    real_t t = 0.0;
    adv.SetTime(t);
@@ -676,7 +685,17 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
 //    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
-   K->Mult(x, z);
+   // Construct current right-hand side for stage state vs. slope solve
+   if (ImplicitVarTypeIsState())
+   {
+      // k, on return, is the stage value u
+      M->Mult(x, z);
+   }
+   else
+   {
+      // k, on return, is the stage slope du/dt
+      K->Mult(x, z);
+   }
    z += b;
    dg_solver->SetTimeStep(dt);
    dg_solver->Mult(z, k);

examples/ginkgo/CMakeLists.txt

@@ -14,6 +14,12 @@ list(APPEND GINKGO_EXAMPLES_SRCS
   ex1.cpp
 )
 
+if (MFEM_USE_MPI AND GINKGO_BUILD_MPI)
+    list(APPEND GINKGO_EXAMPLES_SRCS
+    ex1p.cpp
+)
+endif()
+
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 

examples/ginkgo/ex1.cpp

@@ -207,7 +207,7 @@ int main(int argc, char *argv[])
             Ginkgo::IcPreconditioner ginkgo_precond(exec, "paric", 30);
             Ginkgo::CGSolver ginkgo_solver(exec, ginkgo_precond);
             ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(1e-12);
+            ginkgo_solver.SetRelTol(sqrt(1e-12));
             ginkgo_solver.SetAbsTol(0.0);
             ginkgo_solver.SetMaxIter(400);
             ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -225,7 +225,7 @@ int main(int argc, char *argv[])
             Ginkgo::MFEMPreconditioner gko_M(exec, M);
             Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
             ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(1e-12);
+            ginkgo_solver.SetRelTol(sqrt(1e-12));
             ginkgo_solver.SetAbsTol(0.0);
             ginkgo_solver.SetMaxIter(400);
             ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -283,7 +283,7 @@ int main(int argc, char *argv[])
                Ginkgo::MFEMPreconditioner gko_M(exec, M);
                Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
                ginkgo_solver.SetPrintLevel(print_lvl);
-               ginkgo_solver.SetRelTol(1e-12);
+               ginkgo_solver.SetRelTol(sqrt(1e-12));
                ginkgo_solver.SetAbsTol(0.0);
                ginkgo_solver.SetMaxIter(400);
                ginkgo_solver.SetOperator(*(A.Ptr()));

examples/ginkgo/ex1p.cpp

@@ -0,0 +1,436 @@
+//                       MFEM Example 1 - Parallel Version
+//                             GINKGO Modification
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
+//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
+//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Device sample runs:
+//               mpirun -np 4 ex1p -pa -d cuda
+//               mpirun -np 4 ex1p -fa -d cuda
+//               mpirun -np 4 ex1p -pa -d occa-cuda
+//               mpirun -np 4 ex1p -pa -d raja-omp
+//               mpirun -np 4 ex1p -pa -d ceed-cpu
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/square-mixed.mesh
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
+//             * mpirun -np 4 ex1p -pa -d ceed-cuda
+//             * mpirun -np 4 ex1p -pa -d ceed-hip
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifndef MFEM_USE_GINKGO
+#error This example requires that MFEM is built with MFEM_USE_GINKGO=YES
+#endif
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool pa = false;
+   bool fa = false;
+   const char *device_config = "cpu";
+   bool visualization = true;
+   int solver_config = 0;
+   int print_lvl = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&solver_config, "-s", "--solver-config",
+                  "Solver and preconditioner combination: \n\t"
+                  "   0 - Ginkgo solver and Ginkgo preconditioner, \n\t"
+                  "   1 - Ginkgo solver and MFEM preconditioner, \n\t"
+                  "   2 - MFEM solver and Ginkgo preconditioner, \n\t"
+                  "   3 - MFEM solver and MFEM preconditioner.");
+   args.AddOption(&print_lvl, "-pl", "--print-level",
+                  "Print level for iterative solver (1 prints every iteration).");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.SetGPUAwareMPI(true);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh.UniformRefinement();
+      }
+   }
+
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   bool delete_fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+      delete_fec = true;
+   }
+   else if (pmesh.GetNodes())
+   {
+      fec = pmesh.GetNodes()->OwnFEC();
+      delete_fec = false;
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+      delete_fec = true;
+   }
+   ParFiniteElementSpace fespace(&pmesh, fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 8. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm b(&fespace);
+   ConstantCoefficient one(1.0);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
+
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
+   ParBilinearForm a(&fespace);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (fa)
+   {
+      a.SetAssemblyLevel(AssemblyLevel::FULL);
+      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
+      // when Device::IsEnabled() returns true). This makes the results
+      // bit-for-bit deterministic at the cost of somewhat longer run time.
+      a.EnableSparseMatrixSorting(Device::IsEnabled());
+   }
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 12. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // 13. Solve the linear system A X = B.
+   if (!pa)
+   {
+      switch (solver_config)
+      {
+         // Solve the linear system with CG + Schwarz (with IC) from Ginkgo
+         case 0:
+         {
+            if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
+            Ginkgo::GinkgoExecutor exec(device);
+            Ginkgo::IcPreconditioner local_solver(exec, "exact");
+            Ginkgo::SchwarzPreconditioner gko_M(exec, MPI_COMM_WORLD, local_solver);
+            Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
+            ginkgo_solver.SetPrintLevel(print_lvl);
+            ginkgo_solver.SetRelTol(sqrt(1e-12));
+            ginkgo_solver.SetAbsTol(0.0);
+            ginkgo_solver.SetMaxIter(400);
+            ginkgo_solver.SetOperator(*(A.Ptr()));
+            ginkgo_solver.Mult(B, X);
+            break;
+         }
+
+         // Solve the linear system with CG from Ginkgo + MFEM preconditioner
+         case 1:
+         {
+            if (myid == 0) { cout << "Using Ginkgo solver + MFEM preconditioner...\n"; }
+            Ginkgo::GinkgoExecutor exec(device);
+            //Create MFEM preconditioner and wrap it for Ginkgo's use.
+            HypreBoomerAMG M((HypreParMatrix&)(*A));
+            Ginkgo::MFEMPreconditioner gko_M(exec, M, MPI_COMM_WORLD);
+            Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
+            ginkgo_solver.SetPrintLevel(print_lvl);
+            ginkgo_solver.SetRelTol(sqrt(1e-12));
+            ginkgo_solver.SetAbsTol(0.0);
+            ginkgo_solver.SetMaxIter(400);
+            ginkgo_solver.SetOperator(*(A.Ptr()));
+            ginkgo_solver.Mult(B, X);
+            break;
+         }
+
+         // Ginkgo Schwarz preconditioner (local ParIC) +  MFEM CG solver
+         case 2:
+         {
+            if (myid == 0) { cout << "Using MFEM solver + Ginkgo preconditioner...\n"; }
+            Ginkgo::GinkgoExecutor exec(device);
+            Ginkgo::IcPreconditioner local_M(exec, "exact");
+            Ginkgo::SchwarzPreconditioner M(exec, MPI_COMM_WORLD, local_M);
+            M.SetOperator(*(A.Ptr()));  // Generate the preconditioner for the matrix A.
+            CGSolver cg(MPI_COMM_WORLD);
+            cg.SetRelTol(sqrt(1e-12));
+            cg.SetMaxIter(400);
+            cg.SetPrintLevel(1);
+            cg.SetPreconditioner(M);
+            cg.SetOperator(*A);
+            cg.Mult(B, X);
+            break;
+         }
+
+         // MFEM solver + MFEM preconditioner
+         case 3:
+         {
+            if (myid == 0) { cout << "Using MFEM solver + MFEM preconditioner...\n"; }
+            HypreBoomerAMG M((HypreParMatrix&)(*A));
+            CGSolver cg(MPI_COMM_WORLD);
+            cg.SetRelTol(sqrt(1e-12));
+            cg.SetMaxIter(400);
+            cg.SetPrintLevel(1);
+            cg.SetPreconditioner(M);
+            cg.SetOperator(*A);
+            cg.Mult(B, X);
+            break;
+         }
+      } // End switch on solver_config
+   }
+   // Partial assembly mode.  Cannot use Ginkgo preconditioners, but can use Ginkgo
+   // solvers.
+   else
+   {
+      if (UsesTensorBasis(fespace))
+      {
+         // Use Jacobi preconditioning in partial assembly mode.
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         switch (solver_config)
+         {
+            case 0:
+            {
+               if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
+               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
+                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
+               break;
+            }
+
+            // Use Ginkgo solver with MFEM preconditioner
+            case 1:
+            {
+               if (myid == 0) { cout << "Using Ginkgo solver + MFEM preconditioner...\n"; }
+               Ginkgo::GinkgoExecutor exec(device);
+               // Wrap MFEM preconditioner for Ginkgo's use.
+               Ginkgo::MFEMPreconditioner gko_M(exec, M, MPI_COMM_WORLD);
+               Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
+               ginkgo_solver.SetPrintLevel(print_lvl);
+               ginkgo_solver.SetRelTol(sqrt(1e-12));
+               ginkgo_solver.SetAbsTol(0.0);
+               ginkgo_solver.SetMaxIter(400);
+               ginkgo_solver.SetOperator(*(A.Ptr()));
+               ginkgo_solver.Mult(B, X);
+               break;
+            }
+
+            // No Ginkgo preconditioners work with matrix-free; error
+            case 2:
+            {
+               if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
+               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
+                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
+               break;
+            }
+
+            // Use MFEM solver and preconditioner
+            case 3:
+            {
+               if (myid == 0) { cout << "Using MFEM solver + MFEM preconditioner...\n"; }
+               CGSolver cg(MPI_COMM_WORLD);
+               cg.SetRelTol(sqrt(1e-12));
+               cg.SetMaxIter(400);
+               cg.SetPrintLevel(1);
+               cg.SetPreconditioner(M);
+               cg.SetOperator(*A);
+               cg.Mult(B, X);
+               break;
+            }
+         } // End switch on solver_config
+      }
+      else // CG with no preconditioning
+      {
+         if (myid == 0) { cout << "Using MFEM solver + no preconditioner...\n"; }
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(sqrt(1e-12));
+         cg.SetMaxIter(400);
+         cg.SetPrintLevel(1);
+         cg.SetOperator(*A);
+         cg.Mult(B, X);
+      }
+   }
+
+   // 14. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   if (delete_fec)
+   {
+      delete fec;
+   }
+
+   return 0;
+}

examples/ginkgo/makefile

@@ -20,9 +20,8 @@ CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-# Currently there are only serial Ginkgo examples
 SEQ_EXAMPLES = ex1
-PAR_EXAMPLES =
+PAR_EXAMPLES = ex1p
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else

fem/bilinearform.cpp

@@ -825,14 +825,46 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
                                     Vector &b, OperatorHandle &A, Vector &X,
                                     Vector &B, int copy_interior)
 {
+   const SparseMatrix *P = fes->GetConformingProlongation();
+   const SparseMatrix *R = fes->GetConformingRestriction();
    if (ext)
    {
       if (hybridization)
       {
          FormSystemMatrix(ess_tdof_list, A);
-         ConstrainedOperator A_constrained(this, ess_tdof_list);
-         A_constrained.EliminateRHS(x, b);
-         hybridization->ReduceRHS(b, B);
+
+         std::unique_ptr<ConstrainedOperator> A_constrained([&]()
+         {
+            Operator *op;
+            Operator::FormSystemOperator(ess_tdof_list, op);
+            return dynamic_cast<ConstrainedOperator*>(op);
+         }());
+         MFEM_ASSERT(A_constrained != nullptr, "");
+
+         Vector conf_b, conf_x;
+         if (P)
+         {
+            // Nonconforming
+            conf_b.SetSize(P->Width());
+            conf_x.SetSize(P->Width());
+            P->MultTranspose(b, conf_b);
+            R->Mult(x, conf_x);
+         }
+         else
+         {
+            // Conforming
+            conf_b.MakeRef(b, 0, b.Size());
+            conf_x.MakeRef(x, 0, x.Size());
+         }
+
+         A_constrained->EliminateRHS(conf_x, conf_b);
+
+         if (P)
+         {
+            R->MultTranspose(conf_b, b); // store eliminated rhs in b
+         }
+
+         hybridization->ReduceRHS(conf_b, B);
          X.SetSize(B.Size());
          X = 0.0;
       }
@@ -842,7 +874,6 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       }
       return;
    }
-   const SparseMatrix *P = fes->GetConformingProlongation();
    FormSystemMatrix(ess_tdof_list, A);
 
    // Transform the system and perform the elimination in B, based on the
@@ -878,7 +909,6 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       if (hybridization)
       {
          // Reduction to the Lagrange multipliers system
-         const SparseMatrix *R = fes->GetConformingRestriction();
          Vector conf_b(P->Width()), conf_x(P->Width());
          P->MultTranspose(b, conf_b);
          R->Mult(x, conf_x);
@@ -891,7 +921,6 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       else
       {
          // Variational restriction with P
-         const SparseMatrix *R = fes->GetConformingRestriction();
          B.SetSize(P->Width());
          P->MultTranspose(b, B);
          X.SetSize(R->Height());

fem/bounds.cpp

@@ -39,8 +39,8 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
    b_type = b_type_i;
    cp_type = cp_type_i;
    tol = tol_i;
-   lbound.SetSize(nb, ncp);
-   ubound.SetSize(nb, ncp);
+   lbound.SetSize(ncp, nb);
+   ubound.SetSize(ncp, nb);
    nodes.SetSize(nb);
    weights.SetSize(nb);
    control_points.SetSize(ncp);
@@ -125,21 +125,25 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
       {
          if (j == 0)
          {
-            lbound(i, j) = bv(i);
-            ubound(i, j) = bv(i);
+            lbound(j,i) = bv(i);
+            ubound(j,i) = bv(i);
          }
          else if (j == ncp-1)
          {
-            lbound(i, j) = bv(i);
-            ubound(i, j) = bv(i);
+            lbound(j,i) = bv(i);
+            ubound(j,i) = bv(i);
          }
          else
          {
             vals(0)  = bv(i);
             vals(1) =  bmv(i) +  dm*bdmv(i);
             vals(2) =  bpv(i) +  dp*bdpv(i);
-            lbound(i, j) = vals.Min()-tol; // tolerance for good measure
-            ubound(i, j) = vals.Max()+tol; // tolerance for good measure
+            lbound(j,i) = vals.Min()-tol; // tolerance for good measure
+            ubound(j,i) = vals.Max()+tol; // tolerance for good measure
+            if (b_type == 2)
+            {
+               lbound(j,i) = std::max(lbound(j,i),0_r);
+            }
          }
       }
    }
@@ -273,8 +277,7 @@ void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
    intmax.SetSize(ncp);
    intmin = 0.0;
    intmax = 0.0;
-   Vector coeffm(nb);
-   coeffm = 0.0;
+   Vector coeffm;
 
    real_t a0 = 0.0;
    real_t a1 = 0.0;
@@ -302,6 +305,8 @@ void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
    // compute L2 projection for linear bases: a0 + a1*x
    if (proj)
    {
+      coeffm.SetSize(nb);
+      coeffm = 0.0;
       for (int i = 0; i < nb; i++)
       {
          x = 2.0*nodes_int(i)-1;
@@ -342,8 +347,8 @@ void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
       real_t c = coeffm(i);
       for (int j = 0; j < ncp; j++)
       {
-         intmin(j) += min(lbound(i,j)*c, ubound(i,j)*c);
-         intmax(j) += max(lbound(i,j)*c, ubound(i,j)*c);
+         intmin(j) += min(lbound(j,i)*c, ubound(j,i)*c);
+         intmax(j) += max(lbound(j,i)*c, ubound(j,i)*c);
       }
    }
 }
@@ -474,10 +479,10 @@ void PLBound::Get2DBounds(const Vector &coeff, Vector &intmin,
          real_t w1 = intmaxT(id2++);
          for (int k = 0; k < ncp; k++) // kth row
          {
-            vals(0) = w0*lbound(j,k);
-            vals(1) = w0*ubound(j,k);
-            vals(2) = w1*lbound(j,k);
-            vals(3) = w1*ubound(j,k);
+            vals(0) = w0*lbound(k,j);
+            vals(1) = w0*ubound(k,j);
+            vals(2) = w1*lbound(k,j);
+            vals(3) = w1*ubound(k,j);
             intmin(k*ncp+i) += vals.Min();
             intmax(k*ncp+i) += vals.Max();
          }
@@ -553,17 +558,17 @@ void PLBound::Get3DBounds(const Vector &coeff, Vector &intmin,
             for (int i = 0; i < nb; i++)
             {
                x = 2.0*nodes(i)-1; // x-coordinate
-               minBounds(i) -= a0V(j) + a1V(j)*x;
-               maxBounds(i) -= a0V(j) + a1V(j)*x;
+               minNodalVals(i) -= a0V(j) + a1V(j)*x;
+               maxNodalVals(i) -= a0V(j) + a1V(j)*x;
             }
             // Compute Bernstein coefficients
             LUFactors lu(basisMatLU.GetData(), lu_ip.GetData());
-            lu.Solve(nb, 1, minBounds.GetData());
-            lu.Solve(nb, 1, maxBounds.GetData());
+            lu.Solve(nb, 1, minNodalVals.GetData());
+            lu.Solve(nb, 1, maxNodalVals.GetData());
             for (int i = 0; i < nb; i++)
             {
-               intminT(i*ncp2+j) = minBounds(i);
-               intmaxT(i*ncp2+j) = maxBounds(i);
+               intminT(i*ncp2+j) = minNodalVals(i);
+               intmaxT(i*ncp2+j) = maxNodalVals(i);
             }
          }
       }
@@ -617,10 +622,10 @@ void PLBound::Get3DBounds(const Vector &coeff, Vector &intmin,
          real_t w1 = intmaxT(id2++);
          for (int k = 0; k < ncp; k++) // kth slice
          {
-            vals(0) = w0*lbound(j,k);
-            vals(1) = w0*ubound(j,k);
-            vals(2) = w1*lbound(j,k);
-            vals(3) = w1*ubound(j,k);
+            vals(0) = w0*lbound(k,j);
+            vals(1) = w0*ubound(k,j);
+            vals(2) = w1*lbound(k,j);
+            vals(3) = w1*ubound(k,j);
             intmin(k*ncp2+i) += vals.Min();
             intmax(k*ncp2+i) += vals.Max();
          }
@@ -653,7 +658,8 @@ void PLBound::SetupBernsteinBasisMat(DenseMatrix &basisMat,
                                      Vector &nodesBern) const
 {
    const int nbern = nodesBern.Size();
-   L2_SegmentElement el(nbern-1, 2); // we use L2 to leverage lexicographic order
+   L2_SegmentElement el(nbern-1, 2);
+   // we use L2 to leverage lexicographic order
    Array<int> ordering = el.GetLexicographicOrdering();
    basisMat.SetSize(nbern, nbern);
    Vector shape(nbern);
@@ -666,6 +672,39 @@ void PLBound::SetupBernsteinBasisMat(DenseMatrix &basisMat,
    }
 }
 
+DenseMatrix PLBound::GetBoundingMatrix(int dim, bool is_lower) const
+{
+   if (dim > 1)
+   {
+      const int ncpd = static_cast<int>(std::pow(ncp, dim));
+      const int nbd = static_cast<int>(std::pow(nb, dim));
+      DenseMatrix boundND(ncpd, nbd);
+      Vector phimin, phimax, col;
+      Vector coeffs(nbd);
+      coeffs = 0.0;
+      for (int j = 0; j < nbd; j++)
+      {
+         coeffs(j) = 1.0;
+         boundND.GetColumnReference(j, col);
+         GetNDBounds(dim, coeffs, phimin, phimax);
+         col = is_lower ? phimin : phimax;
+         coeffs(j) = 0.0;
+      }
+      return boundND;
+   }
+   return is_lower ? lbound : ubound;
+}
+
+DenseMatrix PLBound::GetLowerBoundMatrix(int dim) const
+{
+   return GetBoundingMatrix(dim, true);
+}
+
+DenseMatrix PLBound::GetUpperBoundMatrix(int dim) const
+{
+   return GetBoundingMatrix(dim, false);
+}
+
 constexpr int PLBound::min_ncp_gl_x[2][11];
 constexpr int PLBound::min_ncp_gll_x[2][11];
 constexpr int PLBound::min_ncp_pos_x[2][11];
@@ -716,4 +755,4 @@ void PLBound::Print(std::ostream &outp) const
    ubound.Print(outp);
 }
 
-}
+}

fem/bounds.hpp

@@ -19,14 +19,18 @@ namespace mfem
 {
 
 /** @name Piecewise linear bounds of bases
-    \brief Piecewise linear bounds of bases can be used to compute bounds on the grid function in each element. The bounds for the bases are constructed based on the following parameters:
+    \brief Piecewise linear bounds of bases can be used to compute bounds on
+    the grid function in each element. The bounds for the bases are constructed
+    based on the following parameters:
 
     (i) @b nb: number of bases/nodes in 1D (i.e. polynomial order+1),
 
-    (ii) @b b_type: bases type, 0 - Lagrange interpolants on Gauss-Legendre nodes, 1 - Lagrange interpolants on Gauss-Lobatto-Legendre nodes, and
+    (ii) @b b_type: bases type, 0 - Lagrange interpolants on Gauss-Legendre
+    nodes, 1 - Lagrange interpolants on Gauss-Lobatto-Legendre nodes, and
     2 - Positive/Bernstein bases on uniformly distributed nodes,
 
-    (iii) @b ncp: number of control points used to construct the piecewise linear bounds
+    (iii) @b ncp: number of control points used to construct the piecewise
+    linear bounds
 
     (iv) @b cp_type: control point distribution. 0 - GL + end-points,
                     1 - Chebyshev.
@@ -35,7 +39,9 @@ namespace mfem
 
     If the user does not specify @b ncp and @b cp_type, the minimum value of
     @b ncp is used that would bound the bases for the @b cp_type. We default
-    to @b cp_type = 0 as it requires fewer number of points to bound the bases. Typically, @b ncp = 2 @b nb is sufficient to get fairly compact bounds, and increasing @b ncp results in tighter bounds.
+    to @b cp_type = 0 as it requires fewer number of points to bound the bases.
+    Typically, @b ncp = 2 @b nb is sufficient to get fairly compact bounds, and
+    increasing @b ncp results in tighter bounds.
 
     Finally, only tensor-product elements are currently supported.
 
@@ -54,7 +60,7 @@ private:
    bool proj = true; // Use linear projection to compute bounds.
    real_t tol = 0.0; // offset bounds to avoid round-off errors
    Vector nodes, weights, control_points;
-   DenseMatrix lbound, ubound; // nb x ncp matrices with bounds of all bases
+   DenseMatrix lbound, ubound; // ncp x nb matrices with bounds of all bases
    // Some auxillary storage for computing the bounds with Bernstein
    DenseMatrix basisMatNodes; // Bernstein bases at equispaced nodes
    DenseMatrix basisMatInt;   // Bernstein bases at GLL nodes
@@ -80,6 +86,9 @@ private:
       {3,5,8,9,11,12,13,13,14,15,16}
    };
 
+   /// Helper function to extract lower or upper bounding matrix
+   DenseMatrix GetBoundingMatrix(int dim, bool is_lower) const;
+
 public:
    // Constructor
    PLBound(const int nb_i, const int ncp_i, const int b_type_i,
@@ -92,40 +101,82 @@ public:
    PLBound(const FiniteElementSpace *fes,
            const int ncp_i = -1, const int cp_type_i = 0);
 
-   // Get minimum number of control points needed to bound the given bases
+   /// Get minimum number of control points needed to bound the given bases
    int GetMinimumPointsForGivenBases(int nb_i, int b_type_i,
                                      int cp_type_i) const;
 
-   // Print information about the bounds
+   /// Print information about the bounds
    void Print(std::ostream &outp = mfem::out) const;
 
-   // Enable (default) or disable linear projection before bounding.
-   // This projection increases the computational cost but results in tighter
-   // bounds.
+   /** @brief Enable (default) or disable linear projection before bounding.
+    *
+    *  @details This projection increases the computational cost but results in
+    *  tighter bounds.
+    */
    void SetProjectionFlagForBounding(bool proj_) { proj = proj_; }
 
-   /// Compute piecewise linear bounds for the lexicographically-ordered
-   /// coefficients in @a coeff in 1D/2D/3D.
+   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
+    *  nodal coefficients in @a coeff in 1D/2D/3D.
+    *
+    *  @param[in] rdim    The spatial dimension of the element (1, 2, or 3).
+    *  @param[in] coeff   The vector of lexicographically-ordered coefficients.
+    *                     Should be of size nb^rdim, where nb is the number of
+    *                     bases/nodes in 1D. These coefficients must correspond
+    *                     to the bases type and number of bases, used in the
+    *                     constructor of PLBound.
+    *
+    *  @param[out] intmin The vector of minimum bound for all control points.
+    *  @param[out] intmax The vector of maximum bound for all control points.
+    *                     Both intmin and intmax are of size ncp^rdim, where
+    *                     ncp is the number of control points in 1D, and are
+    *                     ordered lexicographically.
+    */
    void GetNDBounds(const int rdim, const Vector &coeff,
                     Vector &intmin, Vector &intmax) const;
 
    /// Get number of control points used to compute the bounds.
    int GetNControlPoints() const { return ncp; }
+
+   /// Get 1D control point locations (lexicographic order) in [0,1].
+   const Vector &GetControlPoints() const { return control_points; }
+
+   /** @brief Get lower and upper bounding matrix (ncp^dim x nb^dim)
+    *
+    *  @details The matrices can be used to compute the bounds at control points
+    *           by a simple matrix-vector product with the
+    *           lexicographically-ordered nodal coefficients.
+    *           The resulting output is also lexicographically-ordered.
+    *
+    *  @note These matrices do not account for the linear projection step that
+    *        is optionally done in GetNDBounds before bounding the function.
+    */
+   ///@{
+   DenseMatrix GetLowerBoundMatrix(int dim = 1) const;
+   DenseMatrix GetUpperBoundMatrix(int dim = 1) const;
+   ///@}
+
 private:
-   /// Compute piecewise linear bounds for the lexicographically-ordered
-   /// coefficients in @a coeff in 1D.
+   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
+    *  nodal coefficients in @a coeff in 1D.
+    *  See GetNDBounds for details of the input and output parameters.
+    */
    void Get1DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /// Compute piecewise linear bounds for the lexicographically-ordered
-   /// coefficients in @a coeff in 2D.
+   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
+    *  nodal coefficients in @a coeff in 2D.
+    *  See GetNDBounds for details of the input and output parameters.
+    */
    void Get2DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /// Compute piecewise linear bounds for the lexicographically-ordered
-   /// coefficients in @a coeff in 3D.
+   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
+    *  nodal coefficients in @a coeff in 3D.
+    *  See GetNDBounds for details of the input and output parameters.
+    */
    void Get3DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /// Setup matrix used to compute values at given 1D locations in [0,1]
-   /// for Bernstein bases.
+   /** @brief Setup matrix used to compute values at given 1D locations in [0,1]
+    *  for Bernstein bases.
+    */
    void SetupBernsteinBasisMat(DenseMatrix &basisMat, Vector &nodesBern) const;
 
    void Setup(const int nb_i, const int ncp_i, const int b_type_i,

fem/coefficient.cpp

@@ -1302,6 +1302,73 @@ real_t TraceCoefficient::Eval(ElementTransformation &T,
    return ma.Trace();
 }
 
+VectorComponentCoefficient::VectorComponentCoefficient(VectorCoefficient &A,
+                                                       int c)
+   : a(&A), va(A.GetVDim())
+{
+   SetComponent(c);
+}
+
+void VectorComponentCoefficient::SetComponent(int c)
+{
+   MFEM_ASSERT(c < a->GetVDim() && c >= 0,
+               "VectorComponentCoefficient:  "
+               "Index not in range.");
+
+   component = c;
+}
+
+void VectorComponentCoefficient::SetTime(real_t t)
+{
+   if (a) { a->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+real_t VectorComponentCoefficient::Eval(ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   a->Eval(va, T, ip);
+   return va[component];
+}
+
+MatrixComponentCoefficient::MatrixComponentCoefficient(MatrixCoefficient &A,
+                                                       int ri, int ci)
+   : a(&A), ma(A.GetHeight(), A.GetWidth())
+{
+   SetRowIndex(ri);
+   SetColumnIndex(ci);
+}
+
+void MatrixComponentCoefficient::SetRowIndex(int ri)
+{
+   MFEM_ASSERT(ri < a->GetHeight() && ri >= 0,
+               "MatrixComponentCoefficient:  "
+               "Row index not in range.");
+
+   row_idx = ri;
+}
+
+void MatrixComponentCoefficient::SetColumnIndex(int ci)
+{
+   MFEM_ASSERT(ci < a->GetWidth() && ci >= 0,
+               "MatrixComponentCoefficient:  "
+               "Column index not in range.");
+   col_idx = ci;
+}
+
+void MatrixComponentCoefficient::SetTime(real_t t)
+{
+   if (a) { a->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+real_t MatrixComponentCoefficient::Eval(ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   a->Eval(ma, T, ip);
+   return ma(row_idx,col_idx);
+}
+
 VectorSumCoefficient::VectorSumCoefficient(int dim)
    : VectorCoefficient(dim),
      ACoef(NULL), BCoef(NULL),

fem/coefficient.hpp

@@ -114,11 +114,10 @@ public:
    /// Construct the constant coefficient using a vector of constants.
    /** @a c should be a vector defined by attributes, so for region with
        attribute @a i @a c[i-1] is the coefficient in that region */
-   PWConstCoefficient(Vector &c)
-   { constants.SetSize(c.Size()); constants=c; }
+   PWConstCoefficient(const Vector &c) { UpdateConstants(c); }
 
    /// Update the constants with vector @a c.
-   void UpdateConstants(Vector &c) { constants.SetSize(c.Size()); constants=c; }
+   void UpdateConstants(const Vector &c) { constants = c; }
 
    /// Return a reference to the i-th constant
    real_t &operator()(int i) { return constants(i-1); }
@@ -1332,8 +1331,8 @@ public:
    /// Get the coefficient located at (i,j) in the matrix.
    Coefficient* GetCoeff (int i, int j) { return Coeff[i*width+j]; }
 
-   /** @brief Set the coefficient located at (i,j) in the matrix.  By default by
-       default this will take ownership of the Coefficient passed in, but this
+   /** @brief Set the coefficient located at (i,j) in the matrix.  By default
+       this will take ownership of the Coefficient passed in, but this
        can be overridden with the @a own parameter. */
    void Set(int i, int j, Coefficient * c, bool own=true);
 
@@ -1873,6 +1872,85 @@ public:
                const IntegrationPoint &ip) override;
 };
 
+/// Scalar coefficient defined as component of a vector coefficient
+class VectorComponentCoefficient : public Coefficient
+{
+private:
+   VectorCoefficient *a = nullptr;
+
+   mutable Vector va;
+   int component;
+
+public:
+   /// Construct with a vector coefficient.
+   VectorComponentCoefficient(VectorCoefficient &A)
+      : a(&A), va(A.GetVDim()), component(0) {};
+
+   VectorComponentCoefficient(VectorCoefficient &A, int c);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t) override;
+
+   /// Reset the vector coefficient
+   void SetACoef(VectorCoefficient &A) { a = &A; }
+
+   /// Return the vector coefficient
+   VectorCoefficient * GetACoef() const { return a; }
+
+   /// Set the component
+   void SetComponent(int c);
+
+   /// Return the component
+   int GetComponent() const { return component; }
+
+   /// Evaluate the trace coefficient at @a ip.
+   real_t Eval(ElementTransformation &T,
+               const IntegrationPoint &ip) override;
+};
+
+/// Scalar coefficient defined as component of a matrix coefficient
+class MatrixComponentCoefficient : public Coefficient
+{
+private:
+   MatrixCoefficient *a = nullptr;
+
+   mutable DenseMatrix ma;
+   int row_idx,col_idx;
+
+public:
+   MatrixComponentCoefficient(MatrixCoefficient &A)
+      : a(&A), ma(A.GetHeight(), A.GetWidth()), row_idx(0), col_idx(0) {};
+
+   /// Construct with the matrix coefficient.
+   MatrixComponentCoefficient(MatrixCoefficient &A, int ri, int ci);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t) override;
+
+   /// Reset the matrix coefficient
+   void SetACoef(MatrixCoefficient &A) { a = &A; }
+
+   /// Return the matrix coefficient
+   MatrixCoefficient * GetACoef() const { return a; }
+
+   /// Reset the index
+   void SetRowIndex(int ri);
+
+   /// Return the index
+   int GetRowIndex() const { return row_idx; }
+
+   /// Reset the index
+   void SetColumnIndex(int ci);
+
+   /// Return the index
+   int GetColumnIndex() const { return col_idx; }
+
+
+   /// Evaluate the trace coefficient at @a ip.
+   real_t Eval(ElementTransformation &T,
+               const IntegrationPoint &ip) override;
+};
+
 /// Vector coefficient defined as the linear combination of two vectors
 class VectorSumCoefficient : public VectorCoefficient
 {

fem/dfem/doperator.hpp

@@ -84,29 +84,33 @@ public:
       const std::vector<derivative_action_t> &derivative_actions,
       const FieldDescriptor &direction,
       const int &daction_l_size,
-      const std::vector<derivative_action_t> &derivative_actions_transpose,
+      const int &derivative_action_tr_l_size,
+      const std::vector<derivative_action_t> &derivative_tr_actions,
       const FieldDescriptor &transpose_direction,
       const int &daction_transpose_l_size,
       const std::vector<Vector *> &solutions_l,
       const std::vector<Vector *> &parameters_l,
       const restriction_callback_t &restriction_callback,
       const std::function<void(Vector &, Vector &)> &prolongation_transpose,
+      const std::function<void(Vector &, Vector &)> &tr_prolongation_transpose,
       const std::vector<assemble_derivative_sparsematrix_callback_t>
       &assemble_derivative_sparsematrix_callbacks,
-      const std::vector<assemble_derivative_hypreparmatrix_callback_t>
-      &assemble_derivative_hypreparmatrix_callbacks) :
+      const assemble_derivative_hypreparmatrix_callback_t
+      &assemble_derivative_hypreparmatrix_callback) :
       Operator(height, width),
       derivative_actions(derivative_actions),
       direction(direction),
       daction_l(daction_l_size),
       daction_l_size(daction_l_size),
-      derivative_actions_transpose(derivative_actions_transpose),
+      derivative_action_tr_l_size(derivative_action_tr_l_size),
+      derivative_tr_actions(derivative_tr_actions),
       transpose_direction(transpose_direction),
       prolongation_transpose(prolongation_transpose),
+      tr_prolongation_transpose(tr_prolongation_transpose),
       assemble_derivative_sparsematrix_callbacks(
          assemble_derivative_sparsematrix_callbacks),
-      assemble_derivative_hypreparmatrix_callbacks(
-         assemble_derivative_hypreparmatrix_callbacks)
+      assemble_derivative_hypreparmatrix_callback(
+         assemble_derivative_hypreparmatrix_callback)
    {
       std::vector<Vector> s_l(solutions_l.size());
       for (size_t i = 0; i < s_l.size(); i++)
@@ -156,18 +160,18 @@ public:
    /// direction_t on T-dofs.
    void MultTranspose(const Vector &direction_t, Vector &result_t) const override
    {
-      MFEM_ASSERT(!derivative_actions_transpose.empty(),
+      MFEM_ASSERT(!derivative_tr_actions.empty(),
                   "derivative can't be used to be multiplied in transpose mode");
 
-      daction_l.SetSize(width);
+      daction_l.SetSize(derivative_action_tr_l_size);
       daction_l = 0.0;
 
       prolongation(transpose_direction, direction_t, direction_l);
-      for (const auto &f : derivative_actions_transpose)
+      for (const auto &f : derivative_tr_actions)
       {
          f(fields_e, direction_l, daction_l);
       }
-      prolongation_transpose(daction_l, result_t);
+      tr_prolongation_transpose(daction_l, result_t);
    };
 
    /// @brief Assemble the derivative operator into a SparseMatrix.
@@ -183,6 +187,10 @@ public:
       {
          f(fields_e, A);
       }
+
+      // SparseMatrix A is finalized after all callbacks have contributed to
+      // it.
+      A->Finalize();
    }
 
    /// @brief Assemble the derivative operator into a HypreParMatrix.
@@ -191,13 +199,7 @@ public:
    /// be an uninitialized object.
    void Assemble(HypreParMatrix *&A)
    {
-      MFEM_ASSERT(!assemble_derivative_hypreparmatrix_callbacks.empty(),
-                  "derivative can't be assembled into a HypreParMatrix");
-
-      for (const auto &f : assemble_derivative_hypreparmatrix_callbacks)
-      {
-         f(fields_e, A);
-      }
+      assemble_derivative_hypreparmatrix_callback(fields_e, A);
    }
 
 private:
@@ -212,10 +214,12 @@ private:
 
    const int daction_l_size;
 
+   const int derivative_action_tr_l_size;
+
    /// Transpose Derivative action callbacks. Depending on the requested
    /// derivatives in DifferentiableOperator the callbacks represent certain
    /// combinations of actions of derivatives of the forward operator.
-   std::vector<derivative_action_t> derivative_actions_transpose;
+   std::vector<derivative_action_t> derivative_tr_actions;
 
    FieldDescriptor transpose_direction;
 
@@ -225,13 +229,15 @@ private:
 
    std::function<void(Vector &, Vector &)> prolongation_transpose;
 
+   std::function<void(Vector &, Vector &)> tr_prolongation_transpose;
+
    /// Callbacks that assemble derivatives into a SparseMatrix.
    std::vector<assemble_derivative_sparsematrix_callback_t>
    assemble_derivative_sparsematrix_callbacks;
 
    /// Callbacks that assemble derivatives into a HypreParMatrix.
-   std::vector<assemble_derivative_hypreparmatrix_callback_t>
-   assemble_derivative_hypreparmatrix_callbacks;
+   assemble_derivative_hypreparmatrix_callback_t
+   assemble_derivative_hypreparmatrix_callback;
 };
 
 /// Class representing a differentiable operator which acts on solution and
@@ -457,7 +463,10 @@ public:
          dir_l = s_l[derivative_idx];
       }
 
-      derivative_setup_callbacks[derivative_id][0](fields_e, dir_l);
+      for (size_t i = 0; i < derivative_setup_callbacks[derivative_id].size(); i++)
+      {
+         derivative_setup_callbacks[derivative_id][i](fields_e, dir_l);
+      }
 
       return std::make_shared<DerivativeOperator>(
                 height,
@@ -465,15 +474,17 @@ public:
                 derivative_action_callbacks[derivative_id],
                 fields[derivative_idx],
                 residual_l.Size(),
-                daction_transpose_callbacks[derivative_id],
+                derivative_action_tr_l_size[derivative_id],
+                derivative_action_tr_callbacks[derivative_id],
                 fields[test_space_field_idx],
                 GetVSize(fields[test_space_field_idx]),
                 sol_l,
                 par_l,
                 restriction_callback,
                 prolongation_transpose,
+                derivative_tr_prolongation_transpose[derivative_id],
                 assemble_derivative_sparsematrix_callbacks[derivative_id],
-                assemble_derivative_hypreparmatrix_callbacks[derivative_id]);
+                assemble_derivative_hypreparmatrix_callback[derivative_id]);
    }
 
 private:
@@ -486,13 +497,15 @@ private:
    std::map<size_t,
        std::vector<derivative_action_t>> derivative_action_callbacks;
    std::map<size_t,
-       std::vector<derivative_action_t>> daction_transpose_callbacks;
+       std::vector<derivative_action_t>> derivative_action_tr_callbacks;
+   std::map<size_t,
+       std::function<void(Vector &, Vector &)>> derivative_tr_prolongation_transpose;
+   std::map<size_t, int> derivative_action_tr_l_size;
    std::map<size_t,
        std::vector<assemble_derivative_sparsematrix_callback_t>>
        assemble_derivative_sparsematrix_callbacks;
-   std::map<size_t,
-       std::vector<assemble_derivative_hypreparmatrix_callback_t>>
-       assemble_derivative_hypreparmatrix_callbacks;
+   std::map<size_t, assemble_derivative_hypreparmatrix_callback_t>
+   assemble_derivative_hypreparmatrix_callback;
 
    std::vector<FieldDescriptor> solutions;
    std::vector<FieldDescriptor> parameters;
@@ -510,7 +523,7 @@ private:
    std::function<void(Vector &, Vector &)> output_restriction_transpose;
    restriction_callback_t restriction_callback;
 
-   std::map<size_t, Vector> derivative_qp_caches;
+   std::map<size_t, std::vector<Vector>> derivative_qp_caches;
 
    std::map<size_t, size_t> assembled_vector_sizes;
 
@@ -769,9 +782,10 @@ void DifferentiableOperator::AddIntegrator(
    auto input_size_on_qp =
       get_input_size_on_qp(inputs, std::make_index_sequence<num_inputs> {});
 
+   // printf("calculate shmem action info\n");
    auto action_shmem_info =
       get_shmem_info<entity_t, num_fields, num_inputs, num_outputs>
-      (input_dtq_maps, output_dtq_maps, fields, num_entities, inputs, num_qp,
+      (input_dtq_maps, output_dtq_maps, fields, num_entities, num_qp,
        input_size_on_qp, residual_size_on_qp, element_dof_ordering);
 
    Vector shmem_cache(action_shmem_info.total_size);
@@ -894,9 +908,10 @@ void DifferentiableOperator::AddIntegrator(
          const int da_size_on_qp =
             GetSizeOnQP<entity_t>(output_fop, fields[test_space_field_idx]);
 
+         // printf("calculate shmem derivative action info\n");
          auto shmem_info =
             get_shmem_info<entity_t, num_fields, num_inputs, num_outputs>(
-               input_dtq_maps, output_dtq_maps, fields, num_entities, inputs,
+               input_dtq_maps, output_dtq_maps, fields, num_entities,
                num_qp, input_size_on_qp, residual_size_on_qp,
                element_dof_ordering, d_field_idx);
 
@@ -968,9 +983,12 @@ void DifferentiableOperator::AddIntegrator(
          // Quadrature point local derivative cache for each element, with data
          // layout:
          // [test_vdim, test_op_dim, trial_vdim, trial_op_dim, qp, num_entities].
-         derivative_qp_caches[derivative_id] = Vector(test_vdim * test_op_dim *
-                                                      trial_vdim *
-                                                      total_trial_op_dim * num_qp * num_entities);
+         derivative_qp_caches[derivative_id].push_back(
+            Vector(test_vdim * test_op_dim * trial_vdim * total_trial_op_dim * num_qp *
+                   num_entities));
+
+         const int cache_index = this->derivative_qp_caches[derivative_id].size() - 1;
+
          // Create local references for MSVC lambda capture compatibility
          auto& fields_ref = this->fields;
          auto& derivative_qp_caches_ref = this->derivative_qp_caches[derivative_id];
@@ -1012,6 +1030,8 @@ void DifferentiableOperator::AddIntegrator(
                trial_vdim,
                inputs_trial_op_dim,
 
+               qpdc_idx = cache_index,
+
                // capture by ref:
                &qpdc_mem = derivative_qp_caches_ref
             ](std::vector<Vector> &f_e, const Vector &dir_l) mutable
@@ -1024,7 +1044,7 @@ void DifferentiableOperator::AddIntegrator(
                                                shmem_info.direction_size,
                                                num_entities);
 
-            auto qpdc = Reshape(qpdc_mem.ReadWrite(), test_vdim, test_op_dim,
+            auto qpdc = Reshape(qpdc_mem[qpdc_idx].ReadWrite(), test_vdim, test_op_dim,
                                 trial_vdim, total_trial_op_dim, num_qp, num_entities);
 
             auto itod = Reshape(inputs_trial_op_dim.Read(), num_inputs);
@@ -1095,6 +1115,7 @@ void DifferentiableOperator::AddIntegrator(
                inputs_trial_op_dim,
                total_trial_op_dim,
                trial_vdim,
+               qpdc_idx = cache_index,
                // capture by ref:
                &qpdc_mem = derivative_qp_caches_ref,
                &or_transpose
@@ -1112,7 +1133,7 @@ void DifferentiableOperator::AddIntegrator(
                                                shmem_info.direction_size,
                                                num_entities);
 
-            auto qpdc = Reshape(qpdc_mem.Read(), test_vdim, test_op_dim,
+            auto qpdc = Reshape(qpdc_mem[qpdc_idx].Read(), test_vdim, test_op_dim,
                                 trial_vdim, total_trial_op_dim, num_qp, num_entities);
 
             auto itod = Reshape(inputs_trial_op_dim.Read(), num_inputs);
@@ -1157,6 +1178,154 @@ void DifferentiableOperator::AddIntegrator(
             or_transpose(derivative_action_e, der_action_l);
          });
 
+         // This prevents Sum/Identity. These are invalid
+         // as input FieldOperators anyways.
+         constexpr auto dummy_fop = Value<0> {};
+
+         auto [input_rt,
+               input_e_sz] = get_restriction_transpose<entity_t>
+                             (fields[d_field_idx],
+                              element_dof_ordering, dummy_fop);
+
+         const auto input_restriction_transpose = input_rt;
+
+         derivative_tr_prolongation_transpose[derivative_id] =
+            get_prolongation_transpose(
+               fields[d_field_idx], dummy_fop, mesh.GetComm());
+
+         const auto d_tr_field_idx = test_space_field_idx;
+         const auto direction_tr = fields[d_tr_field_idx];
+
+         auto output_size_on_qp =
+            get_input_size_on_qp(outputs, std::make_index_sequence<num_outputs> {});
+
+         const int residual_tr_size_on_qp = trial_vdim * total_trial_op_dim;
+
+         auto shmem_tr_info =
+            get_shmem_info<entity_t, num_fields, num_outputs, num_inputs>(
+               output_dtq_maps, input_dtq_maps, fields, num_entities,
+               num_qp, output_size_on_qp, residual_tr_size_on_qp,
+               element_dof_ordering, test_space_field_idx);
+
+         // print_shared_memory_info(shmem_tr_info);
+
+         // TODO: this is a hack to extend the shared memory with a known
+         // offset for a temp variable
+         Vector shmem_tr_cache(shmem_tr_info.total_size + residual_tr_size_on_qp *
+                               num_qp);
+
+         Vector direction_tr_e(get_restriction<entity_t>(
+                                  fields[test_space_field_idx],
+                                  element_dof_ordering)->Height());
+
+         derivative_action_tr_l_size[derivative_id] =
+            get_restriction<entity_t>(fields[d_field_idx],
+                                      element_dof_ordering)->Width();
+
+         Vector derivative_action_tr_e(input_e_sz);
+         derivative_action_tr_e = 0.0;
+
+         derivative_action_tr_callbacks[derivative_id].push_back(
+            [
+               // capture by copy:
+               dimension,             // int
+               num_entities,          // int
+               num_trial_dof,         // int
+               num_qp,                // int
+               q1d,                   // int
+               test_vdim,             // int (= output_fop.vdim)
+               test_op_dim,           // int (derived from output_fop)
+               inputs,                // mfem::future::tuple
+               outputs,               // mfem::future::tuple
+               attributes,            // Array<int>
+               ir_weights,            // DeviceTensor
+               use_sum_factorization, // bool
+               input_dtq_maps,        // std::array<DofToQuadMap, num_fields>
+               output_dtq_maps,       // std::array<DofToQuadMap, num_fields>
+               // output_fop,            // class derived from FieldOperator
+               thread_blocks,         // ThreadBlocks
+               shmem_tr_cache,           // Vector (local)
+               shmem_tr_info,            // SharedMemoryInfo
+               // TODO: make this Array<int> a member of the DifferentiableOperator
+               //       and capture it by ref.
+               elem_attributes,       // Array<int>
+
+               input_is_dependent,
+               direction_tr,             // FieldDescriptor
+               direction_tr_e,           // Vector
+               derivative_action_tr_e,   // Vector
+               element_dof_ordering,  // ElementDofOrdering
+               inputs_trial_op_dim,
+               total_trial_op_dim,
+               trial_vdim,
+               input_restriction_transpose,
+               qpdc_idx = cache_index,
+               // capture by ref:
+               &qpdc_mem = derivative_qp_caches_ref
+            ](
+               std::vector<Vector> &f_e, const Vector &dir_tr_l,
+               Vector &derivative_action_tr_l) mutable
+         {
+            restriction<entity_t>(direction_tr, dir_tr_l, direction_tr_e,
+                                  element_dof_ordering);
+            auto ye = Reshape(derivative_action_tr_e.ReadWrite(), num_trial_dof,
+                              trial_vdim, num_entities);
+            auto wrapped_fields_e = wrap_fields(f_e, shmem_tr_info.field_sizes,
+                                                num_entities);
+            auto wrapped_direction_e = Reshape(direction_tr_e.ReadWrite(),
+                                               shmem_tr_info.direction_size,
+                                               num_entities);
+
+            auto qpdc = Reshape(qpdc_mem[qpdc_idx].Read(), test_vdim, test_op_dim,
+                                trial_vdim, total_trial_op_dim, num_qp, num_entities);
+
+            auto itod = Reshape(inputs_trial_op_dim.Read(), num_inputs);
+
+            const bool has_attr = attributes.Size() > 0;
+            const auto d_attr = attributes.Read();
+            const auto d_elem_attr = elem_attributes->Read();
+
+            derivative_action_tr_e = 0.0;
+            forall([=] MFEM_HOST_DEVICE (int e, real_t *shmem)
+            {
+               if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+               auto [output_dtq_shmem, input_dtq_shmem, fields_shmem,
+                                       direction_shmem, input_shmem,
+                                       shadow_shmem_, residual_shmem,
+                                       scratch_shmem] =
+                        unpack_shmem(shmem, shmem_tr_info, output_dtq_maps, input_dtq_maps,
+                                     wrapped_fields_e, wrapped_direction_e, num_qp, e);
+               auto &shadow_shmem = shadow_shmem_;
+
+               std::array<bool, num_outputs> all_true{true};
+               map_direction_to_quadrature_data_conditional(
+                  shadow_shmem, direction_shmem, output_dtq_shmem, outputs,
+                  ir_weights, scratch_shmem, all_true, dimension,
+                  use_sum_factorization);
+
+               auto fhat = Reshape(&residual_shmem(0, 0), trial_vdim,
+                                   total_trial_op_dim, num_qp);
+
+               auto qpdce = Reshape(&qpdc(0, 0, 0, 0, 0, e), test_vdim, test_op_dim,
+                                    trial_vdim, total_trial_op_dim, num_qp);
+
+               constexpr bool transpose = true;
+               apply_qpdc(fhat, shadow_shmem, qpdce, itod, q1d, dimension,
+                          use_sum_factorization, transpose);
+
+               auto y = Reshape(&ye(0, 0, e), num_trial_dof, trial_vdim);
+               auto fi_shmem = Reshape(shmem + shmem_tr_info.total_size, trial_vdim,
+                                       total_trial_op_dim, num_qp);
+
+               map_quadrature_data_to_fields_conditional(
+                  y, fhat, inputs, itod, input_dtq_shmem, scratch_shmem, fi_shmem,
+                  input_is_dependent, dimension, use_sum_factorization);
+            }, num_entities, thread_blocks, shmem_tr_info.total_size,
+            shmem_tr_cache.ReadWrite());
+            input_restriction_transpose(derivative_action_tr_e, derivative_action_tr_l);
+         });
+
          assemble_derivative_sparsematrix_callbacks[derivative_id].push_back(
             [
                // capture by copy:
@@ -1190,7 +1359,7 @@ void DifferentiableOperator::AddIntegrator(
                inputs_trial_op_dim,
                Ae_mem,
                output_to_field,
-
+               qpdc_idx = cache_index,
                // capture by ref:
                &qpdc_mem = derivative_qp_caches_ref,
                &fields = fields_ref
@@ -1202,7 +1371,7 @@ void DifferentiableOperator::AddIntegrator(
                                                shmem_info.direction_size,
                                                num_entities);
 
-            auto qpdc = Reshape(qpdc_mem.Read(), test_vdim, test_op_dim,
+            auto qpdc = Reshape(qpdc_mem[qpdc_idx].Read(), test_vdim, test_op_dim,
                                 trial_vdim, total_trial_op_dim, num_qp, num_entities);
 
             auto itod = Reshape(inputs_trial_op_dim.Read(), num_inputs);
@@ -1250,7 +1419,10 @@ void DifferentiableOperator::AddIntegrator(
             auto test_fes = *std::get_if<const ParFiniteElementSpace *>
                             (&fields[output_to_field[0]].data);
 
-            A = new SparseMatrix(test_fes->GetVSize(), trial_fes->GetVSize());
+            if (A == nullptr)
+            {
+               A = new SparseMatrix(test_fes->GetVSize(), trial_fes->GetVSize());
+            }
 
             auto tmp = Reshape(Ae_mem.HostReadWrite(), num_test_dof * test_vdim,
                                num_trial_dof * trial_vdim, num_entities);
@@ -1321,14 +1493,15 @@ void DifferentiableOperator::AddIntegrator(
                   A->AddSubMatrix(test_vdofs, trial_vdofs, Aee, 1);
                }
             }
-            A->Finalize();
+            // Don't finalize here since multiple callbacks might contribute to the same matrix
+            // A->Finalize() will be called after all callbacks have contributed
          });
 
          // Create local references for MSVC lambda capture compatibility
          auto& assemble_derivative_sparsematrix_callbacks_ref =
             this->assemble_derivative_sparsematrix_callbacks[derivative_id];
 
-         assemble_derivative_hypreparmatrix_callbacks[derivative_id].push_back(
+         assemble_derivative_hypreparmatrix_callback[derivative_id] =
             [
                input_is_dependent,
                input_to_field,
@@ -1342,6 +1515,7 @@ void DifferentiableOperator::AddIntegrator(
             {
                f(f_e, spmat);
             }
+            spmat->Finalize();
 
             if (spmat == nullptr)
             {
@@ -1395,7 +1569,7 @@ void DifferentiableOperator::AddIntegrator(
                        trial_fes->Dof_TrueDof_Matrix());
             }
             delete spmat;
-         });
+         };
       }, derivative_ids);
    }
 }

fem/dfem/integrate.hpp

@@ -530,4 +530,65 @@ void map_quadrature_data_to_fields(
    }
 }
 
+template <size_t N, typename field_operator_ts>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_conditional(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const field_operator_ts &fops,
+   const DeviceTensor<1, const real_t> &op_dims,
+   const std::array<DofToQuadMap, N> &dtqmaps,
+   std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const DeviceTensor<3> &fi_shmem,
+   const std::array<bool, N> &conditions,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   int offset = 0;
+   for_constexpr<N>([&](auto i)
+   {
+      if (conditions[i])
+      {
+         [[maybe_unused]] const auto [K, unused, M] = f.GetShape();
+         const int L = static_cast<int>(op_dims(static_cast<size_t>(i)));
+         auto fi = Reshape(&fi_shmem(0, 0, 0), K, L, M);
+         for (int k = 0; k < K; k++)
+         {
+            for (int l = 0; l < L; l++)
+            {
+               for (int m = 0; m < M; m++)
+               {
+                  fi(k, l, m) = f(k, l + offset, m);
+               }
+            }
+         }
+
+         if (use_sum_factorization)
+         {
+            if (dimension == 1)
+            {
+               map_quadrature_data_to_fields_tensor_impl_1d(
+                  y, fi, get<i>(fops), dtqmaps[i], scratch_mem);
+            }
+            else if (dimension == 2)
+            {
+               map_quadrature_data_to_fields_tensor_impl_2d(
+                  y, fi, get<i>(fops), dtqmaps[i], scratch_mem);
+            }
+            else if (dimension == 3)
+            {
+               map_quadrature_data_to_fields_tensor_impl_3d(
+                  y, fi, get<i>(fops), dtqmaps[i], scratch_mem);
+            }
+            else { MFEM_ABORT_KERNEL("dimension not supported"); }
+         }
+         else
+         {
+            map_quadrature_data_to_fields_impl(y, fi, get<i>(fops), dtqmaps[i]);
+         }
+         offset += L;
+      }
+   });
+}
+
 } // namespace mfem::future

fem/dfem/interpolate.hpp

@@ -505,13 +505,13 @@ void map_field_to_quadrature_data(
    }
 }
 
-template <typename field_operator_ts, size_t num_inputs, size_t num_fields>
+template <typename field_operator_ts, size_t N, size_t M>
 MFEM_HOST_DEVICE inline
 void map_fields_to_quadrature_data(
-   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
-   const std::array<DeviceTensor<1>, num_fields> &fields_e,
-   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
-   const std::array<size_t, num_inputs> &input_to_field,
+   std::array<DeviceTensor<2>, N> &fields_qp,
+   const std::array<DeviceTensor<1>, M> &fields_e,
+   const std::array<DofToQuadMap, N> &dtqmaps,
+   const std::array<size_t, N> &input_to_field,
    const field_operator_ts &fops,
    const DeviceTensor<1, const real_t> &integration_weights,
    const std::array<DeviceTensor<1>, 6> &scratch_mem,
@@ -523,7 +523,7 @@ void map_fields_to_quadrature_data(
    // attached to them and we create a dummy field which is not accessed
    // inside the functions it is passed to.
    const auto dummy_field_weight = DeviceTensor<1>(nullptr, 0);
-   for_constexpr<num_inputs>([&](auto i)
+   for_constexpr<N>([&](auto i)
    {
       const DeviceTensor<1> &field_e =
          (input_to_field[i] == SIZE_MAX) ? dummy_field_weight :
@@ -549,12 +549,7 @@ void map_fields_to_quadrature_data(
                fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
                integration_weights, scratch_mem);
          }
-         else
-         {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-            MFEM_ABORT("unsupported dimension");
-#endif
-         }
+         else { MFEM_ABORT_KERNEL("unsupported dimension"); }
       }
       else
       {
@@ -627,20 +622,20 @@ void map_fields_to_quadrature_data_conditional(
    });
 }
 
-template <size_t num_inputs, typename field_operator_ts>
+template <size_t N, typename field_operator_ts>
 MFEM_HOST_DEVICE
 void map_direction_to_quadrature_data_conditional(
-   std::array<DeviceTensor<2>, num_inputs> &directions_qp,
+   std::array<DeviceTensor<2>, N> &directions_qp,
    const DeviceTensor<1> &direction_e,
-   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
+   const std::array<DofToQuadMap, N> &dtqmaps,
    field_operator_ts fops,
    const DeviceTensor<1, const real_t> &integration_weights,
    const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const std::array<bool, num_inputs> &conditions,
+   const std::array<bool, N> &conditions,
    const int &dimension,
    const bool &use_sum_factorization)
 {
-   for_constexpr<num_inputs>([&](auto i)
+   for_constexpr<N>([&](auto i)
    {
       if (conditions[i])
       {

fem/dfem/parameterspace.hpp

@@ -82,7 +82,7 @@ protected:
 };
 
 /// @brief Uniform parameter space
-class UniformParameterSpace : public ParameterSpace
+class UniformParameterSpace final : public ParameterSpace
 {
 public:
    /// @brief Constructor for a uniform parameter space

fem/dfem/qfunction_apply.hpp

@@ -379,54 +379,102 @@ namespace detail
 /// @param shadow_shmem the shadow shared memory.
 /// @param qpdc the quadrature point data cache holding the resulting
 /// Jacobians on each quadrature point.
-/// @param itod inputs trial operator dimension.
-/// If input is dependent the value corresponds to the spatial dimension, otherwise
-/// a zero indicates non-dependence on the variable.
+/// @param op_dims operator dimensions.
+/// If an operator is dependent, the value corresponds to the spatial dimension.
+/// Otherwise a zero indicates indepence on the variable.
 /// @param q the current quadrature point index.
-template <size_t num_fields>
+/// @param transpose switch to use transpose action.
+template <size_t N>
 MFEM_HOST_DEVICE inline
 void apply_qpdc(
    DeviceTensor<3> &fhat,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   const std::array<DeviceTensor<2>, N> &shadow_shmem,
    const DeviceTensor<5, const real_t> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &q)
+   const DeviceTensor<1, const real_t> &op_dims,
+   const int &q,
+   bool transpose)
 {
+   const size_t num_ops = op_dims.GetShape()[0];
+
    const int test_vdim = qpdc.GetShape()[0];
    const int test_op_dim = qpdc.GetShape()[1];
    const int trial_vdim = qpdc.GetShape()[2];
-   const int num_qp = qpdc.GetShape()[4];
-   const size_t num_inputs = itod.GetShape()[0];
+   const int total_trial_op_dim = qpdc.GetShape()[3];
 
-   for (int i = 0; i < test_vdim; i++)
+   const int num_qp = qpdc.GetShape()[4];
+
+   if (transpose)
    {
-      for (int k = 0; k < test_op_dim; k++)
+      for (int j = 0; j < trial_vdim; j++)
       {
-         real_t sum = 0.0;
-         int m_offset = 0;
-         for (size_t s = 0; s < num_inputs; s++)
+         for (int m = 0; m < total_trial_op_dim; m++)
          {
-            const int trial_op_dim = static_cast<int>(itod(s));
-            if (trial_op_dim == 0)
-            {
-               continue;
-            }
-            const auto d_qp =
-               Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
-            for (int j = 0; j < trial_vdim; j++)
-            {
-               for (int m = 0; m < trial_op_dim; m++)
-               {
-                  sum += qpdc(i, k, j, m + m_offset, q) * d_qp(j, m, q);
-               }
-            }
-            m_offset += trial_op_dim;
+            fhat(j, m, q) = 0.0;
+         }
+      }
+
+      // Since we don't support more than output space right now
+      // shadow_shmem will always be of size 1.
+      constexpr int shadow_idx_tr = 0;
+      auto d_qp = Reshape(&(shadow_shmem[shadow_idx_tr])[0], test_vdim, test_op_dim,
+                          num_qp);
+
+      int m_offset = 0;
+      for (size_t s = 0; s < num_ops; s++)
+      {
+         const int trial_op_dim = static_cast<int>(op_dims(s));
+         if (trial_op_dim == 0) { continue; }
+
+         for (int j = 0; j < trial_vdim; j++)
+         {
+            for (int m = 0; m < trial_op_dim; m++)
+            {
+               real_t sum = 0.0;
+               for (int i = 0; i < test_vdim; i++)
+               {
+                  for (int k = 0; k < test_op_dim; k++)
+                  {
+                     const real_t contrib = qpdc(i, k, j, m + m_offset, q) * d_qp(i, k, q);
+                     sum += contrib;
+                  }
+               }
+               fhat(j, m + m_offset, q) += sum;
+            }
+         }
+         m_offset += trial_op_dim;
+      }
+   }
+   else
+   {
+      for (int i = 0; i < test_vdim; i++)
+      {
+         for (int k = 0; k < test_op_dim; k++)
+         {
+            real_t sum = 0.0;
+            int m_offset = 0;
+            for (size_t s = 0; s < num_ops; s++)
+            {
+               const int trial_op_dim = static_cast<int>(op_dims(s));
+               if (trial_op_dim == 0) { continue; }
+
+               const auto d_qp =
+                  Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
+               for (int j = 0; j < trial_vdim; j++)
+               {
+                  for (int m = 0; m < trial_op_dim; m++)
+                  {
+                     sum += qpdc(i, k, j, m + m_offset, q) * d_qp(j, m, q);
+                  }
+               }
+               m_offset += trial_op_dim;
+            }
+            fhat(i, k, q) = sum;
          }
-         fhat(i, k, q) = sum;
       }
    }
 }
-}
+
+} // namespace detail
 
 /// @brief Apply the quadrature point data cache (qpdc) to a vector
 /// (usually a direction).
@@ -445,16 +493,18 @@ void apply_qpdc(
 /// @param q1d number of quadrature points in 1D.
 /// @param dimension spatial dimension.
 /// @param use_sum_factorization whether to use sum factorization.
-template <size_t num_fields>
+/// @param T switch to use transpose application.
+template <size_t N>
 MFEM_HOST_DEVICE inline
 void apply_qpdc(
    DeviceTensor<3> &fhat,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   const std::array<DeviceTensor<2>, N> &shadow_shmem,
    const DeviceTensor<5, const real_t> &qpdc,
    const DeviceTensor<1, const real_t> &itod,
    const int &q1d,
    const int &dimension,
-   const bool &use_sum_factorization)
+   const bool &use_sum_factorization,
+   const bool T = false)
 {
    if (use_sum_factorization)
    {
@@ -462,7 +512,7 @@ void apply_qpdc(
       {
          MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
          {
-            detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+            detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q, T);
          }
       }
       else if (dimension == 2)
@@ -472,7 +522,7 @@ void apply_qpdc(
             MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
             {
                const int q = qx + q1d * qy;
-               detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+               detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q, T);
             }
          }
       }
@@ -485,7 +535,7 @@ void apply_qpdc(
                MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
                {
                   const int q = qx + q1d * (qy + q1d * qz);
-                  detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+                  detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q, T);
                }
             }
          }
@@ -500,7 +550,7 @@ void apply_qpdc(
       const int num_qp = qpdc.GetShape()[4];
       MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
       {
-         detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+         detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q, T);
       }
    }
 }

fem/dfem/qfunction_transform.hpp

@@ -243,6 +243,35 @@ void process_qf_arg(
    }
 }
 
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   const DeviceTensor<1, T> &v,
+   tensor<T, n> &arg)
+{
+   for (int i = 0; i < n; i++)
+   {
+      arg(i) = u(i);
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   const DeviceTensor<1, T> &v,
+   tensor<T, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i) = u((i * n) + j);
+      }
+   }
+}
+
 template <typename arg_type>
 MFEM_HOST_DEVICE inline
 void process_qf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)
@@ -327,20 +356,4 @@ void process_qf_result(
    }
 }
 
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   const DeviceTensor<1, T> &v,
-   tensor<T, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i) = u((i * n) + j);
-      }
-   }
-}
-
 } // namespace mfem::future

fem/dfem/tuple.hpp

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 #pragma once
 
-// This is serac's tuple implementation
+// This is smith's tuple implementation
 
 #include <ostream>
 #include "../../config/config.hpp"

fem/dfem/util.hpp

@@ -1202,7 +1202,14 @@ std::function<void(const Vector&, Vector&)> get_prolongation_transpose(
    const Operator *P = get_prolongation(f);
    auto PT = [=](const Vector &r_local, Vector &y)
    {
-      P->MultTranspose(r_local, y);
+      if (P)
+      {
+         P->MultTranspose(r_local, y);
+      }
+      else
+      {
+         y = r_local;
+      }
    };
    return PT;
 }
@@ -1580,14 +1587,13 @@ struct SharedMemoryInfo
    std::array<int, 6> temp_sizes;
 };
 
-template <typename entity_t, std::size_t num_fields, std::size_t num_inputs, std::size_t num_outputs, typename input_t>
+template <typename entity_t, std::size_t num_fields, std::size_t num_inputs, std::size_t num_outputs>
 SharedMemoryInfo<num_fields, num_inputs, num_outputs>
 get_shmem_info(
    const std::array<DofToQuadMap, num_inputs> &input_dtq_maps,
    const std::array<DofToQuadMap, num_outputs> &output_dtq_maps,
    const std::vector<FieldDescriptor> &fields,
    const int &num_entities,
-   const input_t &inputs,
    const int &num_qp,
    const std::vector<int> &input_size_on_qp,
    const int &residual_size_on_qp,

fem/dgmassinv.cpp

@@ -12,7 +12,6 @@
 #include "dgmassinv.hpp"
 #include "bilinearform.hpp"
 #include "dgmassinv_kernels.hpp"
-#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -119,151 +118,6 @@ void DGMassInverse::Update()
 
 DGMassInverse::~DGMassInverse() = default;
 
-template<int DIM, int D1D, int Q1D>
-void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
-{
-   using namespace internal; // host/device kernel functions
-
-   const int NE = fes.GetNE();
-   const int d1d = m->dofs1D;
-   const int q1d = m->quad1D;
-
-   const int ND = static_cast<int>(pow(d1d, DIM));
-
-   const auto B = m->maps->B.Read();
-   const auto Bt = m->maps->Bt.Read();
-   const auto pa_data = m->pa_data.Read();
-   const auto dinv = diag_inv.Read();
-   auto r = r_.Write();
-   auto d = d_.Write();
-   auto z = z_.Write();
-   auto u = u_.ReadWrite();
-
-   const real_t RELTOL = rel_tol;
-   const real_t ABSTOL = abs_tol;
-   const int MAXIT = max_iter;
-   const bool IT_MODE = iterative_mode;
-   const bool CHANGE_BASIS = (d2q != nullptr);
-
-   // b is the right-hand side (if no change of basis, this just points to the
-   // incoming RHS vector, if we have to change basis, this points to the
-   // internal b2 vector where we put the transformed RHS)
-   const real_t *b;
-   // the following are non-null if we have to change basis
-   real_t *b2 = nullptr; // non-const access to b2
-   const real_t *b_orig = nullptr; // RHS vector in "original" basis
-   const real_t *d2q_B = nullptr; // matrix to transform initial guess
-   const real_t *q2d_B = nullptr; // matrix to transform solution
-   const real_t *q2d_Bt = nullptr; // matrix to transform RHS
-   if (CHANGE_BASIS)
-   {
-      d2q_B = d2q->B.Read();
-      q2d_B = B_.Read();
-      q2d_Bt = Bt_.Read();
-
-      b2 = b2_.Write();
-      b_orig = b_.Read();
-      b = b2;
-   }
-   else
-   {
-      b = b_.Read();
-   }
-
-   static constexpr int NB = Q1D ? Q1D : 1; // block size
-
-   mfem::forall_2D(NE, NB, NB, [=] MFEM_HOST_DEVICE (int e)
-   {
-      // Perform change of basis if needed
-      if (CHANGE_BASIS)
-      {
-         // Transform RHS
-         DGMassBasis<DIM,D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
-         if (IT_MODE)
-         {
-            // Transform initial guess
-            DGMassBasis<DIM,D1D>(e, NE, d2q_B, u, u, d1d);
-         }
-      }
-
-      const int tid = MFEM_THREAD_ID(x) + NB*MFEM_THREAD_ID(y);
-
-      // Compute first residual
-      if (IT_MODE)
-      {
-         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, u, r, d1d, q1d);
-         DGMassAxpy(e, NE, ND, 1.0, b, -1.0, r, r); // r = b - r
-      }
-      else
-      {
-         // if not in iterative mode, use zero initial guess
-         const int BX = MFEM_THREAD_SIZE(x);
-         const int BY = MFEM_THREAD_SIZE(y);
-         const int bxy = BX*BY;
-         const auto B = ConstDeviceMatrix(b, ND, NE);
-         auto U = DeviceMatrix(u, ND, NE);
-         auto R = DeviceMatrix(r, ND, NE);
-         for (int i = tid; i < ND; i += bxy)
-         {
-            U(i, e) = 0.0;
-            R(i, e) = B(i, e);
-         }
-         MFEM_SYNC_THREAD;
-      }
-
-      DGMassPreconditioner(e, NE, ND, dinv, r, z);
-      DGMassAxpy(e, NE, ND, 1.0, z, 0.0, z, d); // d = z
-
-      real_t nom = DGMassDot<NB>(e, NE, ND, d, r);
-      if (nom < 0.0) { return; /* Not positive definite */ }
-      real_t r0 = fmax(nom*RELTOL*RELTOL, ABSTOL*ABSTOL);
-      if (nom <= r0) { return; /* Converged */ }
-
-      DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d);
-      real_t den = DGMassDot<NB>(e, NE, ND, z, d);
-      if (den <= 0.0)
-      {
-         DGMassDot<NB>(e, NE, ND, d, d);
-         // d2 > 0 => not positive definite
-         if (den == 0.0) { return; }
-      }
-
-      // start iteration
-      int i = 1;
-      while (true)
-      {
-         const real_t alpha = nom/den;
-         DGMassAxpy(e, NE, ND, 1.0, u, alpha, d, u); // u = u + alpha*d
-         DGMassAxpy(e, NE, ND, 1.0, r, -alpha, z, r); // r = r - alpha*A*d
-
-         DGMassPreconditioner(e, NE, ND, dinv, r, z);
-
-         real_t betanom = DGMassDot<NB>(e, NE, ND, r, z);
-         if (betanom < 0.0) { return; /* Not positive definite */ }
-         if (betanom <= r0) { break; /* Converged */ }
-
-         if (++i > MAXIT) { break; }
-
-         const real_t beta = betanom/nom;
-         DGMassAxpy(e, NE, ND, 1.0, z, beta, d, d); // d = z + beta*d
-         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d); // z = A d
-         den = DGMassDot<NB>(e, NE, ND, d, z);
-         if (den <= 0.0)
-         {
-            DGMassDot<NB>(e, NE, ND, d, d);
-            // d2 > 0 => not positive definite
-            if (den == 0.0) { break; }
-         }
-         nom = betanom;
-      }
-
-      if (CHANGE_BASIS)
-      {
-         DGMassBasis<DIM,D1D>(e, NE, q2d_B, u, u, d1d);
-      }
-   });
-}
-
 void DGMassInverse::Mult(const Vector &Mu, Vector &u) const
 {
    // Dispatch to templated version based on dim, d1d, and q1d.
@@ -306,23 +160,4 @@ DGMassInvKernels::DGMassInvKernels()
    k::Specialization<3,6,7>::Add();
 }
 
-/// @cond Suppress_Doxygen_warnings
-
-template <int DIM, int D1D, int Q1D>
-DGMassInverse::CGKernelType DGMassInverse::CGKernels::Kernel()
-{
-   return &DGMassInverse::DGMassCGIteration<DIM,D1D,Q1D>;
-}
-
-DGMassInverse::CGKernelType DGMassInverse::CGKernels::Fallback(
-   int dim, int, int)
-{
-   if (dim == 1) { return &DGMassInverse::DGMassCGIteration<1>; }
-   else if (dim == 2) { return &DGMassInverse::DGMassCGIteration<2>; }
-   else if (dim == 3) { return &DGMassInverse::DGMassCGIteration<3>; }
-   else { MFEM_ABORT("Unsupported dimension."); }
-}
-
-/// @endcond
-
 } // namespace mfem

fem/dgmassinv_kernels.hpp

@@ -15,6 +15,7 @@
 #include "../linalg/kernels.hpp"
 #include "kernels.hpp"
 #include "integ/bilininteg_mass_kernels.hpp"
+#include "dgmassinv.hpp"
 
 namespace mfem
 {
@@ -333,6 +334,170 @@ void DGMassBasis(const int e,
 
 } // namespace internal
 
+template<int DIM, int D1D, int Q1D>
+void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
+{
+   using namespace internal; // host/device kernel functions
+
+   const int NE = fes.GetNE();
+   const int d1d = m->dofs1D;
+   const int q1d = m->quad1D;
+
+   const int ND = static_cast<int>(pow(d1d, DIM));
+
+   const auto B = m->maps->B.Read();
+   const auto Bt = m->maps->Bt.Read();
+   const auto pa_data = m->pa_data.Read();
+   const auto dinv = diag_inv.Read();
+   auto r = r_.Write();
+   auto d = d_.Write();
+   auto z = z_.Write();
+   auto u = u_.ReadWrite();
+
+   const real_t RELTOL = rel_tol;
+   const real_t ABSTOL = abs_tol;
+   const int MAXIT = max_iter;
+   const bool IT_MODE = iterative_mode;
+   const bool CHANGE_BASIS = (d2q != nullptr);
+
+   // b is the right-hand side (if no change of basis, this just points to the
+   // incoming RHS vector, if we have to change basis, this points to the
+   // internal b2 vector where we put the transformed RHS)
+   const real_t *b;
+   // the following are non-null if we have to change basis
+   real_t *b2 = nullptr; // non-const access to b2
+   const real_t *b_orig = nullptr; // RHS vector in "original" basis
+   const real_t *d2q_B = nullptr; // matrix to transform initial guess
+   const real_t *q2d_B = nullptr; // matrix to transform solution
+   const real_t *q2d_Bt = nullptr; // matrix to transform RHS
+   if (CHANGE_BASIS)
+   {
+      d2q_B = d2q->B.Read();
+      q2d_B = B_.Read();
+      q2d_Bt = Bt_.Read();
+
+      b2 = b2_.Write();
+      b_orig = b_.Read();
+      b = b2;
+   }
+   else
+   {
+      b = b_.Read();
+   }
+
+   static constexpr int NB = Q1D ? Q1D : 1; // block size
+
+   mfem::forall_2D<NB*NB>(NE, NB, NB, [=] MFEM_HOST_DEVICE (int e)
+   {
+      // Perform change of basis if needed
+      if (CHANGE_BASIS)
+      {
+         // Transform RHS
+         DGMassBasis<DIM,D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
+         if (IT_MODE)
+         {
+            // Transform initial guess
+            DGMassBasis<DIM,D1D>(e, NE, d2q_B, u, u, d1d);
+         }
+      }
+
+      const int tid = MFEM_THREAD_ID(x) + NB*MFEM_THREAD_ID(y);
+
+      // Compute first residual
+      if (IT_MODE)
+      {
+         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, u, r, d1d, q1d);
+         DGMassAxpy(e, NE, ND, 1.0, b, -1.0, r, r); // r = b - r
+      }
+      else
+      {
+         // if not in iterative mode, use zero initial guess
+         const int BX = MFEM_THREAD_SIZE(x);
+         const int BY = MFEM_THREAD_SIZE(y);
+         const int bxy = BX*BY;
+         const auto B = ConstDeviceMatrix(b, ND, NE);
+         auto U = DeviceMatrix(u, ND, NE);
+         auto R = DeviceMatrix(r, ND, NE);
+         for (int i = tid; i < ND; i += bxy)
+         {
+            U(i, e) = 0.0;
+            R(i, e) = B(i, e);
+         }
+         MFEM_SYNC_THREAD;
+      }
+
+      DGMassPreconditioner(e, NE, ND, dinv, r, z);
+      DGMassAxpy(e, NE, ND, 1.0, z, 0.0, z, d); // d = z
+
+      real_t nom = DGMassDot<NB>(e, NE, ND, d, r);
+      if (nom < 0.0) { return; /* Not positive definite */ }
+      real_t r0 = fmax(nom*RELTOL*RELTOL, ABSTOL*ABSTOL);
+      if (nom <= r0) { return; /* Converged */ }
+
+      DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d);
+      real_t den = DGMassDot<NB>(e, NE, ND, z, d);
+      if (den <= 0.0)
+      {
+         DGMassDot<NB>(e, NE, ND, d, d);
+         // d2 > 0 => not positive definite
+         if (den == 0.0) { return; }
+      }
+
+      // start iteration
+      int i = 1;
+      while (true)
+      {
+         const real_t alpha = nom/den;
+         DGMassAxpy(e, NE, ND, 1.0, u, alpha, d, u); // u = u + alpha*d
+         DGMassAxpy(e, NE, ND, 1.0, r, -alpha, z, r); // r = r - alpha*A*d
+
+         DGMassPreconditioner(e, NE, ND, dinv, r, z);
+
+         real_t betanom = DGMassDot<NB>(e, NE, ND, r, z);
+         if (betanom < 0.0) { return; /* Not positive definite */ }
+         if (betanom <= r0) { break; /* Converged */ }
+
+         if (++i > MAXIT) { break; }
+
+         const real_t beta = betanom/nom;
+         DGMassAxpy(e, NE, ND, 1.0, z, beta, d, d); // d = z + beta*d
+         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d); // z = A d
+         den = DGMassDot<NB>(e, NE, ND, d, z);
+         if (den <= 0.0)
+         {
+            DGMassDot<NB>(e, NE, ND, d, d);
+            // d2 > 0 => not positive definite
+            if (den == 0.0) { break; }
+         }
+         nom = betanom;
+      }
+
+      if (CHANGE_BASIS)
+      {
+         DGMassBasis<DIM,D1D>(e, NE, q2d_B, u, u, d1d);
+      }
+   });
+}
+
+/// @cond Suppress_Doxygen_warnings
+
+template <int DIM, int D1D, int Q1D>
+inline DGMassInverse::CGKernelType DGMassInverse::CGKernels::Kernel()
+{
+   return &DGMassInverse::DGMassCGIteration<DIM,D1D,Q1D>;
+}
+
+inline DGMassInverse::CGKernelType DGMassInverse::CGKernels::Fallback(
+   int dim, int, int)
+{
+   if (dim == 1) { return &DGMassInverse::DGMassCGIteration<1>; }
+   else if (dim == 2) { return &DGMassInverse::DGMassCGIteration<2>; }
+   else if (dim == 3) { return &DGMassInverse::DGMassCGIteration<3>; }
+   else { MFEM_ABORT("Unsupported dimension."); }
+}
+
+/// @endcond
+
 } // namespace mfem
 
 #endif

fem/fe/face_map_utils.hpp

@@ -69,9 +69,9 @@ inline int ToLexOrdering2D(const int face_id, const int size1d, const int i)
 }
 
 /// @brief Given a face DOF index on a shared face, ordered lexicographically
-/// relative to element the element (where the local face is face_id), and
-/// return the corresponding face DOF index ordered lexicographically relative
-/// to the face itself.
+/// relative to the element (where the local face is face_id), return the
+/// corresponding face DOF index ordered lexicographically relative to the face
+/// itself.
 MFEM_HOST_DEVICE
 inline int PermuteFace2D(const int face_id, const int orientation,
                          const int size1d, const int index)

fem/fe/fe_base.cpp

@@ -231,7 +231,7 @@ void FiniteElement::CalcPhysLaplacian(ElementTransformation &Trans,
    {
       for (int nd = 0; nd < dof; nd++)
       {
-         Laplacian[nd] = hess(nd,0) + hess(nd,4) + hess(nd,5);
+         Laplacian[nd] = hess(nd,0) + hess(nd,3) + hess(nd,5);
       }
    }
    else if (dim == 2)
@@ -268,11 +268,9 @@ void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
       scale[0] =   Gij(0,0);
       scale[1] = 2*Gij(0,1);
       scale[2] = 2*Gij(0,2);
-
-      scale[3] = 2*Gij(1,2);
-      scale[4] =   Gij(2,2);
-
-      scale[5] =   Gij(1,1);
+      scale[3] =   Gij(1,1);
+      scale[4] = 2*Gij(1,2);
+      scale[5] =   Gij(2,2);
    }
    else if (dim == 2)
    {
@@ -309,12 +307,12 @@ void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,
       map[2] = 2;
 
       map[3] = 1;
-      map[4] = 5;
-      map[5] = 3;
+      map[4] = 3;
+      map[5] = 4;
 
       map[6] = 2;
-      map[7] = 3;
-      map[8] = 4;
+      map[7] = 4;
+      map[8] = 5;
    }
    else if (dim == 2)
    {
@@ -382,11 +380,7 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
    #pragma omp critical (DofToQuad)
 #endif
    {
-      for (int i = 0; i < dof2quad_array.Size(); i++)
-      {
-         d2q = dof2quad_array[i];
-         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
-      }
+      d2q = DofToQuad::SearchArray(dof2quad_array, ir, mode);
       if (!d2q)
       {
 #ifdef MFEM_THREAD_SAFE
@@ -661,58 +655,67 @@ void ScalarFiniteElement::ScalarLocalL2Restriction(
 void NodalFiniteElement::CreateLexicographicFullMap(const IntegrationRule &ir)
 const
 {
+   // Get the FULL version of the map. This call contains omp critical region,
+   // so it is done before the critical region below.
+   auto &d2q = GetDofToQuad(ir, DofToQuad::FULL);
 
 #if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
    #pragma omp critical (DofToQuad)
 #endif
    {
-      // Get the FULL version of the map.
-      auto &d2q = GetDofToQuad(ir, DofToQuad::FULL);
-      //Undo the native ordering which is what FiniteElement::GetDofToQuad returns.
-      auto *d2q_new = new DofToQuad(d2q);
-      d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL;
-      const int nqpt = ir.GetNPoints();
-
-      const int b_dim = (range_type == VECTOR) ? dim : 1;
-
-      for (int i = 0; i < nqpt; i++)
+      // Do not run if the new Dof2Quad is already present, e.g. added in a
+      // previous call or added by another omp thread.
+      if (DofToQuad::SearchArray(dof2quad_array, ir,
+                                 DofToQuad::LEXICOGRAPHIC_FULL) == nullptr)
       {
-         for (int d = 0; d < b_dim; d++)
+         // Undo the native ordering which is what FiniteElement::GetDofToQuad
+         // returns.
+         auto *d2q_new = new DofToQuad(d2q);
+         d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL;
+         const int nqpt = ir.GetNPoints();
+
+         const int b_dim = (range_type == VECTOR) ? dim : 1;
+
+         for (int i = 0; i < nqpt; i++)
          {
-            for (int j = 0; j < dof; j++)
+            for (int d = 0; d < b_dim; d++)
             {
-               const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])];
-               d2q_new->B[i+nqpt*(d+b_dim*j)] = val;
-               d2q_new->Bt[j+dof*(i+nqpt*d)] = val;
+               for (int j = 0; j < dof; j++)
+               {
+                  const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])];
+                  d2q_new->B[i+nqpt*(d+b_dim*j)] = val;
+                  d2q_new->Bt[j+dof*(i+nqpt*d)] = val;
+               }
             }
          }
-      }
 
-      const int g_dim = [this]()
-      {
-         switch (deriv_type)
+         const int g_dim = [this]()
          {
-            case GRAD: return dim;
-            case DIV: return 1;
-            case CURL: return cdim;
-            default: return 0;
-         }
-      }();
-
-      for (int i = 0; i < nqpt; i++)
-      {
-         for (int d = 0; d < g_dim; d++)
-         {
-            for (int j = 0; j < dof; j++)
+            switch (deriv_type)
             {
-               const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])];
-               d2q_new->G[i+nqpt*(d+g_dim*j)] = val;
-               d2q_new->Gt[j+dof*(i+nqpt*d)] = val;
+               case GRAD: return dim;
+               case DIV: return 1;
+               case CURL: return cdim;
+               default: return 0;
+            }
+         }();
+
+         for (int i = 0; i < nqpt; i++)
+         {
+            for (int d = 0; d < g_dim; d++)
+            {
+               for (int j = 0; j < dof; j++)
+               {
+                  const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])];
+                  d2q_new->G[i+nqpt*(d+g_dim*j)] = val;
+                  d2q_new->Gt[j+dof*(i+nqpt*d)] = val;
+               }
             }
          }
+
+         dof2quad_array.Append(d2q_new);
       }
 
-      dof2quad_array.Append(d2q_new);
    }
 }
 
@@ -724,13 +727,7 @@ const DofToQuad &NodalFiniteElement::GetDofToQuad(const IntegrationRule &ir,
    #pragma omp critical (DofToQuad)
 #endif
    {
-      //Should make this loop a function of FiniteElement
-      for (int i = 0; i < dof2quad_array.Size(); i++)
-      {
-         d2q = dof2quad_array[i];
-         if (d2q->IntRule == &ir && d2q->mode == mode) { break; }
-         d2q = nullptr;
-      }
+      d2q = DofToQuad::SearchArray(dof2quad_array, ir, mode);
    }
    if (d2q) { return *d2q; }
    if (mode != DofToQuad::LEXICOGRAPHIC_FULL)
@@ -2631,15 +2628,7 @@ const DofToQuad &TensorBasisElement::GetTensorDofToQuad(
    #pragma omp critical (DofToQuad)
 #endif
    {
-      for (int i = 0; i < dof2quad_array.Size(); i++)
-      {
-         auto* d2q_ = dof2quad_array[i];
-         if (d2q_->IntRule == &ir && d2q_->mode == mode)
-         {
-            d2q = d2q_;
-            break;
-         }
-      }
+      d2q = DofToQuad::SearchArray(dof2quad_array, ir, mode);
       if (!d2q)
       {
          d2q = new DofToQuad;

fem/fe/fe_base.hpp

@@ -44,7 +44,7 @@ public:
       NumBasisTypes   = 9   /**< Keep track of maximum types to prevent
                                  hard-coding */
    };
-   /** @brief If the input does not represents a valid BasisType, abort with an
+   /** @brief If the input does not represent a valid BasisType, abort with an
        error; otherwise return the input. */
    static int Check(int b_type)
    {
@@ -52,7 +52,7 @@ public:
                   "unknown BasisType: " << b_type);
       return b_type;
    }
-   /** @brief If the input does not represents a valid nodal BasisType, abort
+   /** @brief If the input does not represent a valid nodal BasisType, abort
        with an error; otherwise return the input. */
    static int CheckNodal(int b_type)
    {
@@ -222,6 +222,12 @@ public:
 
    /// Returns absolute value of the maps
    DofToQuad Abs() const;
+
+   /// Auxiliary function for searching DofToQuad arrays.
+   static inline DofToQuad *SearchArray(
+      const Array<DofToQuad*> &dof2quad_array,
+      const IntegrationRule &ir,
+      DofToQuad::Mode mode);
 };
 
 /// Describes the function space on each element
@@ -407,6 +413,7 @@ public:
    /** Each row of the result DenseMatrix @a Hessian contains upper triangular
        part of the Hessian of one shape function.
        The order in 2D is {u_xx, u_xy, u_yy}.
+       The order in 3D is {u_xx, u_xy, u_xz, u_yy, u_yz, u_zz}.
        The size (#dof x (#dim (#dim+1)/2) of @a Hessian must be set in advance.*/
    virtual void CalcHessian(const IntegrationPoint &ip,
                             DenseMatrix &Hessian) const;
@@ -1120,7 +1127,7 @@ public:
       return GetPoints(p, btype, on_device);
    }
 
-   /// Get coordinates of a closed (GaussLegendre) set of points if degree @a p
+   /// Get coordinates of a closed (GaussLobatto) set of points if degree @a p
    const real_t *ClosedPoints(const int p,
                               const int btype = BasisType::GaussLobatto,
                               bool on_device = false)
@@ -1376,6 +1383,21 @@ public:
 void InvertLinearTrans(ElementTransformation &trans,
                        const IntegrationPoint &pt, Vector &x);
 
+
+// static inline method
+inline DofToQuad *DofToQuad::SearchArray(
+   const Array<DofToQuad*> &dof2quad_array,
+   const IntegrationRule &ir,
+   DofToQuad::Mode mode)
+{
+   for (int i = 0; i < dof2quad_array.Size(); i++)
+   {
+      DofToQuad *d2q = dof2quad_array[i];
+      if (d2q->IntRule == &ir && d2q->mode == mode) { return d2q; }
+   }
+   return nullptr;
+}
+
 } // namespace mfem
 
 #endif

fem/fe/fe_fixed_order.cpp

@@ -60,6 +60,12 @@ void Linear1DFiniteElement::CalcDShape(const IntegrationPoint &ip,
    dshape(1,0) =  1.;
 }
 
+void Linear1DFiniteElement::CalcHessian(const IntegrationPoint &ip,
+                                        DenseMatrix &h) const
+{
+   h = 0.0;
+}
+
 Linear2DFiniteElement::Linear2DFiniteElement()
    : NodalFiniteElement(2, Geometry::TRIANGLE, 3, 1)
 {
@@ -87,6 +93,11 @@ void Linear2DFiniteElement::CalcDShape(const IntegrationPoint &ip,
    dshape(2,0) =  0.; dshape(2,1) =  1.;
 }
 
+void Linear2DFiniteElement::CalcHessian(const IntegrationPoint &ip,
+                                        DenseMatrix &h) const
+{
+   h = 0.0;
+}
 
 BiLinear2DFiniteElement::BiLinear2DFiniteElement()
    : NodalFiniteElement(2, Geometry::SQUARE, 4, 1, FunctionSpace::Qk)
@@ -1256,6 +1267,12 @@ void Linear3DFiniteElement::CalcDShape(const IntegrationPoint &ip,
    }
 }
 
+void Linear3DFiniteElement::CalcHessian(const IntegrationPoint &ip,
+                                        DenseMatrix &h) const
+{
+   h = 0.0;
+}
+
 void Linear3DFiniteElement::GetFaceDofs (int face, int **dofs, int *ndofs)
 const
 {
@@ -1632,6 +1649,37 @@ void TriLinear3DFiniteElement::CalcDShape(const IntegrationPoint &ip,
    dshape(7,2) =   ox *  y;
 }
 
+void TriLinear3DFiniteElement::CalcHessian(const IntegrationPoint &ip,
+                                           DenseMatrix &h) const
+{
+   real_t x = ip.x, y = ip.y, z = ip.z;
+   real_t ox = 1.-x, oy = 1.-y, oz = 1.-z;
+
+   h(0,0) = 0.;   h(0,1) =  oz;   h(0,2) = oy;
+   h(0,3) = 0.;   h(0,4) =  ox;   h(0,5) = 0.;
+
+   h(1,0) = 0.;   h(1,1) = -oz;   h(1,2) = -oy;
+   h(1,3) = 0.;   h(1,4) =  x;    h(1,5) = 0.;
+
+   h(2,0) = 0.;   h(2,1) =  oz;   h(2,2) = -y;
+   h(2,3) = 0.;   h(2,4) =  -x;   h(2,5) = 0.;
+
+   h(3,0) = 0.;   h(3,1) = -oz;   h(3,2) = y;
+   h(3,3) = 0.;   h(3,4) = -ox;   h(3,5) = 0.;
+
+   h(4,0) = 0.;   h(4,1) =  z;    h(4,2) = -oy;
+   h(4,3) = 0.;   h(4,4) = -ox;   h(4,5) = 0.;
+
+   h(5,0) = 0.;   h(5,1) = -z;    h(5,2) = oy;
+   h(5,3) = 0.;   h(5,4) = -x;    h(5,5) = 0.;
+
+   h(6,0) = 0.;   h(6,1) =  z;    h(6,2) = y;
+   h(6,3) = 0.;   h(6,4) =  x;    h(6,5) = 0.;
+
+   h(7,0) = 0.;   h(7,1) = -z;    h(7,2) = -y;
+   h(7,3) = 0.;   h(7,4) = ox;    h(7,5) = 0.;
+}
+
 
 P0SegmentFiniteElement::P0SegmentFiniteElement(int Ord)
    : NodalFiniteElement(1, Geometry::SEGMENT, 1, Ord)   // default Ord = 0

fem/fe/fe_fixed_order.hpp

@@ -50,6 +50,8 @@ public:
        contains the derivative of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
+   void CalcHessian(const IntegrationPoint &ip,
+                    DenseMatrix &h) const override;
 };
 
 /// A 2D linear element on triangle with nodes at the vertices of the triangle
@@ -70,6 +72,8 @@ public:
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
+   void CalcHessian(const IntegrationPoint &ip,
+                    DenseMatrix &h) const override;
    void ProjectDelta(int vertex, Vector &dofs) const override
    { dofs = 0.0; dofs(vertex) = 1.0; }
 };
@@ -404,6 +408,9 @@ public:
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
 
+   void CalcHessian(const IntegrationPoint &ip,
+                    DenseMatrix &h) const override;
+
    void ProjectDelta(int vertex, Vector &dofs) const override
    { dofs = 0.0; dofs(vertex) = 1.0; }
 
@@ -445,7 +452,8 @@ public:
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
-
+   void CalcHessian(const IntegrationPoint &ip,
+                    DenseMatrix &h) const override;
    void ProjectDelta(int vertex, Vector &dofs) const override
    { dofs = 0.0; dofs(vertex) = 1.0; }
 };

fem/fe/fe_h1.cpp

@@ -589,7 +589,7 @@ void H1_TriangleElement::CalcHessian(const IntegrationPoint &ip,
    Vector   shape_x(p + 1),   shape_y(p + 1),   shape_l(p + 1);
    Vector  dshape_x(p + 1),  dshape_y(p + 1),  dshape_l(p + 1);
    Vector ddshape_x(p + 1), ddshape_y(p + 1), ddshape_l(p + 1);
-   DenseMatrix ddu(dof, dim);
+   DenseMatrix ddu(dof, (dim*(dim+1))/2);
 #endif
 
    poly1d.CalcBasis(p, ip.x, shape_x, dshape_x, ddshape_x);

fem/fe/fe_nurbs.cpp

@@ -84,6 +84,46 @@ void NURBS1DFiniteElement::CalcHessian (const IntegrationPoint &ip,
    add(1.0, hess, (-d2sum + 2*dsum*dsum*sum)*sum*sum, shape_x, hess);
 }
 
+void NURBS1DFiniteElement::Project(Coefficient &coeff,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   IntegrationPoint ip;
+
+   for (int i = 0; i <= order; i++)
+   {
+      real_t kx = kv[0]->GetBotella(ijk[0] + i);
+      if (!kv[0]->inSpan(kx, ijk[0]+order)) { continue; }
+      ip.x = kv[0]->GetRefPoint(kx, ijk[0]+order);
+
+      Trans.SetIntPoint(&ip);
+      dofs(i) = coeff.Eval(Trans, ip);
+   }
+}
+
+void NURBS1DFiniteElement::Project(VectorCoefficient &vc,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
+   Vector x(vc.GetVDim());
+   IntegrationPoint ip;
+
+   for (int i = 0; i <= order; i++)
+   {
+      real_t kx = kv[0]->GetBotella(ijk[0] + i);
+      if (!kv[0]->inSpan(kx, ijk[0]+order)) { continue; }
+      ip.x = kv[0]->GetRefPoint(kx, ijk[0]+order);
+
+      Trans.SetIntPoint(&ip);
+      vc.Eval(x, Trans, ip);
+      for (int j = 0; j < x.Size(); j++)
+      {
+         dofs(dof*j+i) = x(j);
+      }
+   }
+}
+
 
 void NURBS2DFiniteElement::SetOrder() const
 {
@@ -215,6 +255,63 @@ void NURBS2DFiniteElement::CalcHessian (const IntegrationPoint &ip,
    }
 }
 
+void NURBS2DFiniteElement::Project(Coefficient &coeff,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   IntegrationPoint ip;
+   for (int o = 0, j = 0; j <= orders[1]; j++)
+   {
+      real_t ky = kv[1]->GetBotella(ijk[1] + j);
+      if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+      {
+         o += orders[0] + 1;
+         continue;
+      }
+      ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         real_t kx = kv[0]->GetBotella(ijk[0] + i);
+         if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+         ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+         Trans.SetIntPoint(&ip);
+         dofs(o) = coeff.Eval(Trans, ip);
+      }
+   }
+}
+
+void NURBS2DFiniteElement::Project(VectorCoefficient &vc,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
+   Vector x(vc.GetVDim());
+   IntegrationPoint ip;
+   for (int o = 0, j = 0; j <= orders[1]; j++)
+   {
+      real_t ky = kv[1]->GetBotella(ijk[1] + j);
+      if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+      {
+         o += orders[0] + 1;
+         continue;
+      }
+      ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         real_t kx = kv[0]->GetBotella(ijk[0] + i);
+         if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+         ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+         for (int v = 0; v < x.Size(); v++)
+         {
+            dofs(dof*v+o) = x(v);
+         }
+      }
+   }
+}
 
 void NURBS3DFiniteElement::SetOrder() const
 {
@@ -348,11 +445,10 @@ void NURBS3DFiniteElement::CalcHessian (const IntegrationPoint &ip,
             d2sum[0] += ( hessian(o,0) = d2sx*sy*sz*weights(o) );
             d2sum[1] += ( hessian(o,1) = dsx*dsy*sz*weights(o) );
             d2sum[2] += ( hessian(o,2) = dsx*sy*dsz*weights(o) );
+            d2sum[3] += ( hessian(o,3) = sx*d2sy*sz*weights(o) );
+            d2sum[4] += ( hessian(o,4) = sx*dsy*dsz*weights(o) );
+            d2sum[5] += ( hessian(o,5) = sx*sy*d2sz*weights(o) );
 
-            d2sum[3] += ( hessian(o,3) = sx*dsy*dsz*weights(o) );
-
-            d2sum[4] += ( hessian(o,4) = sx*sy*d2sz*weights(o) );
-            d2sum[5] += ( hessian(o,5) = sx*d2sy*sz*weights(o) );
          }
       }
    }
@@ -401,6 +497,85 @@ void NURBS3DFiniteElement::CalcHessian (const IntegrationPoint &ip,
    }
 }
 
+void NURBS3DFiniteElement::Project(Coefficient &coeff,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   IntegrationPoint ip;
+
+   for (int o = 0, k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 1)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            dofs(o) = coeff.Eval(Trans, ip);
+         }
+      }
+   }
+}
+
+void NURBS3DFiniteElement::Project(VectorCoefficient &vc,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
+   Vector x(vc.GetVDim());
+   IntegrationPoint ip;
+
+   for (int o = 0, k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 1)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+            for (int v = 0; v < x.Size(); v++)
+            {
+               dofs(dof*v+o) = x(v);
+            }
+         }
+      }
+   }
+}
 
 void NURBS_HDiv2DFiniteElement::SetOrder() const
 {
@@ -517,6 +692,63 @@ void NURBS_HDiv2DFiniteElement::CalcDivShape(const IntegrationPoint &ip,
    }
 }
 
+void NURBS_HDiv2DFiniteElement::Project(VectorCoefficient &vc,
+                                        ElementTransformation &Trans,
+                                        Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == dof, "");
+   MFEM_ASSERT(vc.GetVDim() == 2, "");
+   Vector x(2), mx(2);
+   IntegrationPoint ip;
+   int o = 0;
+
+   for (int j = 0; j <= orders[1]; j++)
+   {
+      real_t ky = kv[1]->GetBotella(ijk[1] + j);
+      if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+      {
+         o += orders[0] + 2;
+         continue;
+      }
+      ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+      for (int i = 0; i <= orders[0]+1; i++, o++)
+      {
+         real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+         if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+         ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+
+         Trans.AdjugateJacobian().Mult(x,mx);
+         dofs(o) = mx(0);
+      }
+   }
+
+   for (int j = 0; j <= orders[1]+1; j++)
+   {
+      real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+      if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+      {
+         o += orders[0] + 1;
+         continue;
+      }
+      ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         real_t kx = kv[0]->GetBotella(ijk[0] + i);
+         if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+         ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+
+         Trans.AdjugateJacobian().Mult(x,mx);
+         dofs(o) = mx(1);
+      }
+   }
+}
+
 NURBS_HDiv2DFiniteElement::~NURBS_HDiv2DFiniteElement()
 {
    if (kv1[0]) { delete kv1[0]; }
@@ -696,6 +928,120 @@ void NURBS_HDiv3DFiniteElement::CalcDivShape(const IntegrationPoint &ip,
    }
 }
 
+
+void NURBS_HDiv3DFiniteElement::Project(VectorCoefficient &vc,
+                                        ElementTransformation &Trans,
+                                        Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == dof, "");
+   MFEM_ASSERT(vc.GetVDim() == 3, "");
+   Vector x(2), mx(3);
+   IntegrationPoint ip;
+
+   int o = 0;
+
+   for (int k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 2)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 2;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+            if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+            ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.AdjugateJacobian().Mult(x,mx);
+            dofs(o) = mx(0);
+         }
+      }
+   }
+
+   for (int k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 1)*(orders[1] + 2);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+         if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.AdjugateJacobian().Mult(x,mx);
+            dofs(o) = mx(1);
+         }
+      }
+   }
+
+   for (int k = 0; k <= orders[2]+1; k++)
+   {
+      real_t kz = kv1[2]->GetBotella(ijk[2] + k);
+      if (!kv1[2]->inSpan(kz, ijk[2]+orders[2]+1))
+      {
+         o += (orders[0] + 1)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv1[2]->GetRefPoint(kz, ijk[2]+orders[2]+1);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.AdjugateJacobian().Mult(x,mx);
+            dofs(o) = mx(2);
+         }
+      }
+   }
+
+}
+
+
 NURBS_HDiv3DFiniteElement::~NURBS_HDiv3DFiniteElement()
 {
    if (kv1[0]) { delete kv1[0]; }
@@ -817,13 +1163,68 @@ void NURBS_HCurl2DFiniteElement::CalcCurlShape(const IntegrationPoint &ip,
    }
 }
 
+void NURBS_HCurl2DFiniteElement::Project(VectorCoefficient &vc,
+                                         ElementTransformation &Trans,
+                                         Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == dof, "");
+   MFEM_ASSERT(vc.GetVDim() == 2, "");
+   Vector x(2), xm(2);
+   IntegrationPoint ip;
+   int i, j, o;
+   for (o = 0, j = 0; j <= orders[1]+1; j++)
+   {
+      real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+      if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+      {
+         o += orders[0] + 1;
+         continue;
+      }
+      ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+      for (i = 0; i <= orders[0]; i++, o++)
+      {
+         real_t kx = kv[0]->GetBotella(ijk[0] + i);
+         if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+         ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+
+         Trans.Jacobian().MultTranspose(x,xm);
+         dofs(o) = xm(0);
+      }
+   }
+
+   for (j = 0; j <= orders[1]; j++)
+   {
+      real_t ky = kv[1]->GetBotella(ijk[1] + j);
+      if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+      {
+         o += orders[0] + 2;
+         continue;
+      }
+      ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+      for (i = 0; i <= orders[0]+1; i++, o++)
+      {
+         real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+         if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+         ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+
+         Trans.Jacobian().MultTranspose(x,xm);
+         dofs(o) = xm(1);
+      }
+   }
+}
+
 NURBS_HCurl2DFiniteElement::~NURBS_HCurl2DFiniteElement()
 {
    if (kv1[0]) { delete kv1[0]; }
    if (kv1[1]) { delete kv1[1]; }
 }
 
-
 void NURBS_HCurl3DFiniteElement::SetOrder() const
 {
    orders[0] = kv[0]->GetOrder();
@@ -1003,11 +1404,124 @@ void NURBS_HCurl3DFiniteElement::CalcCurlShape(const IntegrationPoint &ip,
             curl_shape(o,0) = shape1_x(i)*dsy1_sz;
             curl_shape(o,1) = -dshape1_x(i)*sy1_sz;
             curl_shape(o,2) = 0.0;
+
          }
       }
    }
 }
 
+void NURBS_HCurl3DFiniteElement::Project(VectorCoefficient &vc,
+                                         ElementTransformation &Trans,
+                                         Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == dof, "");
+   MFEM_ASSERT(vc.GetVDim() == 3, "");
+   Vector x(3), xm(3);
+   IntegrationPoint ip;
+
+   int o = 0;
+   for (int k = 0; k <= orders[2]+1; k++)
+   {
+      real_t kz = kv1[2]->GetBotella(ijk[2] + k);
+      if (!kv1[2]->inSpan(kz, ijk[2]+orders[2]+1))
+      {
+         o += (orders[0] + 1)*(orders[1] + 2);
+         continue;
+      }
+      ip.z = kv1[2]->GetRefPoint(kz, ijk[2]+orders[2]+1);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+         if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.Jacobian().MultTranspose(x,xm);
+            dofs(o) = xm(0);
+         }
+      }
+   }
+
+   for (int k = 0; k <= orders[2]+1; k++)
+   {
+      real_t kz = kv1[2]->GetBotella(ijk[2] + k);
+      if (!kv1[2]->inSpan(kz, ijk[2]+orders[2]+1))
+      {
+         o += (orders[0] + 2)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv1[2]->GetRefPoint(kz, ijk[2]+orders[2]+1);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 2;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+            if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+            ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.Jacobian().MultTranspose(x,xm);
+            dofs(o) = xm(1);
+         }
+      }
+   }
+
+   for (int k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 2)*(orders[1] + 2);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+         if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+         {
+            o += orders[0] + 2;
+            continue;
+         }
+         ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+            if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+            ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.Jacobian().MultTranspose(x,xm);
+            dofs(o) = xm(2);
+         }
+      }
+   }
+
+}
+
+
 NURBS_HCurl3DFiniteElement::~NURBS_HCurl3DFiniteElement()
 {
    if (kv1[0]) { delete kv1[0]; }

fem/fe/fe_nurbs.hpp

@@ -86,6 +86,18 @@ public:
                    DenseMatrix &dshape) const override;
    void CalcHessian (const IntegrationPoint &ip,
                      DenseMatrix &hessian) const override;
+
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(Coefficient &coeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
 };
 
 /// An arbitrary order 2D NURBS element on a square
@@ -121,6 +133,18 @@ public:
                    DenseMatrix &dshape) const override;
    void CalcHessian (const IntegrationPoint &ip,
                      DenseMatrix &hessian) const override;
+
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(Coefficient &coeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
 };
 
 /// An arbitrary order 3D NURBS element on a cube
@@ -161,6 +185,18 @@ public:
                    DenseMatrix &dshape) const override;
    void CalcHessian (const IntegrationPoint &ip,
                      DenseMatrix &hessian) const override;
+
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(Coefficient &coeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
 };
 
 
@@ -242,6 +278,13 @@ public:
    void CalcDivShape(const IntegrationPoint &ip,
                      Vector &divshape) const override;
 
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
    ~NURBS_HDiv2DFiniteElement();
 };
 
@@ -336,6 +379,13 @@ public:
    void CalcDivShape(const IntegrationPoint &ip,
                      Vector &divshape) const override;
 
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
    ~NURBS_HDiv3DFiniteElement();
 };
 
@@ -415,6 +465,13 @@ public:
    void CalcCurlShape(const IntegrationPoint &ip,
                       DenseMatrix &curl_shape) const override;
 
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
    ~NURBS_HCurl2DFiniteElement();
 };
 
@@ -506,6 +563,13 @@ public:
    void CalcCurlShape(const IntegrationPoint &ip,
                       DenseMatrix &curl_shape) const override;
 
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
    ~NURBS_HCurl3DFiniteElement();
 };
 

fem/fe_coll.hpp

@@ -111,36 +111,36 @@ public:
    | :------: | :---: | :---: | :-------: | :-----: | :---: |
    | H1_[DIM]_[ORDER] | H1 | * | 1 | VALUE | H1 nodal elements |
    | H1@[BTYPE]_[DIM]_[ORDER] | H1 | * | * | VALUE | H1 nodal elements |
-   | H1Pos_[DIM]_[ORDER] | H1 | * | 1 | VALUE | H1 nodal elements |
+   | H1Pos_[DIM]_[ORDER] | H1 | * | 2 | VALUE | H1 nodal elements |
    | H1Pos_Trace_[DIM]_[ORDER] | H^{1/2} | * | 2 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
    | H1_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
-   | H1_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
+   | H1_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | * | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
    | ND_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | Nedelec vector elements |
    | ND@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | Nedelec vector elements |
-   | ND_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0  | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) |
-   | ND_Trace@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) |
+   | ND_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0  | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces,edges) |
+   | ND_Trace@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H^{1/2} | * | * / * | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces,edges) |
    | ND_R1D_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 1D. |
    | ND_R1D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 1D. |
    | ND_R2D_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 2D. |
    | ND_R2D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 2D. |
    | RT_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | Raviart-Thomas vector elements |
    | RT@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | Raviart-Thomas vector elements |
-   | RT_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
-   | RT_ValTrace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
-   | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
-   | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] |  H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_Trace_[DIM]_[ORDER] | H^{1/2} | * | 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_ValTrace_[DIM]_[ORDER] | H^{1/2} | * | 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | * | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] |  H^{1/2} | * | * | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
    | RT_R1D_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 1D. |
    | RT_R1D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 1D. |
    | RT_R2D_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 2D. |
    | RT_R2D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 2D. |
    | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
-   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
+   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | * | VALUE | Discontinuous L2 elements |
    | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
-   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
+   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | * | INTEGRAL | Discontinuous L2 elements |
    | DG_Iface_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
-   | DG_Iface@[BTYPE]_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
+   | DG_Iface@[BTYPE]_[DIM]_[ORDER] | - | * | * | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
    | DG_IntIface_[DIM]_[ORDER] | - | * | 0 | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
-   | DG_IntIface@[BTYPE]_[DIM]_[ORDER] | - | * | 0 | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
+   | DG_IntIface@[BTYPE]_[DIM]_[ORDER] | - | * | * | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
    | NURBS[ORDER] | - | * | - | VALUE | Non-Uniform Rational B-Splines (NURBS) elements |
    | LinearNonConf3D | - | 1 | 1 | VALUE | Piecewise-linear nonconforming finite elements in 3D |
    | CrouzeixRaviart | - | - | - | - | Crouzeix-Raviart nonconforming elements in 2D |
@@ -172,7 +172,7 @@ public:
    | :------: | :--------: |
    | [DIM]    | Dimension of the elements (1D, 2D, 3D) |
    | [ORDER]  | Approximation order of the elements (P0, P1, P2, ...) |
-   | [BTYPE]  | BasisType of the element (0-GaussLegendre, 1 - GaussLobatto, 2-Bernstein, 3-OpenUniform, 4-CloseUniform, 5-OpenHalfUniform) |
+   | [BTYPE]  | BasisType of the element (0-GaussLegendre, 1-GaussLobatto, 2-Bernstein, 3-OpenUniform, 4-CloseUniform, 5-OpenHalfUniform 6-Serendipity 7-ClosedGL 8-IntegratedGLL) |
    | [OBTYPE] | Open BasisType of the element for elements which have both types |
    | [CBTYPE] | Closed BasisType of the element for elements which have both types |
 

fem/fespace.cpp

@@ -282,14 +282,7 @@ int FiniteElementSpace::DofToVDof(int dof, int vd, int ndofs_) const
 void FiniteElementSpace::AdjustVDofs(Array<int> &vdofs)
 {
    int n = vdofs.Size(), *vdof = vdofs;
-   for (int i = 0; i < n; i++)
-   {
-      int j;
-      if ((j = vdof[i]) < 0)
-      {
-         vdof[i] = -1-j;
-      }
-   }
+   for (int i = 0; i < n; i++) { vdof[i] = UnsignIndex(vdof[i]); }
 }
 
 void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs,
@@ -483,13 +476,14 @@ void FiniteElementSpace::ReorderElementToDofTable()
    for (int k = 0, dof_counter = 0; k < nnz; k++)
    {
       const int sdof = J[k]; // signed dof
-      const int dof = (sdof < 0) ? -1-sdof : sdof;
+      const int dof = UnsignIndex(sdof);
       int new_dof = dof_marker[dof];
       if (new_dof < 0)
       {
          dof_marker[dof] = new_dof = dof_counter++;
       }
-      J[k] = (sdof < 0) ? -1-new_dof : new_dof; // preserve the sign of sdof
+      // Preserve the sign of sdof
+      J[k] = (sdof < 0) ? FlipIndexSign(new_dof) : new_dof;
    }
 }
 
@@ -547,7 +541,7 @@ void MarkDofs(const Array<int> &dofs, Array<int> &mark_array)
 {
    for (auto d : dofs)
    {
-      mark_array[d >= 0 ? d : -1 - d] = -1;
+      mark_array[UnsignIndex(d)] = -1;
    }
 }
 
@@ -931,7 +925,7 @@ void FiniteElementSpace::AddDependencies(
             if (std::abs(coef) > 1e-12)
             {
                const int mdof = master_dofs[j];
-               if (mdof != sdof && mdof != (-1-sdof))
+               if (mdof != sdof && mdof != FlipIndexSign(sdof))
                {
                   deps.Add(sdof, mdof, coef);
                }
@@ -1024,7 +1018,7 @@ int FiniteElementSpace::GetDegenerateFaceDofs(int index, Array<int> &dofs,
    // FiniteElementSpace::AddDependencies.
 
    Array<int> edof;
-   int order = GetEdgeDofs(-1 - index, edof, variant);
+   int order = GetEdgeDofs(FlipIndexSign(index), edof, variant);
 
    int nv = fec->DofForGeometry(Geometry::POINT);
    int ne = fec->DofForGeometry(Geometry::SEGMENT);
@@ -1516,36 +1510,76 @@ const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
    const bool is_dg_space = IsDGSpace();
    const L2FaceValues m = (is_dg_space && mul==L2FaceValues::DoubleValued) ?
                           L2FaceValues::DoubleValued : L2FaceValues::SingleValued;
-   key_face key = std::make_tuple(is_dg_space, f_ordering, type, m);
+   auto key = std::make_tuple(is_dg_space, f_ordering, type, m);
    auto itr = L2F.find(key);
    if (itr != L2F.end())
    {
-      return itr->second;
+      return itr->second.get();
    }
    else
    {
-      FaceRestriction *res;
+      std::unique_ptr<FaceRestriction> res;
       if (is_dg_space)
       {
          if (Conforming())
          {
-            res = new L2FaceRestriction(*this, f_ordering, type, m);
+            res.reset(new L2FaceRestriction(*this, f_ordering, type, m));
          }
          else
          {
-            res = new NCL2FaceRestriction(*this, f_ordering, type, m);
+            res.reset(new NCL2FaceRestriction(*this, f_ordering, type, m));
          }
       }
       else if (dynamic_cast<const DG_Interface_FECollection*>(fec))
       {
-         res = new L2InterfaceFaceRestriction(*this, f_ordering, type);
+         res.reset(new L2InterfaceFaceRestriction(*this, f_ordering, type));
       }
       else
       {
-         res = new ConformingFaceRestriction(*this, f_ordering, type);
+         res.reset(new ConformingFaceRestriction(*this, f_ordering, type));
       }
-      L2F[key] = res;
-      return res;
+      return L2F.emplace(key, std::move(res)).first->second.get();
+   }
+}
+
+const InterpolationManager &FiniteElementSpace::GetInterpolationManager(
+   ElementDofOrdering f_ordering, FaceType type) const
+{
+   const auto key = make_tuple(f_ordering, type);
+
+   auto it = interpolations.find(key);
+   if (it != interpolations.end())
+   {
+      return *it->second;
+   }
+   else
+   {
+      auto interp = make_unique<InterpolationManager>(*this, f_ordering, type);
+
+      int face_idx = 0;
+      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
+      {
+         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
+         if (!face.IsOfFaceType(type) || face.IsNonconformingCoarse())
+         {
+            continue;
+         }
+         if (face.IsConforming() || face.IsBoundary())
+         {
+            interp->RegisterFaceConformingInterpolation(face, face_idx);
+         }
+         else
+         {
+            interp->RegisterFaceCoarseToFineInterpolation(face, face_idx);
+         }
+         ++face_idx;
+      }
+
+      // Transform the interpolation matrix map into contiguous memory.
+      interp->LinearizeInterpolatorMapIntoVector();
+      interp->InitializeNCInterpConfig();
+
+      return *interpolations.emplace(key, std::move(interp)).first->second;
    }
 }
 
@@ -1670,8 +1704,8 @@ SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
 
          for (int i = 0; i < fine_ldof; i++)
          {
-            int r = DofToVDof(dofs[i], vd);
-            int m = (r >= 0) ? r : (-1 - r);
+            const int r = DofToVDof(dofs[i], vd);
+            const int m = UnsignIndex(r);
 
             if (!mark[m])
             {
@@ -1732,7 +1766,7 @@ SparseMatrix *FiniteElementSpace::VariableOrderRefinementMatrix(
          for (int i = 0; i < fine_ldof; i++)
          {
             const int r = DofToVDof(dofs[i], vd);
-            int m = (r >= 0) ? r : (-1 - r);
+            const int m = UnsignIndex(r);
 
             if (!mark[m])
             {
@@ -2442,8 +2476,8 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
          {
             if (!std::isfinite(lR(i, 0))) { continue; }
 
-            int r = DofToVDof(dofs[i], vd);
-            int m = (r >= 0) ? r : (-1 - r);
+            const int r = DofToVDof(dofs[i], vd);
+            const int m = UnsignIndex(r);
 
             if (is_dg || !mark[m])
             {
@@ -3161,7 +3195,7 @@ void FiniteElementSpace::CalcEdgeFaceVarOrders(
             else
             {
                // degenerate face (i.e., edge-face constraint)
-               slave_orders |= edge_orders[-1 - slave.index];
+               slave_orders |= edge_orders[FlipIndexSign(slave.index)];
             }
          }
 
@@ -3969,11 +4003,8 @@ void FiniteElementSpace::Destroy()
       delete E2Q_array[i];
    }
    E2Q_array.SetSize(0);
-   for (auto &x : L2F)
-   {
-      delete x.second;
-   }
    L2F.clear();
+   interpolations.clear();
    for (int i = 0; i < E2IFQ_array.Size(); i++)
    {
       delete E2IFQ_array[i];

fem/fespace.hpp

@@ -13,6 +13,7 @@
 #define MFEM_FESPACE
 
 #include "../config/config.hpp"
+#include "../general/hash_util.hpp"
 #include "../linalg/ordering.hpp"
 #include "../linalg/sparsemat.hpp"
 #include "../mesh/mesh.hpp"
@@ -320,18 +321,11 @@ protected:
    mutable OperatorHandle L2E_nat, L2E_lex;
    /// The face restriction operators, see GetFaceRestriction().
    using key_face = std::tuple<bool, ElementDofOrdering, FaceType, L2FaceValues>;
-   struct key_hash
-   {
-      std::size_t operator()(const key_face& k) const
-      {
-         return std::get<0>(k)
-                + 2 * (int)std::get<1>(k)
-                + 4 * (int)std::get<2>(k)
-                + 8 * (int)std::get<3>(k);
-      }
-   };
-   using map_L2F = std::unordered_map<const key_face,FaceRestriction*,key_hash>;
-   mutable map_L2F L2F;
+   mutable std::unordered_map<key_face,std::unique_ptr<FaceRestriction>,
+           TupleHasher> L2F;
+
+   mutable std::unordered_map<std::tuple<ElementDofOrdering,FaceType>,
+           std::unique_ptr<InterpolationManager>, TupleHasher> interpolations;
 
    mutable Array<QuadratureInterpolator*> E2Q_array;
    mutable Array<FaceQuadratureInterpolator*> E2IFQ_array;
@@ -751,6 +745,9 @@ public:
       ElementDofOrdering f_ordering, FaceType,
       L2FaceValues mul = L2FaceValues::DoubleValued) const;
 
+   const InterpolationManager &GetInterpolationManager(
+      ElementDofOrdering f_ordering, FaceType type) const;
+
    /** @brief Return a QuadratureInterpolator that interpolates E-vectors to
        quadrature point values and/or derivatives (Q-vectors). */
    /** An E-vector represents the element-wise discontinuous version of the FE
@@ -1153,7 +1150,7 @@ public:
 
    /// Helper to return the DOF associated with a sign encoded DOF
    static inline int DecodeDof(int dof)
-   { return (dof >= 0) ? dof : (-1 - dof); }
+   { return UnsignIndex(dof); }
 
    /// Helper to determine the DOF and sign of a sign encoded DOF
    static inline int DecodeDof(int dof, real_t& sign)

fem/gridfunc.hpp

@@ -27,6 +27,24 @@
 namespace mfem
 {
 
+/** This enumerated type describes the three main projection types:
+    - ELEMENT, assigns the degree of freedom per element, as specified in the
+      specific element
+    - GLOBAL_L2, solves a global L2 projection
+    - ELEMENT_L2, solves a element level L2 projection. Inter element
+      connectivity is dealt with similar as in:
+      Bezier-Projection : A unified approach for local projection and
+      quadrature-free refinement and coarsening of NURBS and T-splines with
+      particular application to isogeometric design and analysis
+      [CMAME (284) 2015 pg 55-105]
+    - DEFAULT, for NURBS spaces this is ELEMENT_L2, while for all other spaces
+      this ELEMENT.
+   Note 1: ELEMENT_L2 also works for non NURBS elements
+   Note 2: For NURBS elements the ELEMENT projection gives results without
+   over and undershoots. However, the gradient near the boundary does not
+   converge.*/
+enum class ProjectType { DEFAULT, ELEMENT, GLOBAL_L2, ELEMENT_L2 };
+
 /// Class for grid function - Vector with associated FE space.
 class GridFunction : public Vector
 {
@@ -66,13 +84,17 @@ protected:
        degree of freedom. */
    void ProjectDiscCoefficient(VectorCoefficient &coeff, Array<int> &dof_attr);
 
+   /** Helper function for ProjectCoefficientElementL2 */
+   void ProjectCoefficientElementL2_(Coefficient &coeff, Vector &sol, Vector &Va);
+   void ProjectCoefficientElementL2_(VectorCoefficient &vcoeff, Vector &sol,
+                                     Vector &Va);
+
    /// Loading helper.
    void LegacyNCReorder();
 
    void Destroy();
 
 public:
-
    GridFunction() { fes = NULL; fec_owned = NULL; fes_sequence = 0; UseDevice(true); }
 
    /// Copy constructor. The internal true-dof vector #t_vec is not copied.
@@ -84,6 +106,10 @@ public:
    GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
    { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
 
+   /// Same as above but specify the memory type
+   GridFunction(FiniteElementSpace *f, MemoryType mt) : Vector(f->GetVSize(), mt)
+   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+
    /// Construct a GridFunction using previously allocated array @a data.
    /** The GridFunction does not assume ownership of @a data which is assumed to
        be of size at least `f->GetVSize()`. Similar to the Vector constructor
@@ -420,9 +446,30 @@ public:
    /** @brief Project @a coeff Coefficient to @a this GridFunction. The
        projection computation depends on the choice of the FiniteElementSpace
        #fes. Note that this is usually interpolation at the degrees of freedom
-       in each element (not L2 projection). For NURBS spaces these degrees of
-       freedom are not available and L2 projection is resorted to as fallback. */
-   virtual void ProjectCoefficient(Coefficient &coeff);
+       in each element (not L2 projection). For elements without a projection
+       member function one could use ProjectCoefficientGlobalL2 instead.
+       NOTE: For parallel simulations with NURBS elements some dofs might
+       not be defined, if the evaluation point does not reside on this rank.
+       If that is the case it is defined on another rank, and the issue is
+       rectified with the appropriate communication, see in ParGridFunction.
+       */
+   virtual void ProjectCoefficient(Coefficient &coeff,
+                                   ProjectType type = ProjectType::DEFAULT);
+
+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection is a global L2 projection. This routine can be used a
+       fallback for elements without a projection member function.*/
+   virtual void ProjectCoefficientGlobalL2(Coefficient &coeff,
+                                           real_t rtol = 1e-12,
+                                           int iter = 1000);
+
+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection is an element local L2 projection, with an appropriate
+       weighting for Dofs that are shared between elements. Inspired on
+       Bezier-Projection [CMAME (284) 2015 pg 55-105]
+       This routine can be used a fallback for elements without a projection
+       member function.*/
+   virtual void ProjectCoefficientElementL2(Coefficient &coeff);
 
    /** @brief Project @a coeff Coefficient to @a this GridFunction, using one
        element for each degree of freedom in @a dofs and nodal interpolation on
@@ -432,9 +479,26 @@ public:
    /** @brief Project @a vcoeff VectorCoefficient to @a this GridFunction. The
        projection computation depends on the choice of the FiniteElementSpace
        #fes. Note that this is usually interpolation at the degrees of freedom
-       in each element (not L2 projection). For NURBS spaces these degrees of
-       freedom are not available and L2 projection is resorted to as fallback. */
-   void ProjectCoefficient(VectorCoefficient &vcoeff);
+       in each element (not L2 projection). For elements without a projection
+       member function one could use ProjectCoefficientGlobalL2 instead.
+       NOTE: For parallel simulations with NURBS elements some dofs might
+       not be defined, if the evaluation point does not reside on this rank.
+       If that is the case it is defined on another rank, and the issue is
+       rectified with the appropriate communication, see in ParGridFunction.*/
+   virtual void ProjectCoefficient(VectorCoefficient &vcoeff,
+                                   ProjectType type = ProjectType::DEFAULT);
+
+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection is a global L2 projection. This routine can be used a
+       fallback for elements without a projection member function.*/
+   virtual void ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
+                                           real_t rtol = 1e-12,
+                                           int iter = 1000);
+
+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection is a global L2 projection. This routine can be used a
+       fallback for elements without a projection member function.*/
+   virtual void ProjectCoefficientElementL2(VectorCoefficient &vcoeff);
 
    /** @brief Project @a vcoeff VectorCoefficient to @a this GridFunction, using
        one element for each degree of freedom in @a dofs and nodal interpolation
@@ -500,6 +564,70 @@ protected:
    /// P-refinement version of Update().
    void UpdatePRef();
 
+   /** @brief Estimate the minimum value of the GridFunction in element @a elem
+    *  if it is below a certain @a min_threshold.
+    *
+    *  @details For a given element \p elem and grid function component \p vdim
+    *  an estimate of the function minimum is the minimum of the piecewise
+    *  linear lower bound obtained using the given PLBound object. The actual
+    *  minimum is between [minimum lower bound, minimum upper bound]. We
+    *  improve the estimate of the function minimum by recursively
+    *  subdividing the interval with the lowest lower bound, and computing
+    *  bounds on the sub-intervals.
+    *  This process continues until (i) the maximum recursion depth is reached
+    *  or (ii) the difference between the minimum upper bound and minimum lower
+    *  bound is less than a certain tolerance (\p tol * [initial maximum
+    *  upper bound - initial minimum lower bound]).
+    *  The function also terminates if the lowest minima estimate is found
+    *  to be above the given threshold \p min_threshold. This is useful when
+    *  we are interested in computing the global minimum of the function
+    *  over all elements. In this case we can reject elements where the lowest
+    *  bound is above the current global minimum. In case the function
+    *  minimum on the element is below the global minimum, we update
+    *  \p min_threshold.
+    *
+    *  We return a pair of values that bracket the actual minimum, i.e.
+    *  [min_lower_bound, min_upper_bound].
+    */
+   std::pair<real_t,real_t> EstimateFunctionMinimum(const int elem,
+                                                    const PLBound &plb,
+                                                    const int vdim,
+                                                    const int max_depth,
+                                                    const real_t tol,
+                                                    real_t &min_threshold)const;
+
+   /** @brief Estimate the maximum value of the GridFunction in element @a elem
+    *  if it is below a certain @a max_threshold.
+    *
+    *  @details For a given element \p elem and grid function component \p vdim
+    *  an estimate of the function maximum is the maximum of the piecewise
+    *  linear upper bound obtained using the given PLBound object. The actual
+    *  maximum is between [maximum lower bound, maximum upper bound]. We
+    *  improve the estimate of the function maximum by recursively
+    *  subdividing the interval with the highest upper bound, and computing
+    *  bounds on the sub-intervals.
+    *  This process continues until (i) the maximum recursion depth is reached
+    *  or (ii) the difference between the maximum upper bound and maximum lower
+    *  bound is less than a certain tolerance (\p tol * [initial maximum
+    *  upper bound - initial maximum lower bound]).
+    *  The function also terminates if the highest maxima estimate is found
+    *  to be below the given threshold \p max_threshold. This is useful when
+    *  we are interested in computing the global maximum of the function
+    *  over all elements. In this case we can reject elements where the upper
+    *  bound is below the current global maximum. In case the function
+    *  maximum on the element is above the global maximum, we update
+    *  \p max_threshold.
+    *
+    *  We return a pair of values that bracket the actual maximum, i.e.
+    *  [max_lower_bound, max_upper_bound].
+    */
+   std::pair<real_t,real_t> EstimateFunctionMaximum(const int elem,
+                                                    const PLBound &plb,
+                                                    const int vdim,
+                                                    const int max_depth,
+                                                    const real_t tol,
+                                                    real_t &max_threshold)const;
+
 public:
    /** @brief For each vdof, counts how many elements contain the vdof,
        as containment is determined by FiniteElementSpace::GetElementVDofs(). */
@@ -1598,21 +1726,21 @@ public:
     */
    ///@{
    /// Computes the \ref PLBound for the gridfunction with number of control
-   /// points based on @a ref_factor, and returns the overall bounds for each
-   /// vdim (across all elements) in @b lower and @b upper. We also return the
+   /// points based on \p ref_factor, and returns the overall bounds for each
+   /// vdim (across all elements) in \p lower and \p upper. We also return the
    /// PLBound object used to compute the bounds.
-   /// We compute the bounds for each vdim if @a vdim < 1.
+   /// We compute the bounds for each vdim if \p vdim < 1.
    /// Note: For most cases, this method/interface will be sufficient.
    virtual PLBound GetBounds(Vector &lower, Vector &upper,
                              const int ref_factor=1, const int vdim=-1) const;
 
    /// Computes the \ref PLBound for the gridfunction with number of control
-   /// points based on @a ref_factor, and returns the bounds for each element
-   /// ordered byVDim:
+   /// points based on \p ref_factor, and returns the bounds for each element
+   /// ordered byNodes:
    /// lower_{0,0}, lower_{1,0}, ..., lower_{ne-1,0},
    /// lower_{0,1}, ..., lower_{ne-1,vdim-1}. We also return the
    /// PLBound object used to compute the bounds.
-   /// We compute the bounds for each vdim if @a vdim < 1.
+   /// We compute the bounds for each vdim if \p vdim < 1.
    PLBound GetElementBounds(Vector &lower, Vector &upper,
                             const int ref_factor=1, const int vdim=-1) const;
 
@@ -1623,6 +1751,18 @@ public:
                                         Vector &lower, Vector &upper,
                                         const int vdim = -1) const;
 
+   /** @brief Gets the bounds on given reference range inside an element.
+    *
+    *  @details @a ref_range is a vector of size 2*dim that specifies the
+    *  lower and upper limits in each dimension of the reference element.
+    *  For example, in 2D, ref_range = [rmin, smin, rmax, smax].
+    */
+   void GetElementBoundsAtControlPoints(const int elem, const PLBound &plb,
+                                        const Vector &ref_range,
+                                        const int vdim,
+                                        Vector &lower, Vector &upper,
+                                        Vector &control_pos) const;
+
    /// Compute bounds on the grid function for the given element.
    /// The bounds are stored in @b lower and @b upper.
    void GetElementBounds(const int elem, const PLBound &plb,
@@ -1630,11 +1770,45 @@ public:
                          const int vdim = -1) const;
 
    /// Compute bounds on the grid function for all the elements. The bounds
-   /// are returned in @b lower and @b upper, ordered byVDim:
+   /// are returned in @b lower and @b upper, ordered byNodes:
    /// lower_{0,0}, lower_{1,0}, ..., lower_{ne-1,0},
    /// lower_{0,1}, ..., lower_{ne-1,vdim-1}
    void GetElementBounds(const PLBound &plb, Vector &lower, Vector &upper,
                          const int vdim=-1) const;
+
+   /** @brief Estimate the minimum value of the GridFunction in element @a elem.
+    *
+    *  @details See the protected version of EstimateFunctionMinimum for
+    *  details.
+    */
+   std::pair<real_t, real_t> EstimateFunctionMinimum(const int elem,
+                                                     const PLBound &plb,
+                                                     const int vdim,
+                                                     const int max_depth,
+                                                     const real_t tol) const;
+
+   /** @brief Estimate the minimum value of the GridFunction in element @a elem.
+    *
+    *  @details See the protected version of EstimateFunctionMaximum for
+    *  details.
+    */
+   std::pair<real_t, real_t> EstimateFunctionMaximum(const int elem,
+                                                     const PLBound &plb,
+                                                     const int vdim,
+                                                     const int max_depth,
+                                                     const real_t tol) const;
+
+   /** @brief Estimate the GridFunction minimum across all elements. */
+   virtual std::pair<real_t,real_t> EstimateFunctionMinimum(const int vdim,
+                                                            const PLBound &plb,
+                                                            const int max_depth,
+                                                            const real_t tol) const;
+
+   /** @brief Estimate the GridFunction maximum across all elements. */
+   virtual std::pair<real_t,real_t> EstimateFunctionMaximum(const int vdim,
+                                                            const PLBound &plb,
+                                                            const int max_depth,
+                                                            const real_t tol) const;
    ///@}
 
    /// Destroys grid function.

fem/gslib.cpp

@@ -234,7 +234,7 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
 }
 
 void FindPointsGSLIB::FindPoints(const Vector &point_pos,
-                                 int point_pos_ordering)
+                                 const int point_pos_ordering)
 {
    MFEM_VERIFY(setupflag, "Use FindPointsGSLIB::Setup before finding points.");
    bool dev_mode = (point_pos.UseDevice() && Device::IsEnabled());
@@ -482,7 +482,7 @@ void FindPointsGSLIB::SetupDevice()
 }
 
 void FindPointsGSLIB::FindPointsOnDevice(const Vector &point_pos,
-                                         int point_pos_ordering)
+                                         const int point_pos_ordering)
 {
    if (!DEV.setup_device)
    {
@@ -505,13 +505,13 @@ void FindPointsGSLIB::FindPointsOnDevice(const Vector &point_pos,
 
    if (dim == 2)
    {
-      FindPointsLocal2(point_pos, point_pos_ordering, gsl_code, gsl_elem, gsl_ref,
-                       gsl_dist, points_cnt);
+      FindPointsLocal2(point_pos, point_pos_ordering, gsl_code, gsl_elem,
+                       gsl_ref, gsl_dist, points_cnt);
    }
    else
    {
-      FindPointsLocal3(point_pos, point_pos_ordering, gsl_code, gsl_elem, gsl_ref,
-                       gsl_dist, points_cnt);
+      FindPointsLocal3(point_pos, point_pos_ordering, gsl_code, gsl_elem,
+                       gsl_ref, gsl_dist, points_cnt);
    }
 
    // Sync from device to host
@@ -1085,7 +1085,7 @@ void FindPointsGSLIB::InterpolateOnDevice(const Vector &field_in_evec,
 #else
 void FindPointsGSLIB::SetupDevice() {};
 void FindPointsGSLIB::FindPointsOnDevice(const Vector &point_pos,
-                                         int point_pos_ordering) {};
+                                         const int point_pos_ordering) {};
 void FindPointsGSLIB::InterpolateOnDevice(const Vector &field_in_evec,
                                           Vector &field_out,
                                           const int nel, const int ncomp,
@@ -1094,7 +1094,8 @@ void FindPointsGSLIB::InterpolateOnDevice(const Vector &field_in_evec,
 #endif
 
 void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
-                                 int point_pos_ordering, const double bb_t,
+                                 const int point_pos_ordering,
+                                 const double bb_t,
                                  const double newt_tol, const int npt_max)
 {
    if (!setupflag || (mesh != &m) )
@@ -1105,16 +1106,28 @@ void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
 }
 
 void FindPointsGSLIB::Interpolate(const Vector &point_pos,
-                                  const GridFunction &field_in, Vector &field_out,
-                                  int point_pos_ordering)
+                                  const GridFunction &field_in,
+                                  Vector &field_out,
+                                  const int point_pos_ordering)
 {
    FindPoints(point_pos, point_pos_ordering);
    Interpolate(field_in, field_out);
 }
 
+void FindPointsGSLIB::Interpolate(const Vector &point_pos,
+                                  const GridFunction &field_in,
+                                  Vector &field_out,
+                                  const int point_pos_ordering,
+                                  const int field_out_ordering)
+{
+   FindPoints(point_pos, point_pos_ordering);
+   Interpolate(field_in, field_out, field_out_ordering);
+}
+
 void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
-                                  const GridFunction &field_in, Vector &field_out,
-                                  int point_pos_ordering)
+                                  const GridFunction &field_in,
+                                  Vector &field_out,
+                                  const int point_pos_ordering)
 {
    FindPoints(m, point_pos, point_pos_ordering);
    Interpolate(field_in, field_out);
@@ -1470,7 +1483,7 @@ void FindPointsGSLIB::SetupSplitMeshesAndIntegrationRules(const int order)
 }
 
 void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
-                                     Vector &node_vals)
+                                     Vector &node_vals) const
 {
    const GridFunction *nodes     = gf_in;
    const FiniteElementSpace *fes = nodes->FESpace();
@@ -1758,6 +1771,13 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
 
 void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
                                   Vector &field_out)
+{
+   Interpolate(field_in, field_out, field_in.FESpace()->GetOrdering());
+}
+
+void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
+                                  Vector &field_out,
+                                  const int field_out_ordering)
 {
    const int  gf_order   = field_in.FESpace()->GetMaxElementOrder(),
               mesh_order = mesh->GetNodalFESpace()->GetMaxElementOrder();
@@ -1800,7 +1820,7 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
       const int maxOrder = field_in.FESpace()->GetMaxElementOrder();
 
       InterpolateOnDevice(node_vals, field_out, NE_split_total, ncomp,
-                          maxOrder+1, field_in.FESpace()->GetOrdering());
+                          maxOrder+1, field_out_ordering);
       return;
 #endif
    }
@@ -1812,12 +1832,13 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
        field_in.FESpace()->IsVariableOrder() ==
        mesh->GetNodalFESpace()->IsVariableOrder())
    {
-      InterpolateH1(field_in, field_out);
+      InterpolateH1(field_in, field_out, field_out_ordering);
       return;
    }
    else
    {
-      InterpolateGeneral(field_in, field_out);
+      InterpolateGeneral(field_in, field_out,
+                         field_out_ordering);
       if (!fec_l2 || avgtype == AvgType::NONE) { return; }
    }
 
@@ -1861,11 +1882,11 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
 
       if (gf_order_h1 == mesh_order) // basis is GaussLobatto by default
       {
-         InterpolateH1(field_in_h1, field_out_l2);
+         InterpolateH1(field_in_h1, field_out_l2, field_out_ordering);
       }
       else
       {
-         InterpolateGeneral(field_in_h1, field_out_l2);
+         InterpolateGeneral(field_in_h1, field_out_l2, field_out_ordering);
       }
 
       // Copy interpolated values for the points on element border
@@ -1873,7 +1894,7 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
       {
          for (int i = 0; i < indl2.Size(); i++)
          {
-            int idx = field_in_h1.FESpace()->GetOrdering() == Ordering::byNODES?
+            int idx = field_out_ordering == Ordering::byNODES?
                       indl2[i] + j*points_cnt:
                       indl2[i]*ncomp + j;
             field_out(idx) = field_out_l2(idx);
@@ -1883,7 +1904,8 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
 }
 
 void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
-                                    Vector &field_out)
+                                    Vector &field_out,
+                                    const int field_out_ordering)
 {
    FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
    if (field_in.FESpace()->IsVariableOrder())
@@ -1913,7 +1935,8 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
                 dataptrout = i*points_cnt;
       if (field_in.FESpace()->GetOrdering() == Ordering::byNODES)
       {
-         field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
+         field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin,
+                                        points_fld);
       }
       else
       {
@@ -1945,7 +1968,7 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
                         (gslib::findpts_data_3 *)this->fdataD);
       }
    }
-   if (field_in.FESpace()->GetOrdering() == Ordering::byVDIM)
+   if (field_out_ordering == Ordering::byVDIM)
    {
       Vector field_out_temp = field_out;
       for (int i = 0; i < ncomp; i++)
@@ -1959,7 +1982,8 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
 }
 
 void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
-                                         Vector &field_out)
+                                         Vector &field_out,
+                                         const int field_out_ordering)
 {
    int ncomp   = field_in.VectorDim(),
        nptorig = points_cnt,
@@ -1979,7 +2003,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
          if (dim == 3) { ip.z = gsl_mfem_ref(index*dim + 2); }
          Vector localval(ncomp);
          field_in.GetVectorValue(gsl_mfem_elem[index], ip, localval);
-         if (field_in.FESpace()->GetOrdering() == Ordering::byNODES)
+         if (field_out_ordering == Ordering::byNODES)
          {
             for (int i = 0; i < ncomp; i++)
             {
@@ -2014,7 +2038,10 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
       for (int index = 0; index < npt; index++)
       {
          if (gsl_code[index] == 2) { continue; }
-         for (int d = 0; d < dim; ++d) { pt->r[d]= gsl_mfem_ref(index*dim + d); }
+         for (int d = 0; d < dim; ++d)
+         {
+            pt->r[d]= gsl_mfem_ref(index*dim + d);
+         }
          pt->index = index;
          pt->proc  = gsl_proc[index];
          pt->el    = gsl_mfem_elem[index];
@@ -2104,7 +2131,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
             sdpt = (struct send_pt *)sendpt->ptr;
             for (int index = 0; index < static_cast<int>(sendpt->n); index++)
             {
-               int idx = field_in.FESpace()->GetOrdering() == Ordering::byNODES ?
+               int idx = field_out_ordering == Ordering::byNODES ?
                          sdpt->index + j*nptorig :
                          sdpt->index*ncomp + j;
                field_out(idx) = sdpt->ival;
@@ -2246,7 +2273,7 @@ void FindPointsGSLIB::DistributeInterpolatedValues(const Vector &int_vals,
    }
 }
 
-void FindPointsGSLIB::GetAxisAlignedBoundingBoxes(Vector &aabb)
+void FindPointsGSLIB::GetAxisAlignedBoundingBoxes(Vector &aabb) const
 {
    MFEM_VERIFY(setupflag, "Call FindPointsGSLIB::Setup method first");
    auto *findptsData3 = (gslib::findpts_data_3 *)this->fdataD;
@@ -2317,7 +2344,7 @@ void FindPointsGSLIB::GetAxisAlignedBoundingBoxes(Vector &aabb)
 }
 
 void FindPointsGSLIB::GetOrientedBoundingBoxes(DenseTensor &obbA, Vector &obbC,
-                                               Vector &obbV)
+                                               Vector &obbV) const
 {
    MFEM_VERIFY(setupflag, "Call FindPointsGSLIB::Setup method first");
    auto *findptsData3 = (gslib::findpts_data_3 *)this->fdataD;
@@ -2502,8 +2529,8 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
 }
 
 void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
-                                        Array<unsigned int> &point_id,
-                                        int point_pos_ordering)
+                                        const Array<unsigned int> &point_id,
+                                        const int point_pos_ordering)
 {
    MFEM_VERIFY(setupflag, "Use OversetFindPointsGSLIB::Setup before "
                "finding points.");
@@ -2582,10 +2609,10 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
 }
 
 void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
-                                         Array<unsigned int> &point_id,
+                                         const Array<unsigned int> &point_id,
                                          const GridFunction &field_in,
                                          Vector &field_out,
-                                         int point_pos_ordering)
+                                         const int point_pos_ordering)
 {
    FindPoints(point_pos, point_id, point_pos_ordering);
    Interpolate(field_in, field_out);

fem/gslib.hpp

@@ -119,11 +119,13 @@ protected:
    } DEV;
 
    /// Use GSLIB for communication and interpolation
-   virtual void InterpolateH1(const GridFunction &field_in, Vector &field_out);
+   virtual void InterpolateH1(const GridFunction &field_in, Vector &field_out,
+                              const int field_out_ordering);
    /// Uses GSLIB Crystal Router for communication followed by MFEM's
    /// interpolation functions
    virtual void InterpolateGeneral(const GridFunction &field_in,
-                                   Vector &field_out);
+                                   Vector &field_out,
+                                   const int field_out_ordering);
 
    /// Since GSLIB is designed to work with quads/hexes, we split every
    /// triangle/tet/prism/pyramid element into quads/hexes.
@@ -140,7 +142,7 @@ protected:
    virtual void SetupSplitMeshesAndIntegrationRules(const int order);
 
    /// Get GridFunction value at the points expected by GSLIB.
-   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals);
+   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals) const;
 
    /// Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For simplices,
    /// find the original element number (that was split into micro quads/hexes)
@@ -182,7 +184,7 @@ protected:
        These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
        byVDim: (XYZ,XYZ,....XYZ) specified by @a point_pos_ordering. */
    void FindPointsOnDevice(const Vector &point_pos,
-                           int point_pos_ordering = Ordering::byNODES);
+                           const int point_pos_ordering = Ordering::byNODES);
 
    /** Interpolation of field values at prescribed reference space positions.
        @param[in] field_in_evec E-vector of grid function to be interpolated.
@@ -253,10 +255,15 @@ public:
        #gsl_dist        Distance between the sought and the found point
                         in physical space. */
    void FindPoints(const Vector &point_pos,
-                   int point_pos_ordering = Ordering::byNODES);
+                   const int point_pos_ordering = Ordering::byNODES);
+   /// Convenience function when point positions are in a ParticleVector
+   void FindPoints(const ParticleVector &point_pos)
+   {
+      FindPoints(point_pos, point_pos.GetOrdering());
+   }
    /// Setup FindPoints and search positions
    void FindPoints(Mesh &m, const Vector &point_pos,
-                   int point_pos_ordering = Ordering::byNODES,
+                   const int point_pos_ordering = Ordering::byNODES,
                    const double bb_t = 0.1, const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
@@ -266,20 +273,28 @@ public:
                               \p field_in is in H1 and in the same space as the
                               mesh that was given to Setup().
        @param[out] field_out  Interpolated values. For points that are not found
-                              the value is set to #default_interp_value. */
+                              the value is set to #default_interp_value.
+                              The output ordering is determined from field_in.*/
    virtual void Interpolate(const GridFunction &field_in, Vector &field_out);
+   /// Interpolation of field values, with output ordering specification.
+   virtual void Interpolate(const GridFunction &field_in, Vector &field_out,
+                            const int field_out_ordering);
    /** Search positions and interpolate. The ordering (byNODES or byVDIM) of
        the output values in \p field_out corresponds to the ordering used
        in the input GridFunction \p field_in. */
    void Interpolate(const Vector &point_pos, const GridFunction &field_in,
                     Vector &field_out,
-                    int point_pos_ordering = Ordering::byNODES);
+                    const int point_pos_ordering = Ordering::byNODES);
+   /// Search positions and interpolate with given point and output ordering.
+   void Interpolate(const Vector &point_pos, const GridFunction &field_in,
+                    Vector &field_out, const int point_pos_ordering,
+                    const int field_out_ordering);
    /** Setup FindPoints, search positions and interpolate. The ordering (byNODES
        or byVDIM) of the output values in \p field_out corresponds to the
        ordering used in the input GridFunction \p field_in. */
    void Interpolate(Mesh &m, const Vector &point_pos,
                     const GridFunction &field_in, Vector &field_out,
-                    int point_pos_ordering = Ordering::byNODES);
+                    const int point_pos_ordering = Ordering::byNODES);
 
    /// Average type to be used for L2 functions in-case a point is located at
    /// an element boundary where the function might be multi-valued.
@@ -376,7 +391,7 @@ public:
    /// The size of the returned vector is (nel x nverts x dim), where nel is the
    /// number of elements (after splitting for simplcies), nverts is number of
    /// vertices (4 in 2D, 8 in 3D), and dim is the spatial dimension.
-   void GetAxisAlignedBoundingBoxes(Vector &aabb);
+   void GetAxisAlignedBoundingBoxes(Vector &aabb) const;
 
    /// Return the oriented bounding boxes (OBB) computed during \ref Setup.
    /// Each OBB is represented using the inverse transformation (A^{-1}) and
@@ -386,7 +401,8 @@ public:
    /// size (dim x dim x nel), and the OBB centers are returned in \p obbC,
    /// a vector of size (nel x dim). The vertices of the OBBs are returned in
    /// \p obbV, a vector of size (nel x nverts x dim) .
-   void GetOrientedBoundingBoxes(DenseTensor &obbA, Vector &obbC, Vector &obbV);
+   void GetOrientedBoundingBoxes(DenseTensor &obbA, Vector &obbC,
+                                 Vector &obbV) const;
 };
 
 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
@@ -446,13 +462,14 @@ public:
                                        byNodes: (XXX...,YYY...,ZZZ) or
                                        byVDim: (XYZ,XYZ,....XYZ) */
    void FindPoints(const Vector &point_pos,
-                   Array<unsigned int> &point_id,
-                   int point_pos_ordering = Ordering::byNODES);
+                   const Array<unsigned int> &point_id,
+                   const int point_pos_ordering = Ordering::byNODES);
 
    /** Search positions and interpolate */
-   void Interpolate(const Vector &point_pos, Array<unsigned int> &point_id,
+   void Interpolate(const Vector &point_pos,
+                    const Array<unsigned int> &point_id,
                     const GridFunction &field_in, Vector &field_out,
-                    int point_pos_ordering = Ordering::byNODES);
+                    const int point_pos_ordering = Ordering::byNODES);
    using FindPointsGSLIB::Interpolate;
 };
 

fem/hybridization.cpp

@@ -789,7 +789,6 @@ void Hybridization::ComputeH()
    }
    else
    {
-      // TODO: add ones on the diagonal of zero rows
       V->Finalize();
       Array<HYPRE_BigInt> V_J(V->NumNonZeroElems());
       MFEM_ASSERT(c_pfes, "");
@@ -823,6 +822,13 @@ void Hybridization::ComputeH()
       MFEM_VERIFY(pH.Type() != Operator::PETSC_MATIS, "To be implemented");
       pH.MakePtAP(plpH, pP);
       delete lpH;
+
+      HypreParMatrix *hH = pH.As<HypreParMatrix>();
+      MFEM_ASSERT(hH, "");
+
+      SparseMatrix H_diag;
+      hH->GetDiag(H_diag);
+      H_diag.SetDiagIdentity();
    }
 #endif
 }

fem/hybridization_ext.hpp

@@ -14,8 +14,11 @@
 
 #include "../config/config.hpp"
 #include "../general/array.hpp"
+#include "../linalg/operator.hpp"
 #include "../linalg/vector.hpp"
 
+#include <memory>
+
 namespace mfem
 {
 
@@ -45,15 +48,30 @@ protected:
    Array<int> hat_dof_gather_map;
    Array<DofType> hat_dof_marker;
 
-   Array<int> el_to_face;
-   Array<int> face_to_el;
+   Array<int> el_to_face; ///< Element to face connectivity.
+   Array<int> el_face_offsets; ///< Per-element offsets into @a el_to_face.
+   Array<int> face_to_el; ///< Face-to-element connectivity.
+   Array<int> face_face_offsets; ///< Face-to-face offsets.
+
+   int n_el_face; ///< Total number of element-to-face connections.
+   int n_face_face; ///< Total number of face-to-face connections.
+
    Vector Ct_mat; ///< Constraint matrix (transposed) stored element-wise.
 
+   /// @name For parallel non-conforming meshes
+   ///@{
+   std::unique_ptr<Operator> P_pc; ///< Partially conforming prolongation.
+   std::unique_ptr<Operator> P_nbr; ///< Face-neighbor prolongation.
+   ///@}
+
    Array<int> idofs, bdofs;
 
    Vector Ahat, Ahat_ii, Ahat_ib, Ahat_bi, Ahat_bb;
    Array<int> Ahat_ii_piv, Ahat_bb_piv;
 
+   /// Return the (partially) conforming prolongation on the constraint space.
+   const Operator &GetProlongation() const;
+
 public:
    /// Construct the constraint matrix.
    void ConstructC();

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -1004,13 +1004,16 @@ inline void SmemPADiffusionApply3D(const int NE,
    const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
    MFEM_VERIFY(D1D <= max_d1d, "");
    MFEM_VERIFY(Q1D <= max_q1d, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
+   const auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
    MFEM_VERIFY(D1D <= Q1D, "THREAD_DIRECT requires D1D <= Q1D");
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+
+   mfem::forall_3D<T_Q1D*T_Q1D*T_Q1D>(NE,
+                                      Q1D, Q1D, Q1D,
+                                      [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/integ/bilininteg_mass_kernels.hpp

@@ -1133,11 +1133,11 @@ inline void SmemPAMassApply3D(const int NE,
    const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
    MFEM_VERIFY(D1D <= max_d1d, "");
    MFEM_VERIFY(Q1D <= max_q1d, "");
-   auto b = b_.Read();
-   auto d = d_.Read();
-   auto x = x_.Read();
+   const auto b = b_.Read();
+   const auto d = d_.Read();
+   const auto x = x_.Read();
    auto y = y_.ReadWrite();
-   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
    });
@@ -1156,8 +1156,8 @@ inline void EAMassAssemble1D(const int NE,
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, NE);
+   const auto B = Reshape(basis.Read(), Q1D, D1D);
+   const auto D = Reshape(padata.Read(), Q1D, NE);
    auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(), D1D, D1D, NE);
    mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {

fem/integ/bilininteg_trace_jump_ea.cpp

@@ -28,7 +28,7 @@ void NormalTraceJumpIntegrator::AssembleEAInteriorFaces(
    const FaceType ftype = FaceType::Interior;
    const int nf = mesh.GetNFbyType(ftype);
 
-   const Geometry::Type geom = mesh.GetFaceGeometry(0);
+   const Geometry::Type geom = mesh.GetTypicalFaceGeometry();
    const int trial_order = trial_fes.GetMaxElementOrder();
    const int test_order = test_fes.GetMaxElementOrder();
    const int qorder = test_order + trial_order - 1;
@@ -47,7 +47,7 @@ void NormalTraceJumpIntegrator::AssembleEAInteriorFaces(
       });
    }
 
-   const FiniteElement &trial_face_el = *trial_fes.GetFaceElement(0);
+   const FiniteElement &trial_face_el = *trial_fes.GetTypicalTraceElement();
    const auto maps = &trial_face_el.GetDofToQuad(ir, DofToQuad::TENSOR);
    const int ndof_face = trial_face_el.GetDof();
 
@@ -72,7 +72,7 @@ void NormalTraceJumpIntegrator::AssembleEAInteriorFaces(
       MFEM_ABORT("Unknown kernel.");
    }
 
-   const FiniteElement &test_el = *test_fes.GetFE(0);
+   const FiniteElement &test_el = *test_fes.GetTypicalFE();
    const int n_faces_per_el = 2*dim; // assuming tensor product
    // Get all the local face maps (mapping from lexicographic face index to
    // lexicographic volume index, depending on the local face index).
@@ -90,10 +90,10 @@ void NormalTraceJumpIntegrator::AssembleEAInteriorFaces(
    Array<int> face_info(nf * 4);
    {
       int fidx = 0;
-      for (int f = 0; f < mesh.GetNumFaces(); ++f)
+      for (int f = 0; f < mesh.GetNumFacesWithGhost(); ++f)
       {
          Mesh::FaceInformation finfo = mesh.GetFaceInformation(f);
-         if (!finfo.IsInterior()) { continue; }
+         if (!finfo.IsInterior() || finfo.IsNonconformingCoarse()) { continue; }
          face_info[0 + fidx*4] = finfo.element[0].local_face_id;
          face_info[1 + fidx*4] = finfo.element[0].orientation;
          face_info[2 + fidx*4] = finfo.element[1].local_face_id;
@@ -114,7 +114,7 @@ void NormalTraceJumpIntegrator::AssembleEAInteriorFaces(
    else
    {
       d_emat = emat.Write();
-      mfem::forall(emat.Size(), [=] MFEM_HOST_DEVICE (int i) { d_emat[i] = 0.0; });
+      emat = 0.0; // Will execute on device, since Write() sets the device flag
    }
 
    const auto face_mats = Reshape(mass_emat.Read(), ndof_face, ndof_face, nf);
@@ -133,26 +133,104 @@ void NormalTraceJumpIntegrator::AssembleEAInteriorFaces(
       }
    };
 
-   mfem::forall_3D(nf, ndof_face, ndof_face, 2, [=] MFEM_HOST_DEVICE (int f)
+   auto permute_face_2 = [=] MFEM_HOST_DEVICE(int local_face_1, int local_face_2,
+                                              int orient, int size1d, int index)
    {
-      MFEM_FOREACH_THREAD(el_i, z, 2)
+      if (dim == 2)
       {
-         const int lf_i = d_face_info(0, el_i, f);
-         const int orient = d_face_info(1, el_i, f);
-         // Loop over face indices in "native ordering"
-         MFEM_FOREACH_THREAD(i_lex, x, ndof_face)
+         return internal::PermuteFace2D(local_face_1, local_face_2, orient,
+                                        size1d, index);
+      }
+      else // dim == 3
+      {
+         return internal::PermuteFace3D(local_face_1, local_face_2, orient,
+                                        size1d, index);
+      }
+   };
+
+   if (mesh.Conforming())
+   {
+      mfem::forall_3D(nf, ndof_face, ndof_face, 2, [=] MFEM_HOST_DEVICE (int f)
+      {
+         MFEM_FOREACH_THREAD(el_i, z, 2)
          {
-            // Convert to lexicographic relative to the face itself
-            const int i_face = permute_face(lf_i, orient, d1d, i_lex);
-            // Convert from lexicographic face DOF to volume DOF
-            const int i = d_face_maps(i_lex, lf_i);
-            MFEM_FOREACH_THREAD(j, y, ndof_face)
+            const int lf_i = d_face_info(0, el_i, f);
+            const int orient = d_face_info(1, el_i, f);
+            // Loop over face indices in "native ordering"
+            MFEM_FOREACH_THREAD(i_lex, x, ndof_face)
             {
-               el_mats(i, j, el_i, f) += face_mats(i_face, j, f);
+               // Convert to lexicographic relative to the face itself
+               const int i_face = permute_face(lf_i, orient, d1d, i_lex);
+               // Convert from lexicographic face DOF to volume DOF
+               const int i = d_face_maps(i_lex, lf_i);
+               MFEM_FOREACH_THREAD(j, y, ndof_face)
+               {
+                  el_mats(i, j, el_i, f) += face_mats(i_face, j, f);
+               }
             }
          }
-      }
-   });
+      });
+   }
+   else
+   {
+      const InterpolationManager &interp =
+         test_fes.GetInterpolationManager(ElementDofOrdering::LEXICOGRAPHIC, ftype);
+
+      auto interp_configs = interp.GetFaceInterpConfig().Read();
+      const int nc_size = interp.GetNumInterpolators();
+      auto d_interp = Reshape(interp.GetInterpolators().Read(),
+                              ndof_face, ndof_face, nc_size);
+
+      mfem::forall(nf, [=] MFEM_HOST_DEVICE (int f)
+      {
+         const InterpConfig conf = interp_configs[f];
+         const int master_side = conf.master_side;
+         const int interp_index = conf.index;
+
+         const int lf_0 = d_face_info(0, 0, f);
+
+         for (int el_i = 0; el_i < 2; ++el_i)
+         {
+            const int lf_i = d_face_info(0, el_i, f);
+            const int orient = d_face_info(1, el_i, f);
+
+            for (int j = 0; j < ndof_face; j++)
+            {
+               for (int i_lex = 0; i_lex < ndof_face; i_lex++)
+               {
+                  real_t val = 0.0;
+                  if (conf.is_non_conforming && el_i == master_side)
+                  {
+                     // Interpolate from el_i (coarse element) to the fine face.
+                     // The mapping is given by d_interp, which uses indices
+                     // relative to element 0.
+
+                     // i0 is lexicographic relative to element 0
+                     const int i0 = permute_face_2(lf_i, lf_0, orient, d1d, i_lex);
+
+                     // k0 is lexicographic relative to element 0
+                     for (int k0 = 0; k0 < ndof_face; k0++)
+                     {
+                        // k is relative to the face itself
+                        const int k = permute_face(lf_0, orient, d1d, k0);
+                        val += d_interp(k0, i0, interp_index)
+                               * face_mats(k, j, f);
+                     }
+                  }
+                  else
+                  {
+                     // Convert to lexicographic relative to the face itself
+                     const int i_face = permute_face(lf_i, orient, d1d, i_lex);
+                     val = face_mats(i_face, j, f);
+                  }
+                  // Convert from lexicographic face DOF to volume DOF
+                  const int i = d_face_maps(i_lex, lf_i);
+                  el_mats(i, j, el_i, f) += val;
+               }
+            }
+         }
+      });
+   }
 }
 
 }

fem/integ/bilininteg_vecdiffusion_pa.hpp

@@ -54,7 +54,7 @@ void SmemPAVectorDiffusionApply2D(const int NE,
    const auto XE = Reshape(x.Read(), D1D, D1D, SDIM, NE);
    auto YE = Reshape(y.ReadWrite(), D1D, D1D, SDIM, NE);
 
-   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
    {
       constexpr int MD1 = T_D1D > 0 ? SetMaxOf(T_D1D) : DofQuadLimits::MAX_T1D;
       constexpr int MQ1 = T_Q1D > 0 ? SetMaxOf(T_Q1D) : DofQuadLimits::MAX_T1D;
@@ -120,7 +120,7 @@ void SmemPAVectorDiffusionApply3D(const int NE,
    const auto XE = Reshape(x.Read(), D1D, D1D, D1D, SDIM, NE);
    auto YE = Reshape(y.ReadWrite(), D1D, D1D, D1D, SDIM, NE);
 
-   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
    {
       constexpr int MD1 = T_D1D > 0 ? SetMaxOf(T_D1D) : DofQuadLimits::MAX_T1D;
       constexpr int MQ1 = T_Q1D > 0 ? SetMaxOf(T_Q1D) : DofQuadLimits::MAX_T1D;
@@ -171,15 +171,15 @@ template<int DIM, int T_SDIM, int T_D1D, int T_Q1D>
 VectorDiffusionIntegrator::ApplyKernelType
 VectorDiffusionIntegrator::ApplyPAKernels::Kernel()
 {
-   if (DIM == 2)
+   if constexpr (DIM == 2)
    {
       return internal::SmemPAVectorDiffusionApply2D<T_SDIM, T_D1D, T_Q1D>;
    }
-   else if (DIM == 3)
+   else if constexpr (DIM == 3)
    {
       return internal::SmemPAVectorDiffusionApply3D<T_SDIM, T_D1D, T_Q1D>;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 inline VectorDiffusionIntegrator::ApplyKernelType

fem/integ/bilininteg_vecmass_pa.hpp

@@ -51,7 +51,7 @@ void SmemPAVectorMassApply2D(const int NE,
    const auto X = Reshape(x.Read(), D1D, D1D, VDIM, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, D1D, VDIM, NE);
 
-   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
    {
       constexpr int MD1 = T_D1D > 0 ? SetMaxOf(T_D1D) : DofQuadLimits::MAX_T1D;
       constexpr int MQ1 = T_Q1D > 0 ? SetMaxOf(T_Q1D) : DofQuadLimits::MAX_T1D;
@@ -119,7 +119,7 @@ void SmemPAVectorMassApply3D(const int NE,
    const auto X = Reshape(x.Read(), D1D, D1D, D1D, VDIM, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
 
-   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
    {
       constexpr int MD1 = T_D1D > 0 ? SetMaxOf(T_D1D) : DofQuadLimits::MAX_T1D;
       constexpr int MQ1 = T_Q1D > 0 ? SetMaxOf(T_Q1D) : DofQuadLimits::MAX_T1D;
@@ -182,15 +182,15 @@ template<int DIM, int T_D1D, int T_Q1D>
 VectorMassIntegrator::VectorMassAddMultPAType
 VectorMassIntegrator::VectorMassAddMultPA::Kernel()
 {
-   if (DIM == 2)
+   if constexpr (DIM == 2)
    {
       return internal::SmemPAVectorMassApply2D<T_D1D,T_Q1D>;
    }
-   else if (DIM == 3)
+   else if constexpr (DIM == 3)
    {
       return internal::SmemPAVectorMassApply3D<T_D1D, T_Q1D>;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType

fem/integ/lininteg_domain_kernels.hpp

@@ -301,18 +301,14 @@ template <int DIM, int T_D1D, int T_Q1D>
 DomainLFIntegrator::AssembleKernelType
 DomainLFIntegrator::AssembleKernels::Kernel()
 {
-   switch (DIM)
-   {
-      case 1:
-         return DLFEvalAssemble1D<T_D1D, T_Q1D>;
-      case 2:
-         return DLFEvalAssemble2D<T_D1D, T_Q1D>;
-      case 3:
-         return DLFEvalAssemble3D<T_D1D, T_Q1D>;
-   }
+   if constexpr (DIM == 1) { return DLFEvalAssemble1D<T_D1D, T_Q1D>; }
+   if constexpr (DIM == 2) { return DLFEvalAssemble2D<T_D1D, T_Q1D>; }
+   if constexpr (DIM == 3) { return DLFEvalAssemble3D<T_D1D, T_Q1D>; }
    MFEM_ABORT("");
 }
+
 /// \endcond DO_NOT_DOCUMENT
 
 } // namespace mfem
-#endif
+
+#endif // MFEM_LININTEG_DOMAIN_KERNELS_HPP

fem/kernel_dispatch.hpp

@@ -14,6 +14,7 @@
 
 #include "../config/config.hpp"
 #include "kernel_reporter.hpp"
+#include "../general/hash_util.hpp"
 #include <unordered_map>
 #include <tuple>
 #include <type_traits>
@@ -86,35 +87,6 @@ namespace mfem
     }                                                                          \
   }
 
-/// @brief Hashes variadic packs for which each type contained in the variadic
-/// pack has a specialization of `std::hash` available.
-///
-/// For example, packs containing int, bool, enum values, etc.
-template<typename ...KernelParameters>
-struct KernelDispatchKeyHash
-{
-private:
-   template<int N>
-   size_t operator()(std::tuple<KernelParameters...> value) const { return 0; }
-
-   // The hashing formula here is taken directly from the Boost library, with
-   // the magic number 0x9e3779b9 chosen to minimize hashing collisions.
-   template<std::size_t N, typename THead, typename... TTail>
-   size_t operator()(std::tuple<KernelParameters...> value) const
-   {
-      constexpr int Index = N - sizeof...(TTail) - 1;
-      auto lhs_hash = std::hash<THead>()(std::get<Index>(value));
-      auto rhs_hash = operator()<N, TTail...>(value);
-      return lhs_hash^(rhs_hash + 0x9e3779b9 + (lhs_hash<<6) + (lhs_hash>>2));
-   }
-public:
-   /// Returns the hash of the given @a value.
-   size_t operator()(std::tuple<KernelParameters...> value) const
-   {
-      return operator()<sizeof...(KernelParameters),KernelParameters...>(value);
-   }
-};
-
 namespace internal { template<typename... Types> struct KernelTypeList { }; }
 
 template<typename... T> class KernelDispatchTable { };
@@ -128,8 +100,8 @@ class KernelDispatchTable<Kernels,
          internal::KernelTypeList<Params...>,
          internal::KernelTypeList<OptParams...>>
 {
-   using TableType = std::unordered_map<std::tuple<Params...>,
-         Signature, KernelDispatchKeyHash<Params...>>;
+   using TableType =
+      std::unordered_map<std::tuple<Params...>, Signature, TupleHasher>;
    TableType table;
 
    /// @brief Call function @a f with arguments @a args (perfect forwaring).

fem/lor/lor.cpp

@@ -158,15 +158,16 @@ void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
                int i;
                i = dofmap_lor[off_lor + i1 + i2*2];
                int s1 = i < 0 ? -1 : 1;
-               int idof_lor = vdof_lor[absdof(i)];
+               int idof_lor = vdof_lor[UnsignIndex(i)];
                i = dofmap_ho[off_ho + i1*n1 + i2*n2];
                int s2 = i < 0 ? -1 : 1;
-               int idof_ho = vdof_ho[absdof(i)];
+               int idof_ho = vdof_ho[UnsignIndex(i)];
                int s3 = idof_lor < 0 ? -1 : 1;
                int s4 = idof_ho < 0 ? -1 : 1;
                int s = s1*s2*s3*s4;
-               i = absdof(idof_ho);
-               perm_[absdof(idof_lor)] = s < 0 ? -1-absdof(i) : absdof(i);
+               i = UnsignIndex(idof_ho);
+               perm_[UnsignIndex(idof_lor)] = s < 0 ? -1-UnsignIndex(i) :
+                                              UnsignIndex(i);
             }
          }
       };
@@ -232,7 +233,7 @@ void LORBase::ConstructDofPermutation() const
          int j = l_perm[i];
          int s = j < 0 ? -1 : 1;
          int t_i = pfes_lor->GetLocalTDofNumber(i);
-         int t_j = pfes_ho->GetLocalTDofNumber(absdof(j));
+         int t_j = pfes_ho->GetLocalTDofNumber(UnsignIndex(j));
          // Either t_i and t_j both -1, or both non-negative
          if ((t_i < 0 && t_j >=0) || (t_j < 0 && t_i >= 0))
          {

fem/lor/lor.hpp

@@ -57,8 +57,6 @@ private:
    /// values (after temporarily changing them for LOR assembly).
    void ResetIntegrationRules(GetIntegratorsFn get_integrators);
 
-   static inline int absdof(int i) { return i < 0 ? -1-i : i; }
-
 protected:
    enum FESpaceType { H1, ND, RT, L2, INVALID };
 

fem/lor/lor_dg.hpp

@@ -23,6 +23,8 @@ class BatchedLOR_DG : BatchedLORKernel
 {
    IntegrationRule ir_face; ///< Collocated Gauss-Lobatto face quadrature rule.
    real_t kappa; ///< DG penalty parameter.
+   bool has_bdr_integ; ///< Is there a boundary integrator?
+   const Array<int> *bdr_markers; ///< Boundary integrator markers.
 public:
    template <int ORDER, int SDIM> void Assemble2D();
    template <int ORDER> void Assemble3D();
@@ -38,8 +40,7 @@ public:
       ProjectLORCoefficient<MassIntegrator>(a, c1);
       ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
 
-      auto *integ = GetInteriorFaceIntegrator<DGDiffusionIntegrator>(a);
-      if (integ)
+      if (auto *integ = GetInteriorFaceIntegrator<DGDiffusionIntegrator>(a))
       {
          kappa = integ->GetPenaltyParameter();
       }
@@ -47,6 +48,19 @@ public:
       {
          kappa = 0.0;
       }
+
+      has_bdr_integ = false;
+      auto *bdr_face_integs = a.GetBFBFI();
+      for (int i = 0; i < bdr_face_integs->Size(); ++i)
+      {
+         if (auto *integ = dynamic_cast<DGDiffusionIntegrator*>((*bdr_face_integs)[i]))
+         {
+            kappa = integ->GetPenaltyParameter();
+            bdr_markers = (*a.GetBFBFI_Marker())[i];
+            has_bdr_integ = true;
+            break;
+         }
+      }
    }
 
    /// @brief Compute and return the face info array.

fem/lor/lor_dg_impl.hpp

@@ -22,9 +22,13 @@ namespace mfem
 Array<int> BatchedLOR_DG::GetFaceInfo() const
 {
    Mesh &mesh = *fes_ho.GetMesh();
+   const Array<int> &bdr_face_attrs = mesh.GetBdrFaceAttributes();
    const int nf = mesh.GetNumFaces();
    Array<int> face_info(nf * 6); // (e0, f0, o0, e1, f1, o1)
    auto h_face_info = Reshape(face_info.HostWrite(), 6, nf);
+
+   int bdr_face_counter = 0;
+
    for (int f = 0; f < nf; ++f)
    {
       auto finfo = mesh.GetFaceInformation(f);
@@ -43,6 +47,19 @@ Array<int> BatchedLOR_DG::GetFaceInfo() const
          h_face_info(4, f) = -1;
          h_face_info(5, f) = -1;
       }
+
+      if (finfo.IsBoundary())
+      {
+         // Check if Neumann boundary; skip these when adding boundary penalties
+         const int bdr_attr = bdr_face_attrs[bdr_face_counter];
+         if (!has_bdr_integ || (bdr_markers && !(*bdr_markers)[bdr_attr - 1]))
+         {
+            h_face_info(0, f) = -1;
+            h_face_info(1, f) = -1;
+            h_face_info(2, f) = -1;
+         }
+         bdr_face_counter += 1;
+      }
    }
    return face_info;
 }
@@ -144,6 +161,7 @@ void BatchedLOR_DG::AssembleFaceTerms()
    {
       const int f_0 = d_face_info(1, f);
       const int f_1 = d_face_info(4, f);
+      if (f_0 < 0) { return; } // Skip Neumann boundary faces
       const int nsides = (f_1 >= 0) ? 2 : 1;
       for (int el_i = 0; el_i < nsides; ++el_i)
       {

fem/moonolith/mortarassembler.cpp

@@ -78,10 +78,7 @@ template <int Dim>
 void BuildBoxes(const Mesh &mesh,
                 std::vector<::moonolith::AABB<Dim, double>> &element_boxes)
 {
-#ifndef NDEBUG
-   const int dim = mesh.Dimension();
-   assert(dim == Dim);
-#endif
+   MFEM_ASSERT(mesh.Dimension() == Dim, "Mesh and box dimensions mismatched");
    element_boxes.resize(mesh.GetNE());
 
    DenseMatrix pts;

fem/pbilinearform.cpp

@@ -488,10 +488,16 @@ void ParBilinearForm::FormLinearSystem(
          R.Mult(x, true_X);
 
          FormSystemMatrix(ess_tdof_list, A);
-         ConstrainedOperator *A_constrained;
-         Operator::FormConstrainedSystemOperator(ess_tdof_list, A_constrained);
+
+         std::unique_ptr<ConstrainedOperator> A_constrained([&]()
+         {
+            Operator *op;
+            Operator::FormSystemOperator(ess_tdof_list, op);
+            return dynamic_cast<ConstrainedOperator*>(op);
+         }());
+         MFEM_ASSERT(A_constrained != nullptr, "");
+
          A_constrained->EliminateRHS(true_X, true_B);
-         delete A_constrained;
          R.MultTranspose(true_B, b);
          hybridization->ReduceRHS(true_B, B);
          X.SetSize(B.Size());

fem/pfespace.cpp

@@ -424,7 +424,7 @@ void ParFiniteElementSpace::GetGroupComm(
             {
                if (ind[l] < 0)
                {
-                  dofs[l] = m + (-1-ind[l]);
+                  dofs[l] = m + FlipIndexSign(ind[l]);
                   if (g_ldof_sign)
                   {
                      (*g_ldof_sign)[dofs[l]] = -1;
@@ -462,7 +462,7 @@ void ParFiniteElementSpace::GetGroupComm(
             {
                if (ind[l] < 0)
                {
-                  dofs[l] = m + (-1-ind[l]);
+                  dofs[l] = m + FlipIndexSign(ind[l]);
                   if (g_ldof_sign)
                   {
                      (*g_ldof_sign)[dofs[l]] = -1;
@@ -500,7 +500,7 @@ void ParFiniteElementSpace::GetGroupComm(
             {
                if (ind[l] < 0)
                {
-                  dofs[l] = m + (-1-ind[l]);
+                  dofs[l] = m + FlipIndexSign(ind[l]);
                   if (g_ldof_sign)
                   {
                      (*g_ldof_sign)[dofs[l]] = -1;
@@ -538,16 +538,16 @@ void ParFiniteElementSpace::ApplyLDofSigns(Array<int> &dofs) const
    {
       if (dofs[i] < 0)
       {
-         if (ldof_sign[-1-dofs[i]] < 0)
+         if (ldof_sign[FlipIndexSign(dofs[i])] < 0)
          {
-            dofs[i] = -1-dofs[i];
+            dofs[i] = FlipIndexSign(dofs[i]);
          }
       }
       else
       {
          if (ldof_sign[dofs[i]] < 0)
          {
-            dofs[i] = -1-dofs[i];
+            dofs[i] = FlipIndexSign(dofs[i]);
          }
       }
    }
@@ -646,39 +646,38 @@ const FaceRestriction *ParFiniteElementSpace::GetFaceRestriction(
    auto itr = L2F.find(key);
    if (itr != L2F.end())
    {
-      return itr->second;
+      return itr->second.get();
    }
    else
    {
-      FaceRestriction *res;
+      std::unique_ptr<FaceRestriction> res;
       if (is_dg_space)
       {
          if (Conforming())
          {
-            res = new ParL2FaceRestriction(*this, f_ordering, type, m);
+            res.reset(new ParL2FaceRestriction(*this, f_ordering, type, m));
          }
          else
          {
-            res = new ParNCL2FaceRestriction(*this, f_ordering, type, m);
+            res.reset(new ParNCL2FaceRestriction(*this, f_ordering, type, m));
          }
       }
       else if (dynamic_cast<const DG_Interface_FECollection*>(fec))
       {
-         res = new L2InterfaceFaceRestriction(*this, f_ordering, type);
+         res.reset(new L2InterfaceFaceRestriction(*this, f_ordering, type));
       }
       else
       {
          if (Conforming())
          {
-            res = new ConformingFaceRestriction(*this, f_ordering, type);
+            res.reset(new ConformingFaceRestriction(*this, f_ordering, type));
          }
          else
          {
-            res = new ParNCH1FaceRestriction(*this, f_ordering, type);
+            res.reset(new ParNCH1FaceRestriction(*this, f_ordering, type));
          }
       }
-      L2F[key] = res;
-      return res;
+      return L2F.emplace(key, std::move(res)).first->second.get();
    }
 }
 
@@ -700,7 +699,8 @@ void ParFiniteElementSpace::GetSharedEdgeDofs(
       for (int i = 0; i < dofs.Size(); i++)
       {
          const int di = dofs[i];
-         dofs[i] = (di >= 0) ? rdofs[di] : -1-rdofs[-1-di];
+         dofs[i] = di >= 0 ? rdofs[di] :
+                   FlipIndexSign(rdofs[FlipIndexSign(di)]);
       }
    }
 }
@@ -724,7 +724,8 @@ void ParFiniteElementSpace::GetSharedTriangleDofs(
       for (int i = 0; i < dofs.Size(); i++)
       {
          const int di = dofs[i];
-         dofs[i] = (di >= 0) ? rdofs[di] : -1-rdofs[-1-di];
+         dofs[i] = di >= 0 ? rdofs[di] :
+                   FlipIndexSign(rdofs[FlipIndexSign(di)]);
       }
    }
 }
@@ -748,7 +749,8 @@ void ParFiniteElementSpace::GetSharedQuadrilateralDofs(
       for (int i = 0; i < dofs.Size(); i++)
       {
          const int di = dofs[i];
-         dofs[i] = (di >= 0) ? rdofs[di] : -1-rdofs[-1-di];
+         dofs[i] = (di >= 0) ? rdofs[di] :
+                   FlipIndexSign(rdofs[FlipIndexSign(di)]);
       }
    }
 }
@@ -1488,7 +1490,7 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
          GetElementVDofs(my_elems[i], ldofs);
          for (int j = 0; j < ldofs.Size(); j++)
          {
-            int ldof = (ldofs[j] >= 0 ? ldofs[j] : -1-ldofs[j]);
+            int ldof = UnsignIndex(ldofs[j]);
 
             if (ldof_marker[ldof] != fn)
             {
@@ -1549,7 +1551,7 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
          GetElementVDofs(my_elems[i], ldofs);
          for (int j = 0; j < ldofs.Size(); j++)
          {
-            int ldof = (ldofs[j] >= 0 ? ldofs[j] : -1-ldofs[j]);
+            int ldof = UnsignIndex(ldofs[j]);
 
             if (ldof_marker[ldof] != fn)
             {
@@ -1574,14 +1576,15 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
 
       for (int i = 0; i < num_ldofs; i++)
       {
-         int ldof = (ldofs_fn[i] >= 0 ? ldofs_fn[i] : -1-ldofs_fn[i]);
+         int ldof = UnsignIndex(ldofs_fn[i]);
          ldof_marker[ldof] = i;
       }
 
       for ( ; j < j_end; j++)
       {
-         int ldof = (send_J[j] >= 0 ? send_J[j] : -1-send_J[j]);
-         send_J[j] = (send_J[j] >= 0 ? ldof_marker[ldof] : -1-ldof_marker[ldof]);
+         const int ldof = UnsignIndex(send_J[j]);
+         send_J[j] = (send_J[j] >= 0 ? ldof_marker[ldof] :
+                      FlipIndexSign(ldof_marker[ldof]));
       }
    }
 
@@ -1673,12 +1676,7 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
    {
       for (int j_end = face_nbr_ldof.GetI()[fn+1]; j < j_end; j++)
       {
-         int ldof = face_nbr_ldof.GetJ()[j];
-         if (ldof < 0)
-         {
-            ldof = -1-ldof;
-         }
-
+         const int ldof = UnsignIndex(face_nbr_ldof.GetJ()[j]);
          face_nbr_glob_dof_map[j] = dof_face_nbr_offsets[fn] + ldof;
       }
    }
@@ -1722,7 +1720,7 @@ void ParFiniteElementSpace::GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const
    MFEM_ASSERT(Nonconforming() && i >= pmesh->GetNumFaces(), "");
    int el1, el2, inf1, inf2;
    pmesh->GetFaceElements(i, &el1, &el2);
-   el2 = -1 - el2;
+   el2 = FlipIndexSign(el2);
    pmesh->GetFaceInfos(i, &inf1, &inf2);
    MFEM_ASSERT(0 <= el2 && el2 < face_nbr_element_dof.Size(), "");
    const int nd = face_nbr_element_dof.RowSize(el2);
@@ -1738,7 +1736,8 @@ void ParFiniteElementSpace::GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const
    for (int j = 0; j < vdofs.Size(); j++)
    {
       const int ldof = vdofs[j];
-      vdofs[j] = (ldof >= 0) ? vol_vdofs[ldof] : -1-vol_vdofs[-1-ldof];
+      vdofs[j] = (ldof >= 0) ? vol_vdofs[ldof] :
+                 FlipIndexSign(vol_vdofs[FlipIndexSign(ldof)]);
    }
 }
 
@@ -2062,8 +2061,8 @@ void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
 
          for (int j = 0; j < ne; j++)
          {
-            dofs[offset++] = (ind[j] >= 0) ? (first + ind[j])
-                             /*         */ : (-1 - (first + (-1 - ind[j])));
+            dofs[offset++] = (ind[j] >= 0) ? (first + ind[j]) :
+                             FlipIndexSign(first + FlipIndexSign(ind[j]));
          }
       }
       else
@@ -2073,8 +2072,8 @@ void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
          const int *ind = fec->DofOrderForOrientation(Geometry::SEGMENT, Eo[i]);
          for (int j = 0; j < ne; j++)
          {
-            dofs[offset++] = (ind[j] >= 0) ? (first + ind[j])
-                             /*         */ : (-1 - (first + (-1 - ind[j])));
+            dofs[offset++] = (ind[j] >= 0) ? (first + ind[j]) :
+                             FlipIndexSign(first + FlipIndexSign(ind[j]));
          }
       }
    }
@@ -2867,7 +2866,7 @@ void NeighborRowMessage::Encode(int rank)
 
             if (ind && (edof = ind[edof]) < 0)
             {
-               edof = -1 - edof;
+               edof = FlipIndexSign(edof);
                s = -1;
             }
 
@@ -3068,10 +3067,10 @@ void NeighborRowMessage::Decode(int rank)
 
          // If edof arrived with a negative index, flip it, and the scaling.
          real_t s = (edof < 0) ? -1.0 : 1.0;
-         edof = (edof < 0) ? -1 - edof : edof;
+         edof = UnsignIndex(edof);
          if (ind && (edof = ind[edof]) < 0)
          {
-            edof = -1 - edof;
+            edof = FlipIndexSign(edof);
             s *= -1.0;
          }
 
@@ -3122,10 +3121,10 @@ void NeighborRowMessage::Decode(int rank)
 
             // If edof arrived with a negative index, flip it, and the scaling.
             s = (edof < 0) ? -1.0 : 1.0;
-            edof = (edof < 0) ? -1 - edof : edof;
+            edof = UnsignIndex(edof);
             if (ind && (edof = ind[edof]) < 0)
             {
-               edof = -1 - edof;
+               edof = FlipIndexSign(edof);
                s *= -1.0;
             }
 
@@ -4406,12 +4405,9 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
          {
             for (int j = 0; j < dofs.Size(); j++)
             {
-               int row = DofToVDof(dofs[j], vd);
-               if (row < 0) { row = -1 - row; }
-
-               int col = DofToVDof(old_dofs[j], vd, old_ndofs);
-               if (col < 0) { col = -1 - col; }
-
+               const int row = UnsignIndex(DofToVDof(dofs[j], vd));
+               const int col = UnsignIndex(DofToVDof(old_dofs[j], vd,
+                                                     old_ndofs));
                i_diag[row] = col;
             }
          }
@@ -4436,9 +4432,7 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
       {
          for (int j = 0; j < dofs.Size(); j++)
          {
-            int row = DofToVDof(dofs[j], vd);
-            if (row < 0) { row = -1 - row; }
-
+            const int row = UnsignIndex(DofToVDof(dofs[j], vd));
             if (i_diag[row] == i_diag[row+1]) // diag row empty?
             {
                i_offd[row] = old_dofs[j + vd * dofs.Size()];
@@ -4547,9 +4541,9 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    {
       const Embedding &emb = dtrans.embeddings[k];
 
-      int fine_rank = old_ranks[k];
-      int coarse_rank = (emb.parent < 0) ? (-1 - emb.parent)
-                        : old_pncmesh->ElementRank(emb.parent);
+      const int fine_rank = old_ranks[k];
+      const int coarse_rank = (emb.parent < 0) ? FlipIndexSign(emb.parent)
+                              : old_pncmesh->ElementRank(emb.parent);
 
       if (coarse_rank != MyRank && fine_rank == MyRank)
       {
@@ -4637,8 +4631,8 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
             {
                if (!std::isfinite(lR(i, 0))) { continue; }
 
-               int r = DofToVDof(dofs[i], vd);
-               int m = (r >= 0) ? r : (-1 - r);
+               const int r = DofToVDof(dofs[i], vd);
+               const int m = UnsignIndex(r);
 
                if (is_dg || !mark[m])
                {
@@ -4687,8 +4681,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
             {
                if (!std::isfinite(lR(i, 0))) { continue; }
 
-               int r = DofToVDof(dofs[i], vd);
-               int m = (r >= 0) ? r : (-1 - r);
+               const int m = UnsignIndex(DofToVDof(dofs[i], vd));
 
                if (is_dg || !mark[m])
                {

fem/pfespace.hpp

@@ -483,6 +483,8 @@ public:
    const FiniteElement *GetFaceNbrFaceFE(int i) const;
    const Array<HYPRE_BigInt> &GetFaceNbrGlobalDofMapArray() { return face_nbr_glob_dof_map; }
    const HYPRE_BigInt *GetFaceNbrGlobalDofMap() { return face_nbr_glob_dof_map; }
+   const Array<HYPRE_BigInt> &GetFaceNbrGlobalDofMapArray() const
+   { return face_nbr_glob_dof_map; }
    ElementTransformation *GetFaceNbrElementTransformation(int i) const
    { return pmesh->GetFaceNbrElementTransformation(i); }
 

fem/pgridfunc.cpp

@@ -543,13 +543,22 @@ void ParGridFunction::GetElementDofValues(int el, Vector &dof_vals) const
    }
 }
 
-void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
+void ParGridFunction::ProjectCoefficient(Coefficient &coeff, ProjectType type)
 {
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
 
    if (delta_c == NULL)
    {
-      GridFunction::ProjectCoefficient(coeff);
+      (*this) = std::numeric_limits<real_t>::min();
+      GridFunction::ProjectCoefficient(coeff,type);
+
+      // Accumulate for all vdofs.
+      if (pfes->GetNURBSext())
+      {
+         GroupCommunicator &gcomm = pfes->GroupComm();
+         gcomm.Reduce<real_t>(data, GroupCommunicator::Max);
+         gcomm.Bcast<real_t>(data);
+      }
    }
    else
    {
@@ -565,6 +574,147 @@ void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
    }
 }
 
+void ParGridFunction::ProjectCoefficient(VectorCoefficient &vcoeff,
+                                         ProjectType type)
+{
+   GridFunction::ProjectCoefficient(vcoeff, type);
+
+   // Accumulate for all vdofs.
+   if (pfes->GetNURBSext())
+   {
+      GroupCommunicator &gcomm = pfes->GroupComm();
+      gcomm.Reduce<real_t>(data, GroupCommunicator::Max);
+      gcomm.Bcast<real_t>(data);
+   }
+}
+
+void ParGridFunction::ProjectCoefficientGlobalL2(Coefficient &coeff,
+                                                 real_t rtol,
+                                                 int iter)
+{
+   // Define and assemble linear form
+   ParLinearForm b(pfes);
+   b.AddDomainIntegrator(new DomainLFIntegrator(coeff));
+   b.Assemble();
+
+   // Define and assemble bilinear form
+   ParBilinearForm a(pfes);
+   a.AddDomainIntegrator(new MassIntegrator());
+   a.Assemble();
+
+   // Configure solver
+   OperatorPtr A;
+   Vector B, X, x(*this);
+   Array<int> ess_tdof_list;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   Solver *prec = new HypreBoomerAMG;
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(rtol);
+   cg.SetMaxIter(iter);
+   cg.SetPrintLevel(0);
+   cg.SetPreconditioner(*prec);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   a.RecoverFEMSolution(X, b, x);
+   delete prec;
+}
+
+void ParGridFunction::ProjectCoefficientElementL2(Coefficient &coeff)
+{
+   Vector Va;
+   ProjectCoefficientElementL2_(coeff, *this, Va);
+
+   GroupCommunicator &gcomm = pfes->GroupComm();
+   gcomm.Reduce<real_t>(GetData(), GroupCommunicator::Sum);
+   gcomm.Bcast<real_t>(GetData());
+
+   gcomm.Reduce<real_t>(Va.GetData(), GroupCommunicator::Sum);
+   gcomm.Bcast<real_t>(Va.GetData());
+   (*this)/=Va;
+}
+
+void ParGridFunction::ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
+                                                 real_t rtol, int iter)
+{
+   // Define and assemble linear form
+   ParLinearForm b(pfes);
+   ParBilinearForm a(pfes);
+
+   // Dimension argument to GetRangeType is arbitrary to be 3, could also be 2.
+   if (fes->FEColl()->GetRangeType(3)  == mfem::FiniteElement::VECTOR)
+   {
+      b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(vcoeff));
+      a.AddDomainIntegrator(new VectorFEMassIntegrator());
+   }
+   else
+   {
+      b.AddDomainIntegrator(new VectorDomainLFIntegrator(vcoeff));
+      a.AddDomainIntegrator(new VectorMassIntegrator());
+   }
+   b.Assemble();
+   a.Assemble();
+
+   // Configure solver
+   OperatorPtr A;
+   Vector B, X, x(*this);
+   x = 0.0;
+   Array<int> ess_tdof_list;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   Solver *prec = new HypreBoomerAMG;
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(rtol);
+   cg.SetMaxIter(iter);
+   cg.SetPrintLevel(0);
+   cg.SetPreconditioner(*prec);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   a.RecoverFEMSolution(X, b, x);
+   x.Print();
+   delete prec;
+}
+
+void ParGridFunction::ProjectCoefficientElementL2(VectorCoefficient &vcoeff)
+{
+   if (fes->GetTypicalFE()->GetRangeType() == mfem::FiniteElement::VECTOR)
+   {
+      Vector Va;
+      ProjectCoefficientElementL2_(vcoeff, *this, Va);
+
+      GroupCommunicator &gcomm = pfes->GroupComm();
+      gcomm.Reduce<real_t>(GetData(), GroupCommunicator::Sum);
+      gcomm.Bcast<real_t>(GetData());
+
+      gcomm.Reduce<real_t>(Va.GetData(), GroupCommunicator::Sum);
+      gcomm.Bcast<real_t>(Va.GetData());
+      (*this)/=Va;
+   }
+   else
+   {
+      Array<int> vdofs(fes->GetNDofs());
+      Vector x, Va, gVa(Size());
+      VectorComponentCoefficient coeff(vcoeff,0);
+      *this = 0.0;
+      gVa = 0.0;
+      for (int v = 0; v < VectorDim(); v++)
+      {
+         coeff.SetComponent(v);
+         ProjectCoefficientElementL2_(coeff, x, Va);
+         fes->GetVDofs(v, vdofs);
+         SetSubVector(vdofs, x);
+         gVa.SetSubVector(vdofs, Va);
+      }
+
+      GroupCommunicator &gcomm = pfes->GroupComm();
+      gcomm.Reduce<real_t>(GetData(), GroupCommunicator::Sum);
+      gcomm.Bcast<real_t>(GetData());
+
+      gcomm.Reduce<real_t>(gVa.GetData(), GroupCommunicator::Sum);
+      gcomm.Bcast<real_t>(gVa.GetData());
+      *this /= gVa;
+   }
+}
+
+
 void ParGridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff)
 {
    // local maximal element attribute for each dof
@@ -1418,6 +1568,39 @@ PLBound ParGridFunction::GetBounds(Vector &lower, Vector &upper,
    return plb;
 }
 
+std::pair<real_t, real_t> ParGridFunction::EstimateFunctionMinimum(
+   const int vdim, const PLBound &plb, const int max_depth,
+   const real_t tol) const
+{
+   std::pair<real_t, real_t> minmax =
+      GridFunction::EstimateFunctionMinimum(vdim, plb, max_depth, tol);
+
+   real_t glob_min_lower = minmax.first;
+   real_t glob_min_upper = minmax.second;
+   MPI_Allreduce(MPI_IN_PLACE, &glob_min_lower, 1,
+                 MFEM_MPI_REAL_T, MPI_MIN, pfes->GetComm());
+   MPI_Allreduce(MPI_IN_PLACE, &glob_min_upper, 1,
+                 MFEM_MPI_REAL_T, MPI_MIN, pfes->GetComm());
+
+   return std::make_pair(glob_min_lower, glob_min_upper);
+}
+
+std::pair<real_t, real_t> ParGridFunction::EstimateFunctionMaximum(
+   const int vdim, const PLBound &plb, const int max_depth,
+   const real_t tol) const
+{
+   std::pair<real_t, real_t> minmax =
+      GridFunction::EstimateFunctionMaximum(vdim, plb, max_depth, tol);
+
+   real_t glob_max_lower = minmax.first;
+   real_t glob_max_upper = minmax.second;
+   MPI_Allreduce(MPI_IN_PLACE, &glob_max_lower, 1,
+                 MFEM_MPI_REAL_T, MPI_MAX, pfes->GetComm());
+   MPI_Allreduce(MPI_IN_PLACE, &glob_max_upper, 1,
+                 MFEM_MPI_REAL_T, MPI_MAX, pfes->GetComm());
+   return std::make_pair(glob_max_lower, glob_max_upper);
+}
+
 } // namespace mfem
 
 #endif // MFEM_USE_MPI

fem/pgridfunc.hpp

@@ -72,6 +72,10 @@ public:
 
    ParGridFunction(ParFiniteElementSpace *pf) : GridFunction(pf), pfes(pf) { }
 
+   /// Same as above but specify the device memory type
+   ParGridFunction(ParFiniteElementSpace *pf, MemoryType mt) :
+      GridFunction(pf, mt), pfes(pf) { }
+
    /// Construct a ParGridFunction using previously allocated array @a data.
    /** The ParGridFunction does not assume ownership of @a data which is assumed
        to be of size at least `pf->GetVSize()`. Similar to the GridFunction and
@@ -257,7 +261,11 @@ public:
    void GetElementDofValues(int el, Vector &dof_vals) const override;
 
    using GridFunction::ProjectCoefficient;
-   void ProjectCoefficient(Coefficient &coeff) override;
+   void ProjectCoefficient(Coefficient &coeff,
+                           ProjectType type = ProjectType::DEFAULT) override;
+
+   void ProjectCoefficient(VectorCoefficient &vcoeff,
+                           ProjectType type = ProjectType::DEFAULT) override;
 
    using GridFunction::ProjectDiscCoefficient;
    /** @brief Project a discontinuous vector coefficient as a grid function on
@@ -282,6 +290,18 @@ public:
    void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                      const Array<int> &bdr_attr) override;
 
+   void ProjectCoefficientGlobalL2(Coefficient &coeff,
+                                   real_t rtol = 1e-12,
+                                   int iter = 1000) override;
+
+   void ProjectCoefficientElementL2(Coefficient &coeff) override;
+
+   void ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
+                                   real_t rtol = 1e-12,
+                                   int iter = 1000) override;
+
+   void ProjectCoefficientElementL2(VectorCoefficient &vcoeff) override;
+
    /// @brief Returns ||u_ex - u_h||_L1 in parallel for H1 or L2 elements
    ///
    /// @see GridFunction::ComputeL1Error(Coefficient *exsol[],
@@ -589,6 +609,18 @@ public:
    PLBound GetBounds(Vector &lower, Vector &upper,
                      const int ref_factor=1, const int vdim=-1) const override;
 
+   /** @brief Estimate the GridFunction minimum across all elements. */
+   std::pair<real_t, real_t> EstimateFunctionMinimum(const int vdim,
+                                                     const PLBound &plb,
+                                                     const int max_depth,
+                                                     const real_t tol) const override;
+
+   /** @brief Estimate the GridFunction maximum across all elements. */
+   std::pair<real_t, real_t> EstimateFunctionMaximum(const int vdim,
+                                                     const PLBound &plb,
+                                                     const int max_depth,
+                                                     const real_t tol) const override;
+
    /** Save the local portion of the ParGridFunction. This differs from the
        serial GridFunction::Save in that it takes into account the signs of
        the local dofs. */

fem/prestriction.cpp

@@ -994,7 +994,6 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
       {
          if ( face.IsConforming() )
          {
-            interpolations.RegisterFaceConformingInterpolation(face,f_ind);
             SetFaceDofsScatterIndices1(face,f_ind);
             if ( m==L2FaceValues::DoubleValued )
             {
@@ -1010,7 +1009,6 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
          }
          else // Non-conforming face
          {
-            interpolations.RegisterFaceCoarseToFineInterpolation(face,f_ind);
             SetFaceDofsScatterIndices1(face,f_ind);
             if ( m==L2FaceValues::DoubleValued )
             {
@@ -1028,7 +1026,6 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
       }
       else if (type==FaceType::Boundary && face.IsBoundary())
       {
-         interpolations.RegisterFaceConformingInterpolation(face,f_ind);
          SetFaceDofsScatterIndices1(face,f_ind);
          if ( m==L2FaceValues::DoubleValued )
          {
@@ -1046,10 +1043,6 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
    {
       gather_offsets[i] += gather_offsets[i - 1];
    }
-
-   // Transform the interpolation matrix map into a contiguous memory structure.
-   interpolations.LinearizeInterpolatorMapIntoVector();
-   interpolations.InitializeNCInterpConfig();
 }
 
 void ParNCL2FaceRestriction::ComputeGatherIndices()

fem/prestriction.hpp

@@ -326,9 +326,7 @@ public:
        @param[in] keep_nbr_block When set to true the SparseMatrix will
                                  include the rows (in addition to the columns)
                                  corresponding to face-neighbor dofs. The
-                                 default behavior is to disregard those rows.
-
-       @warning This method is not implemented yet. */
+                                 default behavior is to disregard those rows. */
    void FillI(SparseMatrix &mat,
               const bool keep_nbr_block = false) const override;
 
@@ -364,9 +362,7 @@ public:
        @param[in] keep_nbr_block When set to true the SparseMatrix will
                                  include the rows (in addition to the columns)
                                  corresponding to face-neighbor dofs. The
-                                 default behavior is to disregard those rows.
-
-       @warning This method is not implemented yet. */
+                                 default behavior is to disregard those rows. */
    void FillJAndData(const Vector &fea_data,
                      SparseMatrix &mat,
                      const bool keep_nbr_block = false) const override;

fem/qfunction.hpp

@@ -271,10 +271,7 @@ inline void QuadratureFunction::GetValues(
    const int s_offset = qspace->Offset(idx);
    const int sl_size = qspace->Offset(idx + 1) - s_offset;
    // Make the values matrix memory an alias of the quadrature function memory
-   Memory<real_t> &values_mem = values.GetMemory();
-   values_mem.Delete();
-   values_mem.MakeAlias(GetMemory(), vdim*s_offset, vdim*sl_size);
-   values.SetSize(vdim, sl_size);
+   values.MakeRef(GetMemory(), vdim*s_offset, vdim, sl_size);
 }
 
 inline void QuadratureFunction::GetValues(

fem/qinterp/det.cpp

@@ -50,7 +50,13 @@ QuadratureInterpolator::DetKernelType
 QuadratureInterpolator::DetKernels::Fallback(
    int DIM, int SDIM, int D1D, int Q1D)
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
+   if (DIM == 1)
+   {
+      if (SDIM == 1) { return internal::quadrature_interpolator::Det1D; }
+      else if (SDIM == 2) { return internal::quadrature_interpolator::Det1DSurface<0,0,2>; }
+      else if (SDIM == 3) { return internal::quadrature_interpolator::Det1DSurface<0,0,3>; }
+      else { MFEM_ABORT(""); }
+   }
    else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D; }
    else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface; }
    else if (DIM == 3)

fem/qinterp/det.hpp

@@ -56,6 +56,50 @@ inline void Det1D(const int NE,
    });
 }
 
+template<int T_D1D = 0, int T_Q1D = 0, int T_SDIM = 3>
+inline void Det1DSurface(const int NE,
+                         const real_t *b,
+                         const real_t *g,
+                         const real_t *x,
+                         real_t *y,
+                         const int d1d = 0,
+                         const int q1d = 0,
+                         Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(b);
+   MFEM_CONTRACT_VAR(d_buff);
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x, D1D, T_SDIM, NE);
+   auto Y = Reshape(y, Q1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int q = 0; q < Q1D; q++)
+      {
+         real_t grad[T_SDIM];
+         for (int s = 0; s < T_SDIM; s++) { grad[s] = 0.0; }
+         for (int d = 0; d < D1D; d++)
+         {
+            const real_t gval = G(q, d);
+            for (int s = 0; s < T_SDIM; s++)
+            {
+               grad[s] += gval * X(d, s, e);
+            }
+         }
+         real_t norm2 = 0.0;
+         for (int s = 0; s < T_SDIM; s++)
+         {
+            norm2 += grad[s] * grad[s];
+         }
+         Y(q, e) = std::sqrt(norm2);
+      }
+   });
+}
+
 template<int T_D1D = 0, int T_Q1D = 0>
 inline void Det2D(const int NE,
                   const real_t *b,
@@ -290,11 +334,16 @@ template<int DIM, int SDIM, int D1D, int Q1D>
 QuadratureInterpolator::DetKernelType
 QuadratureInterpolator::DetKernels::Kernel()
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
-   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
-   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1)
+   {
+      if constexpr (SDIM == 1) { return internal::quadrature_interpolator::Det1D; }
+      else if constexpr (SDIM == 2) { return internal::quadrature_interpolator::Det1DSurface<D1D, Q1D, 2>; }
+      else if constexpr (SDIM == 3) { return internal::quadrature_interpolator::Det1DSurface<D1D, Q1D, 3>; }
+   }
+   else if constexpr (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
+   else if constexpr (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
+   MFEM_ABORT("");
 }
 
 /// @endcond

fem/qinterp/eval.hpp

@@ -203,10 +203,10 @@ template<int DIM, QVectorLayout Q_LAYOUT,
 QuadratureInterpolator::TensorEvalKernelType
 QuadratureInterpolator::TensorEvalKernels::Kernel()
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::Values1D<Q_LAYOUT>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::Values2D<Q_LAYOUT, VDIM, D1D, Q1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Values3D<Q_LAYOUT, VDIM, D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::Values1D<Q_LAYOUT>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::Values2D<Q_LAYOUT, VDIM, D1D, Q1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::Values3D<Q_LAYOUT, VDIM, D1D, Q1D>; }
+   MFEM_ABORT("");
 }
 
 /// @endcond

fem/qinterp/eval_hdiv.hpp

@@ -453,8 +453,15 @@ QuadratureInterpolator::TensorEvalHDivKernels::Kernel()
 {
    using namespace internal::quadrature_interpolator;
    static_assert(DIM == 2 || DIM == 3, "only DIM=2 and DIM=3 are implemented!");
-   if (DIM == 2) { return EvalHDiv2D<Q_LAYOUT, FLAGS, D1D, Q1D>; }
-   return EvalHDiv3D<Q_LAYOUT, FLAGS, D1D, Q1D>;
+   if constexpr (DIM == 2)
+   {
+      return EvalHDiv2D<Q_LAYOUT, FLAGS, D1D, Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return EvalHDiv3D<Q_LAYOUT, FLAGS, D1D, Q1D>;
+   }
+   MFEM_ABORT("only DIM=2 and DIM=3 are implemented!");
 }
 
 /// @endcond

fem/qinterp/grad.hpp

@@ -592,10 +592,10 @@ template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
 QuadratureInterpolator::GradKernelType
 QuadratureInterpolator::GradKernels::Kernel()
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::Derivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Derivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::Derivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::Derivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D>; }
+   MFEM_ABORT("");
 }
 
 template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
@@ -603,10 +603,10 @@ template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
 QuadratureInterpolator::CollocatedGradKernelType
 QuadratureInterpolator::CollocatedGradKernels::Kernel()
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D>; }
+   MFEM_ABORT("");
 }
 
 /// @endcond

fem/quadinterpolator.cpp

@@ -542,7 +542,8 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    }
 
    MFEM_ASSERT(!(eval_flags & DETERMINANTS) || dim == vdim ||
-               (dim == 2 && vdim == 3), "Invalid dimensions for determinants.");
+               (dim == 2 && vdim == 3) || (dim == 1 && vdim == 2) ||
+               (dim == 1 && vdim == 3), "Invalid dimensions for determinants.");
    MFEM_ASSERT(fespace->GetMesh()->GetNumGeometries(
                   fespace->GetMesh()->Dimension()) == 1,
                "mixed meshes are not supported");
@@ -751,10 +752,10 @@ template <int DIM, int VDIM, int ND, int NQ>
 EvalKernel QuadratureInterpolator::EvalKernels::Kernel()
 {
    using namespace internal::quadrature_interpolator;
-   if (DIM == 1) { return Eval1D; }
-   else if (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
-   else if (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 1) { return Eval1D; }
+   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
+   MFEM_ABORT("");
 }
 
 template <int DIM>

fem/restriction.cpp

@@ -844,8 +844,6 @@ void ConformingFaceRestriction::ComputeGatherIndices(
    gather_offsets[0] = 0;
 }
 
-static inline int absdof(int i) { return i < 0 ? -1-i : i; }
-
 void ConformingFaceRestriction::SetFaceDofsScatterIndices(
    const Mesh::FaceInformation &face,
    const int face_index,
@@ -868,9 +866,9 @@ void ConformingFaceRestriction::SetFaceDofsScatterIndices(
    {
       const int lex_volume_dof = face_map[face_dof];
       const int s_volume_dof = AsConst(vol_dof_map)[lex_volume_dof]; // signed
-      const int volume_dof = absdof(s_volume_dof);
+      const int volume_dof = UnsignIndex(s_volume_dof);
       const int s_global_dof = elem_map[elem_index*elem_dofs + volume_dof];
-      const int global_dof = absdof(s_global_dof);
+      const int global_dof = UnsignIndex(s_global_dof);
       const int restriction_dof = face_dofs*face_index + face_dof;
       scatter_indices[restriction_dof] = s_global_dof;
       ++gather_offsets[global_dof + 1];
@@ -897,10 +895,10 @@ void ConformingFaceRestriction::SetFaceDofsGatherIndices(
    {
       const int lex_volume_dof = face_map[face_dof];
       const int s_volume_dof = AsConst(vol_dof_map)[lex_volume_dof];
-      const int volume_dof = absdof(s_volume_dof);
+      const int volume_dof = UnsignIndex(s_volume_dof);
       const int s_global_dof = elem_map[elem_index*elem_dofs + volume_dof];
       const int sgn = (s_global_dof >= 0) ? 1 : -1;
-      const int global_dof = absdof(s_global_dof);
+      const int global_dof = UnsignIndex(s_global_dof);
       const int restriction_dof = face_dofs*face_index + face_dof;
       const int s_restriction_dof = (sgn >= 0) ? restriction_dof : -1 -
                                     restriction_dof;
@@ -1506,12 +1504,12 @@ void L2FaceRestriction::EnsureNormalDerivativeRestriction() const
    }
 }
 
-InterpolationManager::InterpolationManager(const FiniteElementSpace &fes,
-                                           ElementDofOrdering ordering,
+InterpolationManager::InterpolationManager(const FiniteElementSpace &fes_,
+                                           ElementDofOrdering ordering_,
                                            FaceType type)
-   : fes(fes),
-     ordering(ordering),
-     interp_config( fes.GetNFbyType(type) ),
+   : fes(fes_),
+     ordering(ordering_),
+     interp_config(fes.GetNFbyType(type)),
      nc_cpt(0)
 { }
 
@@ -1536,7 +1534,8 @@ void InterpolationManager::RegisterFaceCoarseToFineInterpolation(
                         face.element[0].local_face_id +
                         6*face.element[1].local_face_id +
                         36*face.element[1].orientation ;
-   // Unfortunately we can't trust unicity of the ptMat to identify the transformation.
+   // Unfortunately we can't trust uniqueness of the ptMat to identify the
+   // transformation.
    Key key(ptMat, face_key);
    auto itr = interp_map.find(key);
    if ( itr == interp_map.end() )
@@ -1583,17 +1582,27 @@ const DenseMatrix* InterpolationManager::GetCoarseToFineInterpolation(
    IsoparametricTransformation isotr;
    isotr.SetIdentityTransformation(trace_fe->GetGeomType());
    isotr.SetPointMat(*ptMat);
-   DenseMatrix& trans_pt_mat = isotr.GetPointMat();
-   // PointMatrix needs to be flipped in 2D
-   if ( trace_fe->GetGeomType()==Geometry::SEGMENT && !is_ghost_slave )
-   {
-      std::swap(trans_pt_mat(0,0),trans_pt_mat(0,1));
-   }
    DenseMatrix native_interpolator(face_dofs,face_dofs);
    trace_fe->GetLocalInterpolation(isotr, native_interpolator);
+
+   if (trace_fe->GetMapType() == FiniteElement::INTEGRAL)
+   {
+      // Handle potentially inverted Jacobian matrix
+      isotr.SetIntPoint(&Geometries.GetCenter(trace_fe->GetGeomType()));
+      native_interpolator *= (isotr.Weight() >= 0) ? 1.0 : -1.0;
+   }
+
    const int dim = trace_fe->GetDim()+1;
    const int dof1d = trace_fe->GetOrder()+1;
-   const int orientation = face.element[1].orientation;
+   int orientation_i = face.element[1].orientation;
+   const int orientation_j = face.element[1].orientation;
+
+   // In 2D, need to flip orientation of the segments`
+   if (trace_fe->GetGeomType() == Geometry::SEGMENT && !is_ghost_slave)
+   {
+      orientation_i = 1;
+   }
+
    for (int i = 0; i < face_dofs; i++)
    {
       const int ni = (dof_map.Size()==0) ? i : dof_map[i];
@@ -1602,7 +1611,7 @@ const DenseMatrix* InterpolationManager::GetCoarseToFineInterpolation(
       {
          // master side is elem 2, so we permute to order dofs as elem 1.
          li = PermuteFaceL2(dim, face_id2, face_id1,
-                            orientation, dof1d, li);
+                            orientation_i, dof1d, li);
       }
       for (int j = 0; j < face_dofs; j++)
       {
@@ -1611,7 +1620,7 @@ const DenseMatrix* InterpolationManager::GetCoarseToFineInterpolation(
          {
             // master side is elem 2, so we permute to order dofs as elem 1.
             lj = PermuteFaceL2(dim, face_id2, face_id1,
-                               orientation, dof1d, lj);
+                               orientation_j, dof1d, lj);
          }
          const int nj = (dof_map.Size()==0) ? j : dof_map[j];
          (*interpolator)(li,lj) = native_interpolator(ni,nj);
@@ -1676,7 +1685,7 @@ NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
                                          const L2FaceValues m,
                                          bool build)
    : L2FaceRestriction(fes, f_ordering, type, m, false),
-     interpolations(fes, f_ordering, type)
+     interpolations(fes.GetInterpolationManager(ordering, type))
 {
    if (!build) { return; }
    x_interp.UseDevice(true);
@@ -2202,14 +2211,6 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
          {
             PermuteAndSetFaceDofsScatterIndices2(face,f_ind);
          }
-         if ( face.IsConforming() )
-         {
-            interpolations.RegisterFaceConformingInterpolation(face,f_ind);
-         }
-         else // Non-conforming face
-         {
-            interpolations.RegisterFaceCoarseToFineInterpolation(face,f_ind);
-         }
          f_ind++;
       }
       else if ( type==FaceType::Boundary && face.IsBoundary() )
@@ -2219,7 +2220,6 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
          {
             SetBoundaryDofsScatterIndices2(face,f_ind);
          }
-         interpolations.RegisterFaceConformingInterpolation(face,f_ind);
          f_ind++;
       }
    }
@@ -2232,10 +2232,6 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
    {
       gather_offsets[i] += gather_offsets[i - 1];
    }
-
-   // Transform the interpolation matrix map into a contiguous memory structure.
-   interpolations.LinearizeInterpolatorMapIntoVector();
-   interpolations.InitializeNCInterpConfig();
 }
 
 void NCL2FaceRestriction::ComputeGatherIndices()
@@ -2278,6 +2274,18 @@ void NCL2FaceRestriction::ComputeGatherIndices()
    gather_offsets[0] = 0;
 }
 
+static int GetSharedVSize(const FiniteElementSpace &fes)
+{
+#ifdef MFEM_USE_MPI
+   if (auto pfes = dynamic_cast<const ParFiniteElementSpace*>(&fes))
+   {
+      const_cast<ParFiniteElementSpace*>(pfes)->ExchangeFaceNbrData();
+      return pfes->GetFaceNbrVSize();
+   }
+#endif
+   return 0;
+}
+
 L2InterfaceFaceRestriction::L2InterfaceFaceRestriction(
    const FiniteElementSpace& fes_,
    const ElementDofOrdering ordering_,
@@ -2288,25 +2296,54 @@ L2InterfaceFaceRestriction::L2InterfaceFaceRestriction(
      nfaces(fes.GetNFbyType(type)),
      vdim(fes.GetVDim()),
      byvdim(fes.GetOrdering() == Ordering::byVDIM),
-     face_dofs(nfaces > 0 ? fes.GetFaceElement(0)->GetDof() : 0),
+     face_dofs(fes.GetTypicalTraceElement()->GetDof()),
      nfdofs(face_dofs*nfaces),
-     ndofs(fes.GetNDofs())
+     ndofs(fes.GetNDofs()),
+     nsdofs(GetSharedVSize(fes))
 {
    height = nfdofs;
    width = ndofs;
 
+#ifdef MFEM_USE_MPI
+   auto pfes = dynamic_cast<const ParFiniteElementSpace*>(&fes);
+#endif
+
    const Table &face2dof = fes.GetFaceToDofTable();
 
    const Mesh &mesh = *fes.GetMesh();
    int face_idx = 0;
-   gather_map.SetSize(nfdofs);
-   for (int f = 0; f < mesh.GetNumFaces(); ++f)
+   scatter_map.SetSize(nfdofs);
+   gather_map.SetSize(ndofs + nsdofs);
+   gather_map = -1;
+
+   Array<int> dofs;
+   for (int f = 0; f < mesh.GetNumFacesWithGhost(); ++f)
    {
       Mesh::FaceInformation face = mesh.GetFaceInformation(f);
-      if (!face.IsOfFaceType(type)) { continue; }
-      for (int i = 0; i < face_dofs; ++i)
+      if (!face.IsOfFaceType(type) || face.IsNonconformingCoarse()) { continue; }
+
+      if (f < mesh.GetNumFaces())
       {
-         gather_map[i + face_idx*face_dofs] = face2dof.GetJ()[i + f*face_dofs];
+         // Local face
+         face2dof.GetRow(f, dofs);
+         for (int i = 0; i < face_dofs; ++i)
+         {
+            scatter_map[i + face_idx*face_dofs] = dofs[i];
+            gather_map[dofs[i]] = i + face_idx*face_dofs;
+         }
+      }
+      else
+      {
+         // Shared (non-conforming) ghost face
+#ifdef MFEM_USE_MPI
+         MFEM_ASSERT(pfes != nullptr, "");
+         pfes->GetFaceNbrFaceVDofs(f, dofs);
+         for (int i = 0; i < face_dofs; ++i)
+         {
+            scatter_map[i + face_idx*face_dofs] = ndofs + dofs[i];
+            gather_map[ndofs + dofs[i]] = i + face_idx*face_dofs;
+         }
+#endif
       }
       ++face_idx;
    }
@@ -2314,13 +2351,19 @@ L2InterfaceFaceRestriction::L2InterfaceFaceRestriction(
 
 void L2InterfaceFaceRestriction::Mult(const Vector &x, Vector &y) const
 {
+   const int NDOFS = ndofs;
    const int nd = face_dofs;
    const int nf = nfaces;
    const int vd = vdim;
    const bool t = byvdim;
-   const int *map = gather_map.Read();
+   const int *map = scatter_map.Read();
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, type);
+   MFEM_ASSERT(face_nbr_data.Size() / vd == nsdofs, "");
 
    const auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
+   const auto d_x_shared = Reshape(face_nbr_data.Read(),
+                                   t?vd:nsdofs, t?nsdofs:vd);
    auto d_y = Reshape(y.Write(), nd, vd, nf);
 
    mfem::forall(nd*nf, [=] MFEM_HOST_DEVICE (int i)
@@ -2328,7 +2371,8 @@ void L2InterfaceFaceRestriction::Mult(const Vector &x, Vector &y) const
       const int j = map[i];
       for (int c = 0; c < vd; ++c)
       {
-         d_y(i % nd, c, i / nd) = d_x(t?c:j, t?j:c);
+         if (j < NDOFS) { d_y(i % nd, c, i / nd) = d_x(t?c:j, t?j:c); }
+         else { d_y(i % nd, c, i / nd) = d_x_shared(t?c:(j-NDOFS), t?(j-NDOFS):c); }
       }
    });
 }
@@ -2343,15 +2387,39 @@ void L2InterfaceFaceRestriction::AddMultTranspose(
    const int *map = gather_map.Read();
 
    const auto d_x = Reshape(x.Read(), nd, vd, nf);
-   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   auto d_y = Reshape(y.ReadWrite(), t?vd:ndofs, t?ndofs:vd);
 
-   mfem::forall(ndofs, [=] MFEM_HOST_DEVICE (int i) { d_y[i] = 0.0; });
-   mfem::forall(nd*nf, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall(ndofs, [=] MFEM_HOST_DEVICE (int i)
    {
       const int j = map[i];
+      if (j < 0) { return; }
       for (int c = 0; c < vd; ++c)
       {
-         d_y(t?c:j, t?j:c) = d_x(i % nd, c, i / nd);
+         d_y(t?c:i, t?i:c) += a*d_x(j % nd, c, j / nd);
+      }
+   });
+}
+
+void L2InterfaceFaceRestriction::MultTransposeShared(
+   const Vector &x, Vector &y) const
+{
+   const int nd = face_dofs;
+   const int nf = nfaces;
+   const int vd = vdim;
+   const bool t = byvdim;
+   const int *map = gather_map.Read();
+
+   const auto d_x = Reshape(x.Read(), nd, vd, nf);
+   auto d_y = Reshape(y.Write(), t?vd:(ndofs+nsdofs), t?(ndofs+nsdofs):vd);
+   y = 0.0;
+
+   mfem::forall(ndofs + nsdofs, [=] MFEM_HOST_DEVICE (int i)
+   {
+      const int j = map[i];
+      if (j < 0) { return; }
+      for (int c = 0; c < vd; ++c)
+      {
+         d_y(t?c:i, t?i:c) = d_x(j % nd, c, j / nd);
       }
    });
 }
@@ -2361,6 +2429,11 @@ const Array<int> &L2InterfaceFaceRestriction::GatherMap() const
    return gather_map;
 }
 
+const Array<int> &L2InterfaceFaceRestriction::ScatterMap() const
+{
+   return scatter_map;
+}
+
 Vector GetLVectorFaceNbrData(
    const FiniteElementSpace &fes, const Vector &x, FaceType ftype)
 {
@@ -2382,6 +2455,7 @@ Vector GetLVectorFaceNbrData(
          {
             ParGridFunction gf(pfes, const_cast<Vector&>(x));
             gf.ExchangeFaceNbrData();
+            x.SyncMemory(gf);
             return std::move(gf.FaceNbrData());
          }
       }

fem/restriction.hpp

@@ -812,13 +812,12 @@ protected:
        PointMatrix and a local face identifier. */
    using Key = std::pair<const DenseMatrix*,int>;
    /// The temporary map used to store the different interpolators.
-   using Map = std::map<Key, std::pair<int,const DenseMatrix*>>;
+   using Map =
+      std::unordered_map<Key, std::pair<int,const DenseMatrix*>, PairHasher>;
    Map interp_map; // The temporary map that stores the interpolators.
 
 public:
-   InterpolationManager() = delete;
-
-   /** @brief main constructor.
+   /** @brief Constructor.
 
        @param[in] fes      The FiniteElementSpace on which this operates
        @param[in] ordering Request a specific element ordering.
@@ -909,7 +908,7 @@ private:
 class NCL2FaceRestriction : virtual public L2FaceRestriction
 {
 protected:
-   InterpolationManager interpolations;
+   const InterpolationManager &interpolations;
    mutable Vector x_interp;
 
    /** @brief Constructs an NCL2FaceRestriction, this is a specialization of a
@@ -996,9 +995,7 @@ public:
        @param[in] keep_nbr_block When set to true the SparseMatrix will
                                  include the rows (in addition to the columns)
                                  corresponding to face-neighbor dofs. The
-                                 default behavior is to disregard those rows.
-
-       @warning This method is not implemented yet. */
+                                 default behavior is to disregard those rows. */
    void FillI(SparseMatrix &mat,
               const bool keep_nbr_block = false) const override;
 
@@ -1016,9 +1013,7 @@ public:
        @param[in] keep_nbr_block When set to true the SparseMatrix will
                                  include the rows (in addition to the columns)
                                  corresponding to face-neighbor dofs. The
-                                 default behavior is to disregard those rows.
-
-       @warning This method is not implemented yet. */
+                                 default behavior is to disregard those rows. */
    void FillJAndData(const Vector &fea_data,
                      SparseMatrix &mat,
                      const bool keep_nbr_block = false) const override;
@@ -1036,9 +1031,7 @@ public:
                               added the face contributions.
                               The format is: dofs x dofs x ne, where dofs is the
                               number of dofs per element and ne the number of
-                              elements.
-
-       @warning This method is not implemented yet. */
+                              elements. */
    void AddFaceMatricesToElementMatrices(const Vector &fea_data,
                                          Vector &ea_data) const override;
 
@@ -1130,7 +1123,9 @@ protected:
    const int face_dofs; ///< Number of dofs on each face
    const int nfdofs; ///< Total number of dofs on the faces (E-vector size)
    const int ndofs; ///< Number of dofs in the space (L-vector size)
+   const int nsdofs; ///< Number of shared face neighbor (ghost) dofs
    Array<int> gather_map; ///< Gather map
+   Array<int> scatter_map; ///< Scatter map
 
 public:
    /** @brief Constructs an L2InterfaceFaceRestriction.
@@ -1168,7 +1163,24 @@ public:
    void AddMultTranspose(const Vector &x, Vector &y,
                          const real_t a = 1.0) const override;
 
+   /// @brief Gather degrees of freedom, from face E-vector to L-vector and
+   /// shared (ghost) DOFs.
+   ///
+   /// @param[in]     x The face E-Vector degrees of freedom with size
+   ///                  (face_dofs, vdim, nf), where nf is the number of
+   ///                  interior or boundary faces requested by @a type in the
+   ///                  constructor. The face_dofs should be ordered according
+   ///                  to the given ElementDofOrdering
+   /// @param[out]    y Vector of length vsize + face neighbor vsize
+   void MultTransposeShared(const Vector &x, Vector &y) const;
+
    const Array<int> &GatherMap() const override;
+
+   /// @brief Return the low-level mapping from L-dofs to E-dofs.
+   ///
+   /// L-dofs that do not correspond to an E-dof (e.g. that lie on a face of a
+   /// different type) are given index -1.
+   const Array<int> &ScatterMap() const;
 };
 
 /** @brief Convert a dof face index from Native ordering to lexicographic

fem/tmop.cpp

@@ -4102,8 +4102,11 @@ void TMOP_Integrator::GetSurfaceFittingErrors(const Vector &d_loc,
 #ifdef MFEM_USE_MPI
       // Don't count the overlapping DOFs in parallel.
       // The pfes might be ordered byVDIM, while the loop goes consecutively.
-      const int dof_i = pfes->DofToVDof(i, 0);
-      if (parallel && pfes->GetLocalTDofNumber(dof_i) < 0) { continue; }
+      if (parallel)
+      {
+         const int dof_i = pfes->DofToVDof(i, 0);
+         if (pfes->GetLocalTDofNumber(dof_i) < 0) { continue; }
+      }
 #endif
 
       dof_cnt++;

fem/transfer.cpp

@@ -333,6 +333,12 @@ void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
    int nel_ho = mesh_ho->GetNE();
    int nel_lor = mesh_lor->GetNE();
 
+   if (nel_ho == 0)
+   {
+      M_LH.SetSize(0);
+      return;
+   }
+
    const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
 
    int nref_max = 0;
@@ -831,11 +837,17 @@ void L2ProjectionGridTransfer::L2ProjectionL2Space::Mult(
 void L2ProjectionGridTransfer::L2ProjectionL2Space::EAMult(
    const Vector &x, Vector &y) const
 {
+   const int nel_ho = fes_ho.GetMesh()->GetNE();
+
+   if (nel_ho == 0)
+   {
+      return;
+   }
+
    const int iho = 0;
    const int nref = ho2lor.RowSize(iho);
    const int ndof_ho = fes_ho.GetFE(iho)->GetDof();
    const int ndof_lor = fes_lor.GetFE(ho2lor.GetRow(iho)[0])->GetDof();
-   const int nel_ho = fes_ho.GetMesh()->GetNE();
 
    DenseTensor R_dt;
    R_dt.NewMemoryAndSize(R.GetMemory(), ndof_lor*nref, ndof_ho, nel_ho, false);
@@ -887,11 +899,17 @@ void L2ProjectionGridTransfer::L2ProjectionL2Space::MultTranspose(
 void L2ProjectionGridTransfer::L2ProjectionL2Space::EAMultTranspose(
    const Vector &x, Vector &y) const
 {
+   const int nel_ho = fes_ho.GetMesh()->GetNE();
+
+   if (nel_ho == 0)
+   {
+      return;
+   }
+
    const int iho = 0;
    const int nref = ho2lor.RowSize(iho);
    const int ndof_ho = fes_ho.GetFE(iho)->GetDof();
    const int ndof_lor = fes_lor.GetFE(ho2lor.GetRow(iho)[0])->GetDof();
-   const int nel_ho = fes_ho.GetMesh()->GetNE();
 
    DenseTensor R_dt;
    R_dt.NewMemoryAndSize(R.GetMemory(), ndof_lor*nref, ndof_ho, nel_ho, false);
@@ -901,7 +919,6 @@ void L2ProjectionGridTransfer::L2ProjectionL2Space::EAMultTranspose(
 void L2ProjectionGridTransfer::L2ProjectionL2Space::Prolongate(
    const Vector &x, Vector &y) const
 {
-
    if (fes_ho.GetNE() == 0) { return; }
 
    if (use_ea)
@@ -960,14 +977,13 @@ void L2ProjectionGridTransfer::L2ProjectionL2Space::EAProlongate(
 void L2ProjectionGridTransfer::L2ProjectionL2Space::ProlongateTranspose(
    const Vector &x, Vector &y) const
 {
+   if (fes_ho.GetNE() == 0) { return; }
 
    if (use_ea)
    {
       return EAProlongateTranspose(x,y);
    }
 
-
-   if (fes_ho.GetNE() == 0) { return; }
    MFEM_VERIFY(P.Size() > 0, "Prolongation not supported for these spaces.")
    int vdim = fes_ho.GetVDim();
    Array<int> vdofs;
@@ -1244,13 +1260,6 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space
    int ndof_ho = pfes_ho.GetNDofs();
    int ndof_lor = pfes_lor.GetNDofs();
 
-
-   // If the local mesh is empty, skip all computations
-   if (nel_ho == 0)
-   {
-      return;
-   }
-
    const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
 
    int nref_max = 0;
@@ -1860,6 +1869,11 @@ L2ProjectionGridTransfer::H1SpaceMixedMassOperator::H1SpaceMixedMassOperator(
 void L2ProjectionGridTransfer::H1SpaceMixedMassOperator::Mult(const Vector &x,
                                                               Vector &y) const
 {
+   if (fes_ho->GetNE() == 0)
+   {
+      return;
+   }
+
    const Operator* elem_restrict_ho = fes_ho->GetElementRestriction(
                                          ElementDofOrdering::NATIVE);
    const Operator* elem_restrict_lor = fes_lor->GetElementRestriction(
@@ -1906,6 +1920,11 @@ void L2ProjectionGridTransfer::H1SpaceMixedMassOperator::Mult(const Vector &x,
 void L2ProjectionGridTransfer::H1SpaceMixedMassOperator::MultTranspose(
    const Vector &x, Vector &y) const
 {
+   if (fes_ho->GetNE() == 0)
+   {
+      return;
+   }
+
    const Operator* elem_restrict_ho = fes_ho->GetElementRestriction(
                                          ElementDofOrdering::NATIVE);
    const Operator* elem_restrict_lor = fes_lor->GetElementRestriction(

general/CMakeLists.txt

@@ -18,6 +18,7 @@ list(APPEND SRCS
   gecko.cpp
   globals.cpp
   hash.cpp
+  hash_util.cpp
   isockstream.cpp
   mem_manager.cpp
   occa.cpp
@@ -46,6 +47,7 @@ list(APPEND HDRS
   globals.hpp
   zstr.hpp
   hash.hpp
+  hash_util.hpp
   isockstream.hpp
   kdtree.hpp
   mem_alloc.hpp

general/array.hpp

@@ -114,10 +114,22 @@ public:
    Array<T> &operator=(const Array<T> &src) { src.Copy(*this); return *this; }
 
    /// Move assignment operator
+   /** If *this is a non-owning view (e.g., from MakeRef()), the data is copied
+       so that the base is also modified. */
    Array<T> &operator=(Array<T> &&src)
    {
       if (this == &src) { return *this; }
-      Swap(src);  // Swap does not use move assignment!
+      // If *this is a non-owning view (alias), and its capacity is sufficient
+      // to contain src, then copy into *this so that the alias's base memory is
+      // modified.
+      if (!OwnsData() && Capacity() >= src.Size())
+      {
+         *this = src; // Copy assignment.
+      }
+      else
+      {
+         Swap(src); // Swap the pointers only.
+      }
       src.DeleteAll();
       return *this;
    }
@@ -251,6 +263,9 @@ public:
    /// Make this Array a reference to 'master'.
    inline void MakeRef(const Array &master);
 
+   /// Make this Array a reference to the given sub-Memory of @a base.
+   inline void MakeRef(Memory<T> &base, int offset, int size_);
+
    /// Reset the Array to use the given external Memory @a mem and size @a s.
    /** If @a own_mem is false, the Array will not own any of the pointers of
        @a mem.
@@ -1073,6 +1088,14 @@ inline void Array<T>::MakeRef(const Array &master)
    data.MakeAlias(master.GetMemory(), 0, size);
 }
 
+template <class T>
+inline void Array<T>::MakeRef(Memory<T> &base, int offset, int size_)
+{
+   data.Delete();
+   size = size_;
+   data.MakeAlias(base, offset, size_);
+}
+
 template <class T>
 inline void Array<T>::NewMemoryAndSize(
    const Memory<T> &mem, int s, bool own_mem)

general/backends.hpp

@@ -44,6 +44,7 @@
 #endif
 
 #if !defined(MFEM_USE_CUDA_OR_HIP)
+constexpr bool mfem_use_gpu = false;
 #define MFEM_DEVICE
 #define MFEM_HOST
 #define MFEM_LAMBDA
@@ -52,6 +53,7 @@
 #define MFEM_DEVICE_SYNC
 // MFEM_STREAM_SYNC is used for UVM and MPI GPU-Aware kernels
 #define MFEM_STREAM_SYNC
+#define MFEM_LAUNCH_BOUNDS(...)
 #endif
 
 #if !((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) || \

general/cuda.hpp

@@ -20,9 +20,11 @@
 
 #if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 #define MFEM_USE_CUDA_OR_HIP
+constexpr bool mfem_use_gpu = true;
 #define MFEM_DEVICE __device__
 #define MFEM_HOST __host__
 #define MFEM_LAMBDA __host__
+#define MFEM_LAUNCH_BOUNDS __launch_bounds__
 // #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(cudaDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(cudaStreamSynchronize(0))

general/device.cpp

@@ -22,6 +22,8 @@
 
 #include <unordered_map>
 #include <map>
+#include <sstream>
+#include <iomanip>
 
 namespace mfem
 {
@@ -716,6 +718,29 @@ void Device::DeviceMem(size_t *free, size_t *total)
 #endif
 }
 
+std::string Device::GetUUID(const int device_id)
+{
+   std::stringstream res;
+#if defined(MFEM_USE_CUDA)
+   cudaDeviceProp prop;
+   MFEM_GPU_CHECK(cudaGetDeviceProperties(&prop, device_id));
+   for (int i = 0; i < 16; ++i)
+   {
+      res << std::setfill('0') << std::setw(2) << std::hex
+          << static_cast<unsigned>(prop.uuid.bytes[i]);
+   }
+#elif defined(MFEM_USE_HIP)
+   hipUUID uuid;
+   MFEM_GPU_CHECK(hipDeviceGetUuid(&uuid, device_id));
+   for (int i = 0; i < 16; ++i)
+   {
+      res << std::setfill('0') << std::setw(2) << std::hex
+          << static_cast<unsigned>(uuid.bytes[i]);
+   }
+#endif
+   return res.str();
+}
+
 int Device::NumMultiprocessors(int dev)
 {
 #if defined(MFEM_USE_CUDA)

general/device.hpp

@@ -255,6 +255,10 @@ public:
    /// Get the number of available devices (may be called before configuration).
    static int GetDeviceCount();
 
+   /// Gets a string representation of the GPU UUID.
+   /// 0 <= @a device_id < GetDeviceCount()
+   static std::string GetUUID(const int device_id = 0);
+
    /** @brief Return true if any of the backends in the backend mask, @a b_mask,
        are allowed. */
    /** This method can be used with any of the Backend::Id constants, the

general/forall.hpp

@@ -295,11 +295,12 @@ using hip_threads_z =
 #endif
 
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA) && defined(__CUDACC__)
-template <const int BLOCKS = MFEM_CUDA_BLOCKS, typename DBODY>
+template <typename DBODY>
 void RajaCuWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::cuda_exec<BLOCKS,true>>(RAJA::RangeSegment(0,N),d_body);
+   RAJA::forall<RAJA::cuda_exec<MFEM_CUDA_BLOCKS,true>>(RAJA::RangeSegment(0,N),
+                                                        d_body);
 }
 
 template <typename DBODY>
@@ -362,18 +363,18 @@ struct RajaCuWrap;
 template <>
 struct RajaCuWrap<1>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
-      RajaCuWrap1D<BLCK>(N, d_body);
+      RajaCuWrap1D(N, d_body);
    }
 };
 
 template <>
 struct RajaCuWrap<2>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
@@ -384,7 +385,7 @@ struct RajaCuWrap<2>
 template <>
 struct RajaCuWrap<3>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
@@ -395,11 +396,12 @@ struct RajaCuWrap<3>
 #endif
 
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP) && defined(__HIP__)
-template <const int BLOCKS = MFEM_HIP_BLOCKS, typename DBODY>
+template <typename DBODY>
 void RajaHipWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::hip_exec<BLOCKS,true>>(RAJA::RangeSegment(0,N),d_body);
+   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS,true>>(RAJA::RangeSegment(0,N),
+                                                      d_body);
 }
 
 template <typename DBODY>
@@ -462,18 +464,18 @@ struct RajaHipWrap;
 template <>
 struct RajaHipWrap<1>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
-      RajaHipWrap1D<BLCK>(N, d_body);
+      RajaHipWrap1D(N, d_body);
    }
 };
 
 template <>
 struct RajaHipWrap<2>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
@@ -484,7 +486,7 @@ struct RajaHipWrap<2>
 template <>
 struct RajaHipWrap<3>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
@@ -584,12 +586,31 @@ void CuKernel2D(const int N, BODY body)
    body(k);
 }
 
+// __launch_bounds__ second argument is omitted to get the default behavior
+template <int MAX_THREADS_PER_BLOCK, typename BODY>
+__global__
+MFEM_LAUNCH_BOUNDS(MAX_THREADS_PER_BLOCK)
+static void CuKernel2DLaunchBounds(const int N, BODY body)
+{
+   const int k = blockIdx.x*blockDim.z + threadIdx.z;
+   if (k >= N) { return; }
+   body(k);
+}
+
 template <typename BODY> __global__ static
 void CuKernel3D(const int N, BODY body)
 {
    for (int k = blockIdx.x; k < N; k += gridDim.x) { body(k); }
 }
 
+template <int MAX_THREADS_PER_BLOCK, typename BODY>
+__global__
+MFEM_LAUNCH_BOUNDS(MAX_THREADS_PER_BLOCK)
+static void CuKernel3DLaunchBounds(const int N, BODY body)
+{
+   for (int k = blockIdx.x; k < N; k += gridDim.x) { body(k); }
+}
+
 template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
 void CuWrap1D(const int N, DBODY &&d_body)
 {
@@ -604,6 +625,8 @@ void CuWrap2D(const int N, DBODY &&d_body,
               const int X, const int Y, const int BZ)
 {
    if (N==0) { return; }
+   // required for optimized GCC/NVCC builds to prevent runtime
+   // ODR/linkage violations of inlined templated kernel helpers
    MFEM_VERIFY(BZ>0, "");
    const int GRID = (N+BZ-1)/BZ;
    const dim3 BLCK(X,Y,BZ);
@@ -611,6 +634,19 @@ void CuWrap2D(const int N, DBODY &&d_body,
    MFEM_GPU_CHECK(cudaGetLastError());
 }
 
+template <int MAX_THREADS_PER_BLOCK, typename DBODY>
+void CuWrap2DLaunchBounds(const int N, DBODY &&d_body,
+                          const int X, const int Y, const int BZ)
+{
+   if (N==0) { return; }
+   MFEM_VERIFY(BZ>0, "");
+   const int GRID = (N+BZ-1)/BZ;
+   const dim3 BLCK(X,Y,BZ);
+   static_assert(MAX_THREADS_PER_BLOCK > 0);
+   CuKernel2DLaunchBounds<MAX_THREADS_PER_BLOCK><<<GRID,BLCK>>>(N, d_body);
+   MFEM_GPU_CHECK(cudaGetLastError());
+}
+
 template <typename DBODY>
 void CuWrap3D(const int N, DBODY &&d_body,
               const int X, const int Y, const int Z, const int G)
@@ -622,24 +658,35 @@ void CuWrap3D(const int N, DBODY &&d_body,
    MFEM_GPU_CHECK(cudaGetLastError());
 }
 
-template <int Dim>
-struct CuWrap;
-
-template <>
-struct CuWrap<1>
+template <int MAX_THREADS_PER_BLOCK, typename DBODY>
+void CuWrap3DLaunchBounds(const int N, DBODY &&d_body,
+                          const int X, const int Y, const int Z, const int G)
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   if (N==0) { return; }
+   const int GRID = G == 0 ? N : G;
+   const dim3 BLCK(X,Y,Z);
+   static_assert(MAX_THREADS_PER_BLOCK > 0);
+   CuKernel3DLaunchBounds<MAX_THREADS_PER_BLOCK><<<GRID, BLCK>>>(N, d_body);
+   MFEM_GPU_CHECK(cudaGetLastError());
+}
+
+template <int Dim, int MAX_THREADS_PER_BLOCK> struct CuWrap;
+
+template <int MAX_THREADS_PER_BLOCK>
+struct CuWrap<1, MAX_THREADS_PER_BLOCK>
+{
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
-      CuWrap1D<BLCK>(N, d_body);
+      CuWrap1D<MFEM_CUDA_BLOCKS>(N, d_body);
    }
 };
 
 template <>
-struct CuWrap<2>
+struct CuWrap<2, 0>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
@@ -647,10 +694,22 @@ struct CuWrap<2>
    }
 };
 
-template <>
-struct CuWrap<3>
+template <int MAX_THREADS_PER_BLOCK>
+struct CuWrap<2, MAX_THREADS_PER_BLOCK>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
+   static void run(const int N, DBODY &&d_body,
+                   const int X, const int Y, const int Z, const int G)
+   {
+      static_assert(MAX_THREADS_PER_BLOCK > 0);
+      CuWrap2DLaunchBounds<MAX_THREADS_PER_BLOCK>(N, d_body, X, Y, Z);
+   }
+};
+
+template <>
+struct CuWrap<3, 0>
+{
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
@@ -658,6 +717,17 @@ struct CuWrap<3>
    }
 };
 
+template <int MAX_THREADS_PER_BLOCK>
+struct CuWrap<3, MAX_THREADS_PER_BLOCK>
+{
+   template <typename DBODY>
+   static void run(const int N, DBODY &&d_body,
+                   const int X, const int Y, const int Z, const int G)
+   {
+      CuWrap3DLaunchBounds<MAX_THREADS_PER_BLOCK>(N, d_body, X, Y, Z, G);
+   }
+};
+
 #endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 
 
@@ -680,13 +750,31 @@ void HipKernel2D(const int N, BODY body)
    body(k);
 }
 
+template <int MAX_THREADS_PER_BLOCK, typename BODY>
+__global__
+MFEM_LAUNCH_BOUNDS(MAX_THREADS_PER_BLOCK)
+static void HipKernel2DLaunchBounds(const int N, BODY body)
+{
+   const int k = hipBlockIdx_x*hipBlockDim_z + hipThreadIdx_z;
+   if (k >= N) { return; }
+   body(k);
+}
+
 template <typename BODY> __global__ static
 void HipKernel3D(const int N, BODY body)
 {
    for (int k = hipBlockIdx_x; k < N; k += hipGridDim_x) { body(k); }
 }
 
-template <const int BLCK = MFEM_HIP_BLOCKS, typename DBODY>
+template <int MAX_THREADS_PER_BLOCK, typename BODY>
+__global__
+MFEM_LAUNCH_BOUNDS(MAX_THREADS_PER_BLOCK)
+static void HipKernel3DLaunchBounds(const int N, BODY body)
+{
+   for (int k = hipBlockIdx_x; k < N; k += hipGridDim_x) { body(k); }
+}
+
+template <int BLCK = MFEM_HIP_BLOCKS, typename DBODY>
 void HipWrap1D(const int N, DBODY &&d_body)
 {
    if (N==0) { return; }
@@ -700,12 +788,27 @@ void HipWrap2D(const int N, DBODY &&d_body,
                const int X, const int Y, const int BZ)
 {
    if (N==0) { return; }
+   MFEM_VERIFY(BZ>0, "");
    const int GRID = (N+BZ-1)/BZ;
    const dim3 BLCK(X,Y,BZ);
    hipLaunchKernelGGL(HipKernel2D,GRID,BLCK,0,nullptr,N,d_body);
    MFEM_GPU_CHECK(hipGetLastError());
 }
 
+template <int MAX_THREADS_PER_BLOCK, typename DBODY>
+void HipWrap2DLaunchBounds(const int N, DBODY &&d_body,
+                           const int X, const int Y, const int BZ)
+{
+   if (N==0) { return; }
+   MFEM_VERIFY(BZ>0, "");
+   const int GRID = (N+BZ-1)/BZ;
+   const dim3 BLCK(X,Y,BZ);
+   static_assert(MAX_THREADS_PER_BLOCK > 0);
+   HipKernel2DLaunchBounds<MAX_THREADS_PER_BLOCK><<<dim3(GRID), dim3(BLCK), 0, 0>>>
+   (N, d_body);
+   MFEM_GPU_CHECK(hipGetLastError());
+}
+
 template <typename DBODY>
 void HipWrap3D(const int N, DBODY &&d_body,
                const int X, const int Y, const int Z, const int G)
@@ -717,24 +820,36 @@ void HipWrap3D(const int N, DBODY &&d_body,
    MFEM_GPU_CHECK(hipGetLastError());
 }
 
-template <int Dim>
-struct HipWrap;
-
-template <>
-struct HipWrap<1>
+template <int MAX_THREADS_PER_BLOCK, typename DBODY>
+void HipWrap3DLaunchBounds(const int N, DBODY &&d_body,
+                           const int X, const int Y, const int Z, const int G)
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   if (N==0) { return; }
+   const int GRID = G == 0 ? N : G;
+   const dim3 BLCK(X,Y,Z);
+   static_assert(MAX_THREADS_PER_BLOCK > 0);
+   HipKernel3DLaunchBounds<MAX_THREADS_PER_BLOCK><<<dim3(GRID), dim3(BLCK), 0, 0>>>
+   (N, d_body);
+   MFEM_GPU_CHECK(hipGetLastError());
+}
+
+template <int Dim, int MAX_THREADS_PER_BLOCK> struct HipWrap;
+
+template <int MAX_THREADS_PER_BLOCK>
+struct HipWrap<1, MAX_THREADS_PER_BLOCK>
+{
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
-      HipWrap1D<BLCK>(N, d_body);
+      HipWrap1D<MFEM_HIP_BLOCKS>(N, d_body);
    }
 };
 
 template <>
-struct HipWrap<2>
+struct HipWrap<2, 0>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
@@ -742,10 +857,21 @@ struct HipWrap<2>
    }
 };
 
-template <>
-struct HipWrap<3>
+template <int MAX_THREADS_PER_BLOCK>
+struct HipWrap<2, MAX_THREADS_PER_BLOCK>
 {
-   template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
+   template <typename DBODY>
+   static void run(const int N, DBODY &&d_body,
+                   const int X, const int Y, const int Z, const int G)
+   {
+      HipWrap2DLaunchBounds<MAX_THREADS_PER_BLOCK>(N, d_body, X, Y, Z);
+   }
+};
+
+template <>
+struct HipWrap<3, 0>
+{
+   template <typename DBODY>
    static void run(const int N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
@@ -753,11 +879,24 @@ struct HipWrap<3>
    }
 };
 
+template <int MAX_THREADS_PER_BLOCK>
+struct HipWrap<3, MAX_THREADS_PER_BLOCK>
+{
+   template <typename DBODY>
+   static void run(const int N, DBODY &&d_body,
+                   const int X, const int Y, const int Z, const int G)
+   {
+      HipWrap3DLaunchBounds<MAX_THREADS_PER_BLOCK>(N, d_body, X, Y, Z, G);
+   }
+};
+
 #endif // defined(MFEM_USE_HIP) && defined(__HIP__)
 
 
-/// The forall kernel body wrapper
-template <const int DIM, typename d_lambda, typename h_lambda>
+///////////////////////////////////////////////////////////////////////////////
+/// Forall host & device kernel dispatch
+template <int DIM, int MAX_THREADS_PER_BLOCK = 0,
+          typename d_lambda, typename h_lambda>
 inline void ForallWrap(const bool use_dev, const int N,
                        d_lambda &&d_body, h_lambda &&h_body,
                        const int X=0, const int Y=0, const int Z=0,
@@ -790,7 +929,7 @@ inline void ForallWrap(const bool use_dev, const int N,
    // If Backend::CUDA is allowed, use it
    if (Device::Allows(Backend::CUDA))
    {
-      return CuWrap<DIM>::run(N, d_body, X, Y, Z, G);
+      return CuWrap<DIM, MAX_THREADS_PER_BLOCK>::run(N, d_body, X, Y, Z, G);
    }
 #endif
 
@@ -798,7 +937,7 @@ inline void ForallWrap(const bool use_dev, const int N,
    // If Backend::HIP is allowed, use it
    if (Device::Allows(Backend::HIP))
    {
-      return HipWrap<DIM>::run(N, d_body, X, Y, Z, G);
+      return HipWrap<DIM, MAX_THREADS_PER_BLOCK>::run(N, d_body, X, Y, Z, G);
    }
 #endif
 
@@ -827,7 +966,9 @@ backend_cpu:
    for (int k = 0; k < N; k++) { h_body(k); }
 }
 
-template <const int DIM, typename lambda>
+///////////////////////////////////////////////////////////////////////////////
+/// Forall host & device kernel wrappers
+template <int DIM, typename lambda>
 inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
                        const int X=0, const int Y=0, const int Z=0,
                        const int G=0)
@@ -835,6 +976,16 @@ inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
    ForallWrap<DIM>(use_dev, N, body, body, X, Y, Z, G);
 }
 
+template <int DIM, int MAX_THREADS_PER_BLOCK, typename lambda>
+inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
+                       const int X=0, const int Y=0, const int Z=0,
+                       const int G=0)
+{
+   ForallWrap<DIM, MAX_THREADS_PER_BLOCK>(use_dev, N, body, body, X, Y, Z, G);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// forall interfaces
 template<typename lambda>
 inline void forall(int N, lambda &&body) { ForallWrap<1>(true, N, body); }
 
@@ -843,7 +994,7 @@ inline void forall(int Nx, int Ny, lambda &&body)
 {
    if (Device::Allows(Backend::DEVICE_MASK))
    {
-      forall(Nx * Ny, [=] MFEM_HOST_DEVICE(int idx)
+      mfem::forall(Nx * Ny, [=] MFEM_HOST_DEVICE(int idx)
       {
          int j = idx / Nx;
          int i = idx % Nx;
@@ -879,7 +1030,7 @@ inline void forall(int Nx, int Ny, int Nz, lambda &&body)
 {
    if (Device::Allows(Backend::DEVICE_MASK))
    {
-      forall(Nx * Ny * Nz, [=] MFEM_HOST_DEVICE(int idx)
+      mfem::forall(Nx * Ny * Nz, [=] MFEM_HOST_DEVICE(int idx)
       {
          int i = idx % Nx;
          int j = idx / Nx;
@@ -927,6 +1078,12 @@ inline void forall_2D(int N, int X, int Y, lambda &&body)
    ForallWrap<2>(true, N, body, X, Y, 1);
 }
 
+template<int MAX_THREADS_PER_BLOCK, typename lambda>
+inline void forall_2D(int N, int X, int Y, lambda &&body)
+{
+   ForallWrap<2, MAX_THREADS_PER_BLOCK>(true, N, body, X, Y, 1);
+}
+
 template<typename lambda>
 inline void forall_2D_batch(int N, int X, int Y, int BZ, lambda &&body)
 {
@@ -939,6 +1096,12 @@ inline void forall_3D(int N, int X, int Y, int Z, lambda &&body)
    ForallWrap<3>(true, N, body, X, Y, Z, 0);
 }
 
+template<int MAX_THREADS_PER_BLOCK, typename lambda>
+inline void forall_3D(int N, int X, int Y, int Z, lambda &&body)
+{
+   ForallWrap<3, MAX_THREADS_PER_BLOCK>(true, N, body, X, Y, Z, 0);
+}
+
 template<typename lambda>
 inline void forall_3D_grid(int N, int X, int Y, int Z, int G, lambda &&body)
 {

general/globals.hpp

@@ -113,6 +113,10 @@ void SetGlobalMPI_Comm(MPI_Comm comm);
 /// to suppress the warning.
 const char* GetEnv(const char* name);
 
+/// Signed indices i -> -1 - i are used as a convention to encode orientation.
+inline MFEM_HOST_DEVICE int FlipIndexSign(int i) { return -1 - i; }
+inline MFEM_HOST_DEVICE int UnsignIndex(int i) { return i < 0 ? -1 - i : i; }
+
 } // namespace mfem
 
 #endif

general/hash.cpp

@@ -80,159 +80,4 @@ std::string HashFunction::GetHash() const
    return hash;
 }
 
-constexpr static uint64_t rotl64(uint64_t x, int r)
-{
-   return (x << r) | (x >> (64 - r));
-}
-
-void Hasher::init(uint64_t seed)
-{
-   data[0] = seed;
-   data[1] = seed;
-   nbytes = 0;
-}
-
-void Hasher::add_block(uint64_t k1, uint64_t k2)
-{
-   constexpr uint64_t c1 = 0x87c37b91114253d5ull;
-   constexpr uint64_t c2 = 0x4cf5ad432745937full;
-
-   k1 *= c1;
-   k1 = rotl64(k1, 31);
-   k1 *= c2;
-   data[0] ^= k1;
-
-   data[0] = rotl64(data[0], 27);
-   data[0] += data[1];
-   data[0] = data[0] * 5 + 0x52dce729ull;
-
-   k2 *= c2;
-   k2 = rotl64(k2, 33);
-   k2 *= c1;
-   data[1] ^= k2;
-
-   data[1] = rotl64(data[1], 31);
-   data[1] += data[0];
-   data[1] = data[1] * 5 + 0x38495ab5ull;
-}
-
-static uint64_t fmix64(uint64_t k)
-{
-   // http://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html
-   // mix13
-   k ^= k >> 30;
-   k *= 0xbf58476d1ce4e5b9ull;
-   k ^= k >> 27;
-   k *= 0x94d049bb133111ebull;
-   k ^= k >> 31;
-   return k;
-}
-
-void Hasher::append(const uint8_t *vs, uint64_t bytes)
-{
-   if (bytes == 0)
-   {
-      return;
-   }
-   auto rem = nbytes % 16;
-   nbytes += bytes;
-   uint8_t *tmp = reinterpret_cast<uint8_t *>(buf_);
-   while (true)
-   {
-      if (bytes + rem >= 16)
-      {
-         std::copy(vs, vs + 16 - rem, tmp + rem);
-         add_block(buf_[0], buf_[1]);
-         vs += (16 - rem);
-         bytes -= (16 - rem);
-         rem = 0;
-      }
-      else
-      {
-         std::copy(vs, vs + bytes, tmp + rem);
-         return;
-      }
-   }
-}
-
-void Hasher::finalize()
-{
-   auto rem = nbytes % 16;
-   if (rem > 0)
-   {
-      nbytes -= rem;
-      if (rem <= 8)
-      {
-         finalize(buf_[0], rem);
-      }
-      else
-      {
-         finalize(buf_[0], buf_[1], rem);
-      }
-      return;
-   }
-   data[0] ^= nbytes;
-   data[1] ^= nbytes;
-
-   data[0] += data[1];
-   data[1] += data[0];
-
-   data[0] = fmix64(data[0]);
-   data[1] = fmix64(data[1]);
-
-   data[0] += data[1];
-   data[1] += data[0];
-}
-
-void Hasher::finalize(uint64_t k1, int num)
-{
-   constexpr uint64_t c1 = 0x87c37b91114253d5ull;
-   constexpr uint64_t c2 = 0x4cf5ad432745937full;
-   nbytes += num;
-   k1 *= c1;
-   k1 = rotl64(k1, 31);
-   k1 *= c2;
-   data[0] ^= k1;
-
-   data[0] ^= nbytes;
-   data[1] ^= nbytes;
-
-   data[0] += data[1];
-   data[1] += data[0];
-
-   data[0] = fmix64(data[0]);
-   data[1] = fmix64(data[1]);
-
-   data[0] += data[1];
-   data[1] += data[0];
-}
-
-void Hasher::finalize(uint64_t k1, uint64_t k2, int num)
-{
-   constexpr uint64_t c1 = 0x87c37b91114253d5ull;
-   constexpr uint64_t c2 = 0x4cf5ad432745937full;
-   nbytes += num;
-   k2 *= c2;
-   k2 = rotl64(k2, 33);
-   k2 *= c1;
-   data[1] ^= k2;
-
-   k1 *= c1;
-   k1 = rotl64(k1, 31);
-   k1 *= c2;
-   data[0] ^= k1;
-
-   data[0] ^= nbytes;
-   data[1] ^= nbytes;
-
-   data[0] += data[1];
-   data[1] += data[0];
-
-   data[0] = fmix64(data[0]);
-   data[1] = fmix64(data[1]);
-
-   data[0] += data[1];
-   data[1] += data[0];
-}
-
 } // namespace mfem

general/hash.hpp

@@ -15,8 +15,8 @@
 #include "../config/config.hpp"
 #include "array.hpp"
 #include "globals.hpp"
+#include "hash_util.hpp"
 
-#include <array>
 #include <cstdint>
 #include <type_traits>
 #include <utility>
@@ -457,75 +457,6 @@ protected:
    int BinSize(int idx) const;
 };
 
-///
-/// @brief streaming implementation for murmurhash3 128 (x64).
-/// Constructs the hash in 3 stages: init, append, finalize.
-///
-struct Hasher
-{
-   /// where the final hash result is stored after finalize. Use data[1] when
-   /// only 64 bits are required.
-   uint64_t data[2] = {0, 0};
-
-private:
-   uint64_t nbytes = 0;
-
-   uint64_t buf_[2] = {0, 0};
-
-public:
-
-   /// resets this hasher back to an initial seed
-   void init(uint64_t seed = 0);
-   void append(const uint8_t *vs, uint64_t bytes);
-
-   void finalize();
-
-private:
-   // add 16 bytes
-   void add_block(uint64_t k1, uint64_t k2);
-
-   // add [1-8] more bytes, then finalize
-   void finalize(uint64_t k1, int num);
-
-   // add [1-15] more bytes, then finalize
-   // 0 < num < 16
-   void finalize(uint64_t k1, uint64_t k2, int num);
-};
-
-/// Helper class for hashing std::pair. Usable in place of std::hash<std::pair<T,U>>
-struct PairHasher
-{
-   template <class T, class V>
-   size_t operator()(const std::pair<T, V> &v) const noexcept
-   {
-      Hasher hash;
-      // chosen randomly with a 2^64-sided dice
-      hash.init(0xfebd1fe69813c14full);
-      hash.append(reinterpret_cast<const uint8_t *>(&v.first), sizeof(T));
-      hash.append(reinterpret_cast<const uint8_t *>(&v.second), sizeof(V));
-      hash.finalize();
-      return hash.data[1];
-   }
-};
-
-/// Helper class for hashing std::array. Usable in place of std::hash<std::array<T,N>>
-struct ArrayHasher
-{
-   template <class T, size_t N>
-   size_t operator()(const std::array<T, N> &v) const noexcept
-   {
-      Hasher hash;
-      // chosen randomly with a 2^64-sided dice
-      hash.init(0xfebd1fe69813c14full);
-      for (size_t i = 0; i < N; ++i)
-      {
-         hash.append(reinterpret_cast<const uint8_t *>(&v[i]), sizeof(T));
-      }
-      hash.finalize();
-      return hash.data[1];
-   }
-};
-
 /// Hash function for data sequences.
 /** Depends on GnuTLS for SHA-256 hashing. */
 class HashFunction

general/hash_util.cpp

@@ -0,0 +1,172 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "hash_util.hpp"
+
+namespace mfem
+{
+
+constexpr static uint64_t rotl64(uint64_t x, int r)
+{
+   return (x << r) | (x >> (64 - r));
+}
+
+void Hasher::init(uint64_t seed)
+{
+   data[0] = seed;
+   data[1] = seed;
+   nbytes = 0;
+}
+
+void Hasher::add_block(uint64_t k1, uint64_t k2)
+{
+   constexpr uint64_t c1 = 0x87c37b91114253d5ull;
+   constexpr uint64_t c2 = 0x4cf5ad432745937full;
+
+   k1 *= c1;
+   k1 = rotl64(k1, 31);
+   k1 *= c2;
+   data[0] ^= k1;
+
+   data[0] = rotl64(data[0], 27);
+   data[0] += data[1];
+   data[0] = data[0] * 5 + 0x52dce729ull;
+
+   k2 *= c2;
+   k2 = rotl64(k2, 33);
+   k2 *= c1;
+   data[1] ^= k2;
+
+   data[1] = rotl64(data[1], 31);
+   data[1] += data[0];
+   data[1] = data[1] * 5 + 0x38495ab5ull;
+}
+
+static uint64_t fmix64(uint64_t k)
+{
+   // http://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html
+   // mix13
+   k ^= k >> 30;
+   k *= 0xbf58476d1ce4e5b9ull;
+   k ^= k >> 27;
+   k *= 0x94d049bb133111ebull;
+   k ^= k >> 31;
+   return k;
+}
+
+void Hasher::append(const std::byte *vs, uint64_t bytes)
+{
+   if (bytes == 0)
+   {
+      return;
+   }
+   auto rem = nbytes % 16;
+   nbytes += bytes;
+   std::byte *tmp = reinterpret_cast<std::byte *>(buf_);
+   while (true)
+   {
+      if (bytes + rem >= 16)
+      {
+         std::copy(vs, vs + 16 - rem, tmp + rem);
+         add_block(buf_[0], buf_[1]);
+         vs += (16 - rem);
+         bytes -= (16 - rem);
+         rem = 0;
+      }
+      else
+      {
+         std::copy(vs, vs + bytes, tmp + rem);
+         return;
+      }
+   }
+}
+
+void Hasher::finalize()
+{
+   auto rem = nbytes % 16;
+   if (rem > 0)
+   {
+      nbytes -= rem;
+      if (rem <= 8)
+      {
+         finalize(buf_[0], rem);
+      }
+      else
+      {
+         finalize(buf_[0], buf_[1], rem);
+      }
+      return;
+   }
+   data[0] ^= nbytes;
+   data[1] ^= nbytes;
+
+   data[0] += data[1];
+   data[1] += data[0];
+
+   data[0] = fmix64(data[0]);
+   data[1] = fmix64(data[1]);
+
+   data[0] += data[1];
+   data[1] += data[0];
+}
+
+void Hasher::finalize(uint64_t k1, int num)
+{
+   constexpr uint64_t c1 = 0x87c37b91114253d5ull;
+   constexpr uint64_t c2 = 0x4cf5ad432745937full;
+   nbytes += num;
+   k1 *= c1;
+   k1 = rotl64(k1, 31);
+   k1 *= c2;
+   data[0] ^= k1;
+
+   data[0] ^= nbytes;
+   data[1] ^= nbytes;
+
+   data[0] += data[1];
+   data[1] += data[0];
+
+   data[0] = fmix64(data[0]);
+   data[1] = fmix64(data[1]);
+
+   data[0] += data[1];
+   data[1] += data[0];
+}
+
+void Hasher::finalize(uint64_t k1, uint64_t k2, int num)
+{
+   constexpr uint64_t c1 = 0x87c37b91114253d5ull;
+   constexpr uint64_t c2 = 0x4cf5ad432745937full;
+   nbytes += num;
+   k2 *= c2;
+   k2 = rotl64(k2, 33);
+   k2 *= c1;
+   data[1] ^= k2;
+
+   k1 *= c1;
+   k1 = rotl64(k1, 31);
+   k1 *= c2;
+   data[0] ^= k1;
+
+   data[0] ^= nbytes;
+   data[1] ^= nbytes;
+
+   data[0] += data[1];
+   data[1] += data[0];
+
+   data[0] = fmix64(data[0]);
+   data[1] = fmix64(data[1]);
+
+   data[0] += data[1];
+   data[1] += data[0];
+}
+
+}

general/hash_util.hpp

@@ -0,0 +1,172 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_HASH_UTIL_HPP
+#define MFEM_HASH_UTIL_HPP
+
+#include <array>
+#include <cstddef>
+#include <tuple>
+#include <functional>
+#include <utility>
+#include <cstdint>
+
+namespace mfem
+{
+
+/// @brief streaming implementation for murmurhash3 128 (x64).
+///
+/// Constructs the hash in 3 stages: init, append, finalize.
+struct Hasher
+{
+   /// @brief Storage for the final hash result after finalize() is called.
+   ///
+   /// Use data[1] when only 64 bits are required.
+   uint64_t data[2] = {0, 0};
+
+private:
+   uint64_t nbytes = 0;
+   uint64_t buf_[2] = {0, 0};
+
+public:
+
+   /// Resets the Hasher back to an initial seed
+   void init(uint64_t seed = 0);
+
+   /// Append data @a vs of size @a bytes.
+   void append(const std::byte *vs, uint64_t bytes);
+
+   void finalize();
+
+private:
+   /// Add a block of 16 bytes.
+   void add_block(uint64_t k1, uint64_t k2);
+
+   /// @brief Add [1-8] more bytes, then finalize.
+   ///
+   /// @a num must satisfy 0 < num < 9.
+   void finalize(uint64_t k1, int num);
+
+   /// @brief Add [1-15] more bytes, then finalize.
+   ///
+   /// @a num must satisfy 0 < num < 16.
+   void finalize(uint64_t k1, uint64_t k2, int num);
+};
+
+template <class T> struct ChainedHasher
+{
+   static void Append(Hasher &hasher, const T &value)
+   {
+      if constexpr (std::is_fundamental_v<T> || std::is_pointer_v<T>)
+      {
+         hasher.append(reinterpret_cast<const std::byte *>(&value), sizeof(T));
+      }
+      else
+      {
+         std::hash<T> h;
+         auto v = h(value);
+         hasher.append(reinterpret_cast<std::byte *>(&v), sizeof(v));
+      }
+   }
+};
+
+template <class T, class V> struct ChainedHasher<std::pair<T, V>>
+{
+   static void Append(Hasher &hasher, const std::pair<T, V> &value)
+   {
+      ChainedHasher<T>::Append(hasher, value.first);
+      ChainedHasher<V>::Append(hasher, value.second);
+   }
+};
+
+template <class T, size_t N> struct ChainedHasher<std::array<T, N>>
+{
+   static void Append(Hasher &hasher, const std::array<T, N> &value)
+   {
+      for (size_t i = 0; i < N; ++i)
+      {
+         ChainedHasher<T>::Append(hasher, value[i]);
+      }
+   }
+};
+
+template<class... Ts> struct ChainedHasher<std::tuple<Ts...>>
+{
+private:
+   template <size_t N>
+   static void AppendImpl(Hasher &hasher, const std::tuple<Ts...> &value)
+   {
+      ChainedHasher<std::decay_t<decltype(std::get<N>(value))>>::Append(
+                                                                hasher, std::get<N>(value));
+      if constexpr (N + 1 < sizeof...(Ts))
+      {
+         AppendImpl<N + 1>(hasher, value);
+      }
+   }
+
+public:
+   static void Append(Hasher &hasher, const std::tuple<Ts...> &value)
+   {
+      if constexpr (sizeof...(Ts))
+      {
+         AppendImpl<0>(hasher, value);
+      }
+   }
+};
+
+/// Helper class for hashing std::pair of hashable types.
+struct PairHasher
+{
+   template <class T, class V>
+   size_t operator()(const std::pair<T, V> &v) const noexcept
+   {
+      Hasher hash;
+      // chosen randomly with a 2^64-sided dice
+      hash.init(0xfebd1fe69813c14full);
+      ChainedHasher<std::pair<T, V>>::Append(hash, v);
+      hash.finalize();
+      return hash.data[1];
+   }
+};
+
+/// Helper class for hashing std::array of a hashable type.
+struct ArrayHasher
+{
+   template <class T, size_t N>
+   size_t operator()(const std::array<T, N> &v) const noexcept
+   {
+      Hasher hash;
+      // chosen randomly with a 2^64-sided dice
+      hash.init(0xfebd1fe69813c14full);
+      ChainedHasher<std::array<T, N>>::Append(hash, v);
+      hash.finalize();
+      return hash.data[1];
+   }
+};
+
+/// Helper class for hashing std::tuple of hashable types.
+struct TupleHasher
+{
+   template <class T>
+   size_t operator()(const T &v) const noexcept
+   {
+      Hasher hash;
+      // chosen randomly with a 2^64-sided dice
+      hash.init(0xfebd1fe69813c14full);
+      ChainedHasher<T>::Append(hash, v);
+      hash.finalize();
+      return hash.data[1];
+   }
+};
+
+} // namespace mfem
+
+#endif

general/hip.hpp

@@ -20,9 +20,11 @@
 
 #if defined(MFEM_USE_HIP) && defined(__HIP__)
 #define MFEM_USE_CUDA_OR_HIP
+constexpr bool mfem_use_gpu = true;
 #define MFEM_DEVICE __device__
 #define MFEM_HOST __host__
 #define MFEM_LAMBDA __host__ __device__
+#define MFEM_LAUNCH_BOUNDS __launch_bounds__
 // #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(hipDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(hipStreamSynchronize(0))

general/mem_manager.cpp

@@ -22,6 +22,14 @@
 //#define _WIN32
 //#define _aligned_malloc(s,a) malloc(s)
 
+#ifdef NVTX_DEBUG_HPP
+#undef NVTX_COLOR
+#define NVTX_COLOR ::nvtx::kGold
+#include NVTX_DEBUG_HPP
+#else
+#define dbg(...)
+#endif
+
 #ifndef _WIN32
 #include <unistd.h>
 #include <signal.h>
@@ -759,7 +767,14 @@ private:
    {
       switch (mt)
       {
-         case MT::HOST_DEBUG: return new MmuHostMemorySpace();
+         case MT::HOST_DEBUG:
+            if (GetEnv("MFEM_MMU_STD"))
+            {
+               dbg("Using STD memory space for debug device!");
+               return new StdHostMemorySpace();
+            }
+            dbg("Using MMU memory space for debug device!");
+            return new MmuHostMemorySpace();
 #ifdef MFEM_USE_UMPIRE
          case MT::HOST_UMPIRE:
             return new UmpireHostMemorySpace(
@@ -788,7 +803,9 @@ private:
          case MT::DEVICE_UMPIRE: return new NoDeviceMemorySpace();
          case MT::DEVICE_UMPIRE_2: return new NoDeviceMemorySpace();
 #endif
-         case MT::DEVICE_DEBUG: return new MmuDeviceMemorySpace();
+         case MT::DEVICE_DEBUG:
+            if (GetEnv("MFEM_MMU_STD")) { return new StdDeviceMemorySpace(); }
+            return new MmuDeviceMemorySpace();
          case MT::DEVICE:
          {
 #if defined(MFEM_USE_CUDA)