merge compile error fix

Use MPI_C_BOOL for PETSc>=3.24.0

.gitignore

@@ -313,6 +313,8 @@ miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex3
 miniapps/nurbs/nurbs_ex5
+miniapps/nurbs/nurbs_ex10
+miniapps/nurbs/nurbs_ex10p
 miniapps/nurbs/nurbs_ex11p
 miniapps/nurbs/nurbs_ex24
 miniapps/nurbs/nurbs_solenoidal
@@ -338,7 +340,14 @@ miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
 miniapps/nurbs/glvis_naca-cmesh.mesh
 miniapps/nurbs/Naca_cmesh
+miniapps/nurbs/nurbs_mesh_info
+miniapps/nurbs/k*_*.dat
 miniapps/nurbs/*-Surface.mesh
+miniapps/nurbs/*.mesh
+miniapps/nurbs/*.sol
+miniapps/nurbs/deformed.*
+miniapps/nurbs/elastic_energy.*
+miniapps/nurbs/velocity.*
 
 miniapps/performance/ex1
 miniapps/performance/ex1p

CHANGELOG

@@ -11,6 +11,22 @@
 Version 4.9.1 (development)
 ===========================
 
+Discretization improvements
+---------------------------
+- Improved the gridfunction projection routines. Projections work for Scalar,
+  Vector and VectorFE, also NURBS versions. Optionally different types of
+  projections can be selected, default behaviour has not changed.
+
+Meshing improvements
+--------------------
+- Improved support for 1D NURBS meshes with variable order, including using
+  the patches construct for 1D NURBS meshes.
+
+New and updated examples and miniapps
+-------------------------------------
+- Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
+  capability.
+
 
 Version 4.9, released on Dec 11, 2025
 =====================================

CMakeLists.txt

@@ -723,6 +723,7 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
 
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
+target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
 if (TPL_TARGETS)
@@ -869,11 +870,12 @@ add_dependencies(exec
 #  - https://cmake.org/Bug/view.php?id=8438
 
 # Add a target to copy the mfem data directory to the build directory
-add_custom_command(OUTPUT data_is_copied
-  COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/data data
-  COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
-  COMMENT "Copying the data directory ...")
-add_custom_target(copy_data DEPENDS data_is_copied)
+# Implementable as a single copy_directory_if_different command w/ CMake >= 3.26
+file(GLOB DATA_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/data/*)
+add_custom_target(copy_data
+  COMMAND ${CMAKE_COMMAND} -E make_directory data
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${DATA_FILES} data
+  COMMENT "Syncing the data directory ...")
 # Add 'copy_data' as a prerequisite for all executables, if the source and the
 # build directories are not the same.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
@@ -1005,6 +1007,10 @@ install(FILES
 install(EXPORT ${PROJECT_NAME_UC}Targets
     DESTINATION ${INSTALL_CMAKE_DIR})
 
+# Install the data directory if present, i.e. if the copy_data target is built
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
+  DESTINATION ${MFEM_INSTALL_DIR} OPTIONAL)
+
 #-------------------------------------------------------------------------------
 # Create 'config.mk' from 'config.mk.in' for the build and install locations and
 # define install rules for 'config.mk' and 'test.mk'

INSTALL

@@ -725,7 +725,9 @@ The specific libraries and their options are:
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
            Debug).
-  Versions: Ginkgo >= 1.9.0.
+  Versions: Ginkgo >= 1.9.0. When building Ginkgo with distributed support, a
+            recent version of the "develop" branch is required (1.11 as defined
+            in include/ginkgo/config.hpp).
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
   URL: https://github.com/NVIDIA/AMGX

config/defaults.mk

@@ -18,6 +18,7 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))
 
+ASTYLE_BIN = astyle
 ETAGS_BIN = $(shell command -v etags 2> /dev/null)
 EGREP_BIN = $(shell command -v egrep 2> /dev/null)
 
@@ -407,7 +408,7 @@ AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt
 # MAGMA library configuration
 MAGMA_DIR = @MFEM_DIR@/../magma
 MAGMA_OPT = -I$(MAGMA_DIR)/include
-MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a -lcublas -lcusparse $(LAPACK_LIB)
+MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a $(LAPACK_LIB)
 
 # GnuTLS library configuration
 GNUTLS_OPT =

data/nurbs-segments2d-patches-multispan.mesh

@@ -0,0 +1,86 @@
+MFEM NURBS mesh v1.0
+
+dimension
+1
+
+# Four segments with different NURBS orders, described via patches.
+elements
+4
+1 1 0 1
+2 1 2 3
+3 1 4 5
+4 1 6 7
+
+boundary
+0
+
+edges
+4
+0 0 1
+1 2 3
+2 4 5
+3 6 7
+
+vertices
+8
+
+patches
+
+# Patch 0: linear (order 1, 3 spans)
+knotvectors
+1
+1 4  0 0  .4 .6 1 1
+
+dimension
+2
+
+controlpoints
+0.0  0.0  1.0
+0.6  0.4  1.0
+0.4  0.6  1.0
+1.0  1.0  1.0
+
+# Patch 1: quadratic (order 2, 2 spans)
+knotvectors
+1
+2 4  0 0 0  .5 1 1 1
+
+dimension
+2
+
+controlpoints
+1.0  0.0  1.0
+1.9  0.0  1.21
+2.0  0.9  1.22
+2.0  1.0  1.0
+
+# Patch 2: cubic (order 3, 3 spans)
+knotvectors
+1
+3 6  0 0 0 0 .33 .66 1 1 1 1
+
+dimension
+2
+
+controlpoints
+2.0   0.0  1.0
+2.1   0.2  1.31
+3.5   0.4  1.32
+2.5   0.6  1.33
+2.9   1.0  1.34
+3.0   1.0  1.0
+
+# Patch 3: quartic (order 4, 1 span)
+knotvectors
+1
+4 5  0 0 0 0 0 1 1 1 1 1
+
+dimension
+2
+
+controlpoints
+3.0    0.0  1.0
+3.45   0.5  1.41
+3.50   1.0  1.42
+3.75   0.8  1.43
+4.0    0.0  1.0

data/nurbs-segments2d-patches.mesh

@@ -0,0 +1,79 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+# Three segments with different NURBS orders, described via patches.
+elements
+3
+1 1 0 1
+2 1 2 3
+3 1 4 5
+
+boundary
+6
+1 0 0
+1 0 1
+1 0 2
+1 0 3
+1 0 4
+1 0 5
+
+edges
+3
+0 0 1
+1 2 3
+2 4 5
+
+vertices
+6
+
+patches
+
+# Patch 0: linear (order 1, 2 control points)
+knotvectors
+1
+1 2  0 0  1 1
+
+dimension
+2
+
+controlpoints
+0.0  0.0  1.0
+1.0  1.0  1.0
+
+# Patch 1: quadratic (order 2, 3 control points)
+knotvectors
+1
+2 3  0 0 0  1 1 1
+
+dimension
+2
+
+controlpoints
+1.0  0.0   1.0
+1.02 1.02  1.2
+2.0  1.0   1.0
+
+# Patch 2: cubic (order 3, 4 control points)
+knotvectors
+1
+3 4  0 0 0 0  1 1 1 1
+
+dimension
+2
+
+controlpoints
+2.0   0.0   1.0
+2.03  0.83  1.31
+2.33  1.03  1.32
+3.0   1.0   1.0
+

data/nurbs-segments2d.mesh

@@ -0,0 +1,72 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+elements
+3
+1 1 0 1
+2 1 2 3
+3 1 4 5
+
+boundary
+6
+1 0 0
+1 0 1
+1 0 2
+1 0 3
+1 0 4
+1 0 5
+
+edges
+3
+0 0 1
+1 2 3
+2 4 5
+
+vertices
+6
+
+# Edge 0: linear (order 1, 2 control points)
+# Edge 1: quadratic (order 2, 3 control points)
+# Edge 2: cubic (order 3, 4 control points)
+knotvectors
+3
+1 2  0 0  1 1
+2 3  0 0 0  1 1 1
+3 4  0 0 0 0  1 1 1 1
+
+# One weight per control point, in the same order as the control points; (2 + 3 + 4) = 9 weights total
+weights
+1
+1
+1
+1
+1
+1
+1.2
+1.31
+1.32
+
+FiniteElementSpace
+FiniteElementCollection: NURBS
+VDim: 2
+Ordering: 1
+
+0.0  0.0
+1.0  1.0
+1.0  0.0
+2.0  1.0
+2.0  0.0
+3.0  1.0
+1.02 1.02
+2.03 0.83
+2.33 1.03

data/nurbs-segments3d-patches.mesh

@@ -0,0 +1,79 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+# Three segments with different NURBS orders, described via patches.
+elements
+3
+1 1 0 1
+2 1 2 3
+3 1 4 5
+
+boundary
+6
+1 0 0
+1 0 1
+1 0 2
+1 0 3
+1 0 4
+1 0 5
+
+edges
+3
+0 0 1
+1 2 3
+2 4 5
+
+vertices
+6
+
+patches
+
+# Patch 0: linear (order 1, 2 control points)
+knotvectors
+1
+1 2  0 0  1 1
+
+dimension
+3
+
+controlpoints
+0.0  0.0  0.01 1.0
+1.0  1.0  1.01 1.0
+
+# Patch 1: quadratic (order 2, 3 control points)
+knotvectors
+1
+2 3  0 0 0  1 1 1
+
+dimension
+3
+
+controlpoints
+1.0  0.0   0.02 1.0
+1.02 1.02  0.52 1.2
+2.0  1.0   1.02 1.0
+
+# Patch 2: cubic (order 3, 4 control points)
+knotvectors
+1
+3 4  0 0 0 0  1 1 1 1
+
+dimension
+3
+
+controlpoints
+2.0   0.0   0.03 1.0
+2.03  0.83  0.33 1.31
+2.33  1.03  0.63 1.32
+3.0   1.0   1.03 1.0
+

data/nurbs-segments3d.mesh

@@ -0,0 +1,72 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+elements
+3
+1 1 0 1
+2 1 2 3
+3 1 4 5
+
+boundary
+6
+1 0 0
+1 0 1
+1 0 2
+1 0 3
+1 0 4
+1 0 5
+
+edges
+3
+0 0 1
+1 2 3
+2 4 5
+
+vertices
+6
+
+# Edge 0: linear (order 1, 2 control points)
+# Edge 1: quadratic (order 2, 3 control points)
+# Edge 2: cubic (order 3, 4 control points)
+knotvectors
+3
+1 2  0 0  1 1
+2 3  0 0 0  1 1 1
+3 4  0 0 0 0  1 1 1 1
+
+# One weight per control point, in the same order as the control points; (2 + 3 + 4) = 9 weights total
+weights
+1
+1
+1
+1
+1
+1
+1.2
+1.31
+1.32
+
+FiniteElementSpace
+FiniteElementCollection: NURBS
+VDim: 3
+Ordering: 1
+
+0.0  0.0   0.01
+1.0  1.0   1.01
+1.0  0.0   0.02
+2.0  1.0   1.02
+2.0  0.0   0.03
+3.0  1.0   1.03
+1.02 1.02  0.52
+2.03 0.83  0.33
+2.33 1.03  0.63

doc/CodeDocumentation.dox

@@ -117,6 +117,8 @@ namespace mfem {
  * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
  * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
  * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
+ * - <a class="el" href="ex41_8cpp_source.html">Example 41</a>: DG/CG IMEX time-dependent advection-diffusion
+ * - <a class="el" href="ex41p_8cpp_source.html">Example 41p</a>: parallel DG/CG IMEX time-dependent advection-diffusion
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples
@@ -188,6 +190,8 @@ namespace mfem {
  *  <a class="el" href="nurbs__ex1p_8cpp_source.html">1p</a>,
  *  <a class="el" href="nurbs__ex3_8cpp_source.html">3</a>,
  *  <a class="el" href="nurbs__ex5_8cpp_source.html">5</a>,
+ *  <a class="el" href="nurbs__ex10_8cpp_source.html">10</a>,
+ *  <a class="el" href="nurbs__ex10p_8cpp_source.html">10p</a>,
  *  <a class="el" href="nurbs__ex11p_8cpp_source.html">11p</a>, and
  *  <a class="el" href="nurbs__ex24_8cpp_source.html">24</a>,
  *   demonstrating howto perform NURBS-based Isogeometric Analysis.
@@ -196,6 +200,7 @@ namespace mfem {
  * - <a class="el" href="nurbs__curveint_8cpp_source.html">NURBS Interpolation</a>: NURBS interpolation of given geometry
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
+ * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -234,7 +239,8 @@ namespace mfem {
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Poisson problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Poisson problem
  * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
- * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Tribol</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="contact_8cpp_source.html">Contact</a>: Frictionless contact examples using <a class="el" href="classmfem_1_1IPSolver.html#details">IP optimization</a> and the <a class="el" href="classmfem_1_1AMGFSolver.html#details">AMGF solver</a>
  * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem

examples/ex10.cpp

@@ -412,7 +412,7 @@ void ReducedSystemOperator::Mult(const Vector &k, Vector &y) const
 Operator &ReducedSystemOperator::GetGradient(const Vector &k) const
 {
    delete Jacobian;
-   Jacobian = Add((real_t)1.0, M->SpMat(), dt, S->SpMat());
+   Jacobian = Add(1.0, M->SpMat(), dt, S->SpMat());
    add(*v, dt, k, w);
    add(*x, dt, w, z);
    SparseMatrix *grad_H = dynamic_cast<SparseMatrix *>(&H->GetGradient(z));

examples/ex10p.cpp

@@ -476,7 +476,7 @@ void ReducedSystemOperator::Mult(const Vector &k, Vector &y) const
 Operator &ReducedSystemOperator::GetGradient(const Vector &k) const
 {
    delete Jacobian;
-   SparseMatrix *localJ = Add((real_t)1.0, M->SpMat(), dt, S->SpMat());
+   SparseMatrix *localJ = Add(1.0, M->SpMat(), dt, S->SpMat());
    add(*v, dt, k, w);
    add(*x, dt, w, z);
    localJ->Add(dt*dt, H->GetLocalGradient(z));

examples/ex16.cpp

@@ -105,6 +105,7 @@ int main(int argc, char *argv[])
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
+   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -126,6 +127,9 @@ int main(int argc, char *argv[])
                   "Alpha coefficient.");
    args.AddOption(&kappa, "-k", "--kappa",
                   "Kappa coefficient offset.");
+   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
+                  "-imp-slope", "--implicit-slope",
+                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -179,6 +183,11 @@ int main(int argc, char *argv[])
 
    // 7. Initialize the conduction operator and the visualization.
    ConductionOperator oper(fespace, alpha, kappa, u);
+   using ImplicitVariableType = ConductionOperator::ImplicitVariableType;
+   ImplicitVariableType imp_var = solve_implicit_state ?
+                                  ImplicitVariableType::STATE
+                                  : ImplicitVariableType::SLOPE;
+   oper.SetImplicitVariableType(imp_var);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -316,21 +325,35 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 }
 
 void ConductionOperator::ImplicitSolve(const real_t dt,
-                                       const Vector &u, Vector &du_dt)
+                                       const Vector &u, Vector &k)
 {
    // Solve the equation:
-   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    M*k = -K(u + dt*k) for k = du/dt, if solving for stage-slope
+   // or
+   //    M*k = -dt*K(k) + M*u for k = u_s, if solving for stage-state
+   // where K is linearized by using u from the previous timestep, and
+   // the stage-state and slope relation: du/dt = (u_s - u)/dt.
    if (!T)
    {
-      T = Add((real_t)1.0, Mmat, dt, Kmat);
+      T = Add(1.0, Mmat, dt, Kmat);
       current_dt = dt;
       T_solver.SetOperator(*T);
    }
    MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-   Kmat.Mult(u, z);
-   z.Neg();
-   T_solver.Mult(z, du_dt);
+
+   // Construct current right-hand side for stage state vs. slope solve
+   if (ImplicitVarTypeIsState())
+   {
+      // k, on return, is the stage value u_s
+      Mmat.Mult(u, z);
+   }
+   else
+   {
+      // k, on return, is the stage slope du/dt
+      Kmat.Mult(u, z);
+      z.Neg();
+   }
+   T_solver.Mult(z, k);
 }
 
 void ConductionOperator::SetParameters(const Vector &u)

examples/ex16p.cpp

@@ -115,6 +115,7 @@ int main(int argc, char *argv[])
    bool visit = false;
    int vis_steps = 5;
    bool adios2 = false;
+   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -138,6 +139,9 @@ int main(int argc, char *argv[])
                   "Alpha coefficient.");
    args.AddOption(&kappa, "-k", "--kappa",
                   "Kappa coefficient offset.");
+   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
+                  "-imp-slope", "--implicit-slope",
+                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -212,6 +216,11 @@ int main(int argc, char *argv[])
 
    // 9. Initialize the conduction operator and the VisIt visualization.
    ConductionOperator oper(fespace, alpha, kappa, u);
+   using ImplicitVariableType = ConductionOperator::ImplicitVariableType;
+   ImplicitVariableType imp_var = solve_implicit_state ?
+                                  ImplicitVariableType::STATE
+                                  : ImplicitVariableType::SLOPE;
+   oper.SetImplicitVariableType(imp_var);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -407,21 +416,35 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 }
 
 void ConductionOperator::ImplicitSolve(const real_t dt,
-                                       const Vector &u, Vector &du_dt)
+                                       const Vector &u, Vector &k)
 {
    // Solve the equation:
-   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    M*k = -K(u + dt*k) for k = du/dt, if solving for stage-slope
+   // or
+   //    M*k = -dt*K(k) + M*u for k = u_s, if solving for stage-state
+   // where K is linearized by using u from the previous timestep, and
+   // the stage-state and slope relation: du/dt = (u_s - u)/dt.
    if (!T)
    {
-      T = Add((real_t)1.0, Mmat, dt, Kmat);
+      T = Add(1.0, Mmat, dt, Kmat);
       current_dt = dt;
       T_solver.SetOperator(*T);
    }
    MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-   Kmat.Mult(u, z);
-   z.Neg();
-   T_solver.Mult(z, du_dt);
+
+   // Construct current right-hand side for stage state vs. slope solve
+   if (ImplicitVarTypeIsState())
+   {
+      // k, on return, is the stage value u
+      Mmat.Mult(u, z);
+   }
+   else
+   {
+      // k, on return, is the stage slope du/dt
+      Kmat.Mult(u, z);
+      z.Neg();
+   }
+   T_solver.Mult(z, k);
 }
 
 void ConductionOperator::SetParameters(const Vector &u)

examples/ex21.cpp

@@ -119,7 +119,7 @@ int main(int argc, char *argv[])
    }
 
    LinearForm b(&fespace);
-   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
+   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
 
    // 6. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the linear elasticity integrator with piece-wise

examples/ex21p.cpp

@@ -140,7 +140,7 @@ int main(int argc, char *argv[])
    }
 
    ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
+   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
 
    // 6. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the linear elasticity integrator with piece-wise

examples/ex23.cpp

@@ -139,7 +139,7 @@ void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
    // for d2udt2
    if (!T)
    {
-      T = Add((real_t)1.0, Mmat, fac0, Kmat);
+      T = Add(1.0, Mmat, fac0, Kmat);
       T_solver.SetOperator(*T);
    }
    K->FullMult(u, z);

examples/ex3.cpp

@@ -56,51 +56,6 @@ void f_exact(const Vector &, Vector &);
 real_t freq = 1.0, kappa;
 int dim;
 
-void SolveSingle(SparseMatrix &A, const Vector &B, Vector &X)
-{
-   VectorMP<float> Bs, Xs;
-   const real_t *data = A.GetData();
-
-   const int n = A.GetI()[A.NumRows()];
-   int *Icopy = new int[A.NumRows() + 1];
-   int *Jcopy = new int[n];
-
-   float *sdata = new float[n];
-   for (int i=0; i<n; ++i)
-   {
-      sdata[i] = data[i];
-      Jcopy[i] = A.GetJ()[i];
-   }
-
-   for (int i=0; i<A.NumRows() + 1; ++i)
-   {
-      Icopy[i] = A.GetI()[i];
-   }
-
-   SparseMatrixMP<float> As(Icopy, Jcopy, sdata, A.NumRows(), A.NumCols());
-
-   Bs.SetSize(B.Size());
-   Xs.SetSize(X.Size());
-
-   for (int i=0; i<B.Size(); ++i)
-   {
-      Bs[i] = B[i];
-   }
-
-   for (int i=0; i<X.Size(); ++i)
-   {
-      Xs[i] = X[i];
-   }
-
-   GSSmootherMP<float> Ms(As);
-   PCG<float>(As, Ms, Bs, Xs, 1, 500, 1e-12, 0.0);
-
-   for (int i=0; i<X.Size(); ++i)
-   {
-      X[i] = Xs[i];
-   }
-}
-
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options.
@@ -230,7 +185,6 @@ int main(int argc, char *argv[])
 
    cout << "Size of linear system: " << A->Height() << endl;
 
-   /*
    // 11. Solve the linear system A X = B.
    if (pa) // Jacobi preconditioning in partial assembly mode
    {
@@ -239,23 +193,20 @@ int main(int argc, char *argv[])
    }
    else
    {
-   #ifndef MFEM_USE_SUITESPARSE
+#ifndef MFEM_USE_SUITESPARSE
       // 11. Define a simple symmetric Gauss-Seidel preconditioner and use it to
       //     solve the system Ax=b with PCG.
       GSSmoother M((SparseMatrix&)(*A));
       PCG(*A, M, B, X, 1, 500, 1e-12, 0.0);
-   #else
+#else
       // 11. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
       //     system.
       UMFPackSolver umf_solver;
       umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
       umf_solver.SetOperator(*A);
       umf_solver.Mult(B, X);
-   #endif
+#endif
    }
-   */
-
-   SolveSingle((SparseMatrix&)(*A), B, X);
 
    // 12. Recover the solution as a finite element grid function.
    a->RecoverFEMSolution(X, *b, x);

examples/ex3p.cpp

@@ -297,18 +297,6 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   VectorMP<float> xf(x.Size());
-   for (int i=0; i<x.Size(); ++i)
-   {
-      xf[i] = x[i];
-   }
-
-   if (myid == 0)
-   {
-      cout << "Norm of x " << x.Norml2() << endl;
-      cout << "Norm of xf " << xf.Norml2() << endl;
-   }
-
    // 18. Free the used memory.
    delete a;
    delete sigma;

examples/ex9.cpp

@@ -160,6 +160,7 @@ int main(int argc, char *argv[])
    bool paraview = false;
    bool binary = false;
    int vis_steps = 5;
+   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -187,6 +188,9 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
+   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
+                  "-imp-slope", "--implicit-slope",
+                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -366,6 +370,11 @@ int main(int argc, char *argv[])
    //    right-hand side, and perform time-integration (looping over the time
    //    iterations, ti, with a time-step dt).
    FE_Evolution adv(m, k, b);
+   using ImplicitVariableType = FE_Evolution::ImplicitVariableType;
+   ImplicitVariableType imp_var = solve_implicit_state ?
+                                  ImplicitVariableType::STATE
+                                  : ImplicitVariableType::SLOPE;
+   adv.SetImplicitVariableType(imp_var);
 
    real_t t = 0.0;
    adv.SetTime(t);
@@ -459,7 +468,17 @@ void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
    MFEM_VERIFY(dg_solver != NULL,
                "Implicit time integration is not supported with partial assembly");
-   K.Mult(x, z);
+   // Construct current right-hand side for stage state vs. slope solve
+   if (ImplicitVarTypeIsState())
+   {
+      // k, on return, is the stage value u
+      M.Mult(x, z);
+   }
+   else
+   {
+      // k, on return, is the stage slope du/dt
+      K.Mult(x, z);
+   }
    z += b;
    dg_solver->SetTimeStep(dt);
    dg_solver->Mult(z, k);

examples/ex9p.cpp

@@ -257,6 +257,7 @@ int main(int argc, char *argv[])
    bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
+   bool solve_implicit_state = false;
 #if MFEM_HYPRE_VERSION >= 21800
    PrecType prec_type = PrecType::AIR;
 #else
@@ -290,6 +291,9 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
+   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
+                  "-imp-slope", "--implicit-slope",
+                  "Implicitly solve for stage state or slope.");
    args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
                   "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -536,6 +540,11 @@ int main(int argc, char *argv[])
    //     right-hand side, and perform time-integration (looping over the time
    //     iterations, ti, with a time-step dt).
    FE_Evolution adv(*m, *k, *B, prec_type);
+   using ImplicitVariableType = FE_Evolution::ImplicitVariableType;
+   ImplicitVariableType imp_var = solve_implicit_state ?
+                                  ImplicitVariableType::STATE
+                                  : ImplicitVariableType::SLOPE;
+   adv.SetImplicitVariableType(imp_var);
 
    real_t t = 0.0;
    adv.SetTime(t);
@@ -676,7 +685,17 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
 //    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
-   K->Mult(x, z);
+   // Construct current right-hand side for stage state vs. slope solve
+   if (ImplicitVarTypeIsState())
+   {
+      // k, on return, is the stage value u
+      M->Mult(x, z);
+   }
+   else
+   {
+      // k, on return, is the stage slope du/dt
+      K->Mult(x, z);
+   }
    z += b;
    dg_solver->SetTimeStep(dt);
    dg_solver->Mult(z, k);

examples/ginkgo/CMakeLists.txt

@@ -14,6 +14,12 @@ list(APPEND GINKGO_EXAMPLES_SRCS
   ex1.cpp
 )
 
+if (MFEM_USE_MPI AND GINKGO_BUILD_MPI)
+    list(APPEND GINKGO_EXAMPLES_SRCS
+    ex1p.cpp
+)
+endif()
+
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 

examples/ginkgo/ex1.cpp

@@ -207,7 +207,7 @@ int main(int argc, char *argv[])
             Ginkgo::IcPreconditioner ginkgo_precond(exec, "paric", 30);
             Ginkgo::CGSolver ginkgo_solver(exec, ginkgo_precond);
             ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(1e-12);
+            ginkgo_solver.SetRelTol(sqrt(1e-12));
             ginkgo_solver.SetAbsTol(0.0);
             ginkgo_solver.SetMaxIter(400);
             ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -225,7 +225,7 @@ int main(int argc, char *argv[])
             Ginkgo::MFEMPreconditioner gko_M(exec, M);
             Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
             ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(1e-12);
+            ginkgo_solver.SetRelTol(sqrt(1e-12));
             ginkgo_solver.SetAbsTol(0.0);
             ginkgo_solver.SetMaxIter(400);
             ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -283,7 +283,7 @@ int main(int argc, char *argv[])
                Ginkgo::MFEMPreconditioner gko_M(exec, M);
                Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
                ginkgo_solver.SetPrintLevel(print_lvl);
-               ginkgo_solver.SetRelTol(1e-12);
+               ginkgo_solver.SetRelTol(sqrt(1e-12));
                ginkgo_solver.SetAbsTol(0.0);
                ginkgo_solver.SetMaxIter(400);
                ginkgo_solver.SetOperator(*(A.Ptr()));

examples/ginkgo/ex1p.cpp

@@ -0,0 +1,436 @@
+//                       MFEM Example 1 - Parallel Version
+//                             GINKGO Modification
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
+//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
+//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Device sample runs:
+//               mpirun -np 4 ex1p -pa -d cuda
+//               mpirun -np 4 ex1p -fa -d cuda
+//               mpirun -np 4 ex1p -pa -d occa-cuda
+//               mpirun -np 4 ex1p -pa -d raja-omp
+//               mpirun -np 4 ex1p -pa -d ceed-cpu
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/square-mixed.mesh
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
+//             * mpirun -np 4 ex1p -pa -d ceed-cuda
+//             * mpirun -np 4 ex1p -pa -d ceed-hip
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifndef MFEM_USE_GINKGO
+#error This example requires that MFEM is built with MFEM_USE_GINKGO=YES
+#endif
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool pa = false;
+   bool fa = false;
+   const char *device_config = "cpu";
+   bool visualization = true;
+   int solver_config = 0;
+   int print_lvl = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&solver_config, "-s", "--solver-config",
+                  "Solver and preconditioner combination: \n\t"
+                  "   0 - Ginkgo solver and Ginkgo preconditioner, \n\t"
+                  "   1 - Ginkgo solver and MFEM preconditioner, \n\t"
+                  "   2 - MFEM solver and Ginkgo preconditioner, \n\t"
+                  "   3 - MFEM solver and MFEM preconditioner.");
+   args.AddOption(&print_lvl, "-pl", "--print-level",
+                  "Print level for iterative solver (1 prints every iteration).");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.SetGPUAwareMPI(true);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh.UniformRefinement();
+      }
+   }
+
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   bool delete_fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+      delete_fec = true;
+   }
+   else if (pmesh.GetNodes())
+   {
+      fec = pmesh.GetNodes()->OwnFEC();
+      delete_fec = false;
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+      delete_fec = true;
+   }
+   ParFiniteElementSpace fespace(&pmesh, fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 8. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm b(&fespace);
+   ConstantCoefficient one(1.0);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
+
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
+   ParBilinearForm a(&fespace);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (fa)
+   {
+      a.SetAssemblyLevel(AssemblyLevel::FULL);
+      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
+      // when Device::IsEnabled() returns true). This makes the results
+      // bit-for-bit deterministic at the cost of somewhat longer run time.
+      a.EnableSparseMatrixSorting(Device::IsEnabled());
+   }
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 12. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // 13. Solve the linear system A X = B.
+   if (!pa)
+   {
+      switch (solver_config)
+      {
+         // Solve the linear system with CG + Schwarz (with IC) from Ginkgo
+         case 0:
+         {
+            if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
+            Ginkgo::GinkgoExecutor exec(device);
+            Ginkgo::IcPreconditioner local_solver(exec, "exact");
+            Ginkgo::SchwarzPreconditioner gko_M(exec, MPI_COMM_WORLD, local_solver);
+            Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
+            ginkgo_solver.SetPrintLevel(print_lvl);
+            ginkgo_solver.SetRelTol(sqrt(1e-12));
+            ginkgo_solver.SetAbsTol(0.0);
+            ginkgo_solver.SetMaxIter(400);
+            ginkgo_solver.SetOperator(*(A.Ptr()));
+            ginkgo_solver.Mult(B, X);
+            break;
+         }
+
+         // Solve the linear system with CG from Ginkgo + MFEM preconditioner
+         case 1:
+         {
+            if (myid == 0) { cout << "Using Ginkgo solver + MFEM preconditioner...\n"; }
+            Ginkgo::GinkgoExecutor exec(device);
+            //Create MFEM preconditioner and wrap it for Ginkgo's use.
+            HypreBoomerAMG M((HypreParMatrix&)(*A));
+            Ginkgo::MFEMPreconditioner gko_M(exec, M, MPI_COMM_WORLD);
+            Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
+            ginkgo_solver.SetPrintLevel(print_lvl);
+            ginkgo_solver.SetRelTol(sqrt(1e-12));
+            ginkgo_solver.SetAbsTol(0.0);
+            ginkgo_solver.SetMaxIter(400);
+            ginkgo_solver.SetOperator(*(A.Ptr()));
+            ginkgo_solver.Mult(B, X);
+            break;
+         }
+
+         // Ginkgo Schwarz preconditioner (local ParIC) +  MFEM CG solver
+         case 2:
+         {
+            if (myid == 0) { cout << "Using MFEM solver + Ginkgo preconditioner...\n"; }
+            Ginkgo::GinkgoExecutor exec(device);
+            Ginkgo::IcPreconditioner local_M(exec, "exact");
+            Ginkgo::SchwarzPreconditioner M(exec, MPI_COMM_WORLD, local_M);
+            M.SetOperator(*(A.Ptr()));  // Generate the preconditioner for the matrix A.
+            CGSolver cg(MPI_COMM_WORLD);
+            cg.SetRelTol(sqrt(1e-12));
+            cg.SetMaxIter(400);
+            cg.SetPrintLevel(1);
+            cg.SetPreconditioner(M);
+            cg.SetOperator(*A);
+            cg.Mult(B, X);
+            break;
+         }
+
+         // MFEM solver + MFEM preconditioner
+         case 3:
+         {
+            if (myid == 0) { cout << "Using MFEM solver + MFEM preconditioner...\n"; }
+            HypreBoomerAMG M((HypreParMatrix&)(*A));
+            CGSolver cg(MPI_COMM_WORLD);
+            cg.SetRelTol(sqrt(1e-12));
+            cg.SetMaxIter(400);
+            cg.SetPrintLevel(1);
+            cg.SetPreconditioner(M);
+            cg.SetOperator(*A);
+            cg.Mult(B, X);
+            break;
+         }
+      } // End switch on solver_config
+   }
+   // Partial assembly mode.  Cannot use Ginkgo preconditioners, but can use Ginkgo
+   // solvers.
+   else
+   {
+      if (UsesTensorBasis(fespace))
+      {
+         // Use Jacobi preconditioning in partial assembly mode.
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         switch (solver_config)
+         {
+            case 0:
+            {
+               if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
+               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
+                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
+               break;
+            }
+
+            // Use Ginkgo solver with MFEM preconditioner
+            case 1:
+            {
+               if (myid == 0) { cout << "Using Ginkgo solver + MFEM preconditioner...\n"; }
+               Ginkgo::GinkgoExecutor exec(device);
+               // Wrap MFEM preconditioner for Ginkgo's use.
+               Ginkgo::MFEMPreconditioner gko_M(exec, M, MPI_COMM_WORLD);
+               Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
+               ginkgo_solver.SetPrintLevel(print_lvl);
+               ginkgo_solver.SetRelTol(sqrt(1e-12));
+               ginkgo_solver.SetAbsTol(0.0);
+               ginkgo_solver.SetMaxIter(400);
+               ginkgo_solver.SetOperator(*(A.Ptr()));
+               ginkgo_solver.Mult(B, X);
+               break;
+            }
+
+            // No Ginkgo preconditioners work with matrix-free; error
+            case 2:
+            {
+               if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
+               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
+                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
+               break;
+            }
+
+            // Use MFEM solver and preconditioner
+            case 3:
+            {
+               if (myid == 0) { cout << "Using MFEM solver + MFEM preconditioner...\n"; }
+               CGSolver cg(MPI_COMM_WORLD);
+               cg.SetRelTol(sqrt(1e-12));
+               cg.SetMaxIter(400);
+               cg.SetPrintLevel(1);
+               cg.SetPreconditioner(M);
+               cg.SetOperator(*A);
+               cg.Mult(B, X);
+               break;
+            }
+         } // End switch on solver_config
+      }
+      else // CG with no preconditioning
+      {
+         if (myid == 0) { cout << "Using MFEM solver + no preconditioner...\n"; }
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(sqrt(1e-12));
+         cg.SetMaxIter(400);
+         cg.SetPrintLevel(1);
+         cg.SetOperator(*A);
+         cg.Mult(B, X);
+      }
+   }
+
+   // 14. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   if (delete_fec)
+   {
+      delete fec;
+   }
+
+   return 0;
+}

examples/ginkgo/makefile

@@ -20,9 +20,8 @@ CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-# Currently there are only serial Ginkgo examples
 SEQ_EXAMPLES = ex1
-PAR_EXAMPLES =
+PAR_EXAMPLES = ex1p
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else

fem/ceed/interface/util.hpp

@@ -33,6 +33,7 @@ class FiniteElement;
 class FiniteElementSpace;
 class ElementTransformation;
 class IntegrationRule;
+class Vector;
 
 /** @brief Function that determines if a CEED kernel should be used, based on
     the current mfem::Device configuration. */

fem/coefficient.cpp

@@ -1302,6 +1302,73 @@ real_t TraceCoefficient::Eval(ElementTransformation &T,
    return ma.Trace();
 }
 
+VectorComponentCoefficient::VectorComponentCoefficient(VectorCoefficient &A,
+                                                       int c)
+   : a(&A), va(A.GetVDim())
+{
+   SetComponent(c);
+}
+
+void VectorComponentCoefficient::SetComponent(int c)
+{
+   MFEM_ASSERT(c < a->GetVDim() && c >= 0,
+               "VectorComponentCoefficient:  "
+               "Index not in range.");
+
+   component = c;
+}
+
+void VectorComponentCoefficient::SetTime(real_t t)
+{
+   if (a) { a->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+real_t VectorComponentCoefficient::Eval(ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   a->Eval(va, T, ip);
+   return va[component];
+}
+
+MatrixComponentCoefficient::MatrixComponentCoefficient(MatrixCoefficient &A,
+                                                       int ri, int ci)
+   : a(&A), ma(A.GetHeight(), A.GetWidth())
+{
+   SetRowIndex(ri);
+   SetColumnIndex(ci);
+}
+
+void MatrixComponentCoefficient::SetRowIndex(int ri)
+{
+   MFEM_ASSERT(ri < a->GetHeight() && ri >= 0,
+               "MatrixComponentCoefficient:  "
+               "Row index not in range.");
+
+   row_idx = ri;
+}
+
+void MatrixComponentCoefficient::SetColumnIndex(int ci)
+{
+   MFEM_ASSERT(ci < a->GetWidth() && ci >= 0,
+               "MatrixComponentCoefficient:  "
+               "Column index not in range.");
+   col_idx = ci;
+}
+
+void MatrixComponentCoefficient::SetTime(real_t t)
+{
+   if (a) { a->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+real_t MatrixComponentCoefficient::Eval(ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   a->Eval(ma, T, ip);
+   return ma(row_idx,col_idx);
+}
+
 VectorSumCoefficient::VectorSumCoefficient(int dim)
    : VectorCoefficient(dim),
      ACoef(NULL), BCoef(NULL),

fem/coefficient.hpp

@@ -114,11 +114,10 @@ public:
    /// Construct the constant coefficient using a vector of constants.
    /** @a c should be a vector defined by attributes, so for region with
        attribute @a i @a c[i-1] is the coefficient in that region */
-   PWConstCoefficient(Vector &c)
-   { constants.SetSize(c.Size()); constants=c; }
+   PWConstCoefficient(const Vector &c) { UpdateConstants(c); }
 
    /// Update the constants with vector @a c.
-   void UpdateConstants(Vector &c) { constants.SetSize(c.Size()); constants=c; }
+   void UpdateConstants(const Vector &c) { constants = c; }
 
    /// Return a reference to the i-th constant
    real_t &operator()(int i) { return constants(i-1); }
@@ -1332,8 +1331,8 @@ public:
    /// Get the coefficient located at (i,j) in the matrix.
    Coefficient* GetCoeff (int i, int j) { return Coeff[i*width+j]; }
 
-   /** @brief Set the coefficient located at (i,j) in the matrix.  By default by
-       default this will take ownership of the Coefficient passed in, but this
+   /** @brief Set the coefficient located at (i,j) in the matrix.  By default
+       this will take ownership of the Coefficient passed in, but this
        can be overridden with the @a own parameter. */
    void Set(int i, int j, Coefficient * c, bool own=true);
 
@@ -1873,6 +1872,85 @@ public:
                const IntegrationPoint &ip) override;
 };
 
+/// Scalar coefficient defined as component of a vector coefficient
+class VectorComponentCoefficient : public Coefficient
+{
+private:
+   VectorCoefficient *a = nullptr;
+
+   mutable Vector va;
+   int component;
+
+public:
+   /// Construct with a vector coefficient.
+   VectorComponentCoefficient(VectorCoefficient &A)
+      : a(&A), va(A.GetVDim()), component(0) {};
+
+   VectorComponentCoefficient(VectorCoefficient &A, int c);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t) override;
+
+   /// Reset the vector coefficient
+   void SetACoef(VectorCoefficient &A) { a = &A; }
+
+   /// Return the vector coefficient
+   VectorCoefficient * GetACoef() const { return a; }
+
+   /// Set the component
+   void SetComponent(int c);
+
+   /// Return the component
+   int GetComponent() const { return component; }
+
+   /// Evaluate the trace coefficient at @a ip.
+   real_t Eval(ElementTransformation &T,
+               const IntegrationPoint &ip) override;
+};
+
+/// Scalar coefficient defined as component of a matrix coefficient
+class MatrixComponentCoefficient : public Coefficient
+{
+private:
+   MatrixCoefficient *a = nullptr;
+
+   mutable DenseMatrix ma;
+   int row_idx,col_idx;
+
+public:
+   MatrixComponentCoefficient(MatrixCoefficient &A)
+      : a(&A), ma(A.GetHeight(), A.GetWidth()), row_idx(0), col_idx(0) {};
+
+   /// Construct with the matrix coefficient.
+   MatrixComponentCoefficient(MatrixCoefficient &A, int ri, int ci);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t) override;
+
+   /// Reset the matrix coefficient
+   void SetACoef(MatrixCoefficient &A) { a = &A; }
+
+   /// Return the matrix coefficient
+   MatrixCoefficient * GetACoef() const { return a; }
+
+   /// Reset the index
+   void SetRowIndex(int ri);
+
+   /// Return the index
+   int GetRowIndex() const { return row_idx; }
+
+   /// Reset the index
+   void SetColumnIndex(int ci);
+
+   /// Return the index
+   int GetColumnIndex() const { return col_idx; }
+
+
+   /// Evaluate the trace coefficient at @a ip.
+   real_t Eval(ElementTransformation &T,
+               const IntegrationPoint &ip) override;
+};
+
 /// Vector coefficient defined as the linear combination of two vectors
 class VectorSumCoefficient : public VectorCoefficient
 {

fem/dfem/tuple.hpp

@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.
 #pragma once
 
-// This is serac's tuple implementation
+// This is smith's tuple implementation
 
 #include <ostream>
 #include "../../config/config.hpp"

fem/dgmassinv.cpp

@@ -12,7 +12,6 @@
 #include "dgmassinv.hpp"
 #include "bilinearform.hpp"
 #include "dgmassinv_kernels.hpp"
-#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -119,151 +118,6 @@ void DGMassInverse::Update()
 
 DGMassInverse::~DGMassInverse() = default;
 
-template<int DIM, int D1D, int Q1D>
-void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
-{
-   using namespace internal; // host/device kernel functions
-
-   const int NE = fes.GetNE();
-   const int d1d = m->dofs1D;
-   const int q1d = m->quad1D;
-
-   const int ND = static_cast<int>(pow(d1d, DIM));
-
-   const auto B = m->maps->B.Read();
-   const auto Bt = m->maps->Bt.Read();
-   const auto pa_data = m->pa_data.Read();
-   const auto dinv = diag_inv.Read();
-   auto r = r_.Write();
-   auto d = d_.Write();
-   auto z = z_.Write();
-   auto u = u_.ReadWrite();
-
-   const real_t RELTOL = rel_tol;
-   const real_t ABSTOL = abs_tol;
-   const int MAXIT = max_iter;
-   const bool IT_MODE = iterative_mode;
-   const bool CHANGE_BASIS = (d2q != nullptr);
-
-   // b is the right-hand side (if no change of basis, this just points to the
-   // incoming RHS vector, if we have to change basis, this points to the
-   // internal b2 vector where we put the transformed RHS)
-   const real_t *b;
-   // the following are non-null if we have to change basis
-   real_t *b2 = nullptr; // non-const access to b2
-   const real_t *b_orig = nullptr; // RHS vector in "original" basis
-   const real_t *d2q_B = nullptr; // matrix to transform initial guess
-   const real_t *q2d_B = nullptr; // matrix to transform solution
-   const real_t *q2d_Bt = nullptr; // matrix to transform RHS
-   if (CHANGE_BASIS)
-   {
-      d2q_B = d2q->B.Read();
-      q2d_B = B_.Read();
-      q2d_Bt = Bt_.Read();
-
-      b2 = b2_.Write();
-      b_orig = b_.Read();
-      b = b2;
-   }
-   else
-   {
-      b = b_.Read();
-   }
-
-   static constexpr int NB = Q1D ? Q1D : 1; // block size
-
-   mfem::forall_2D(NE, NB, NB, [=] MFEM_HOST_DEVICE (int e)
-   {
-      // Perform change of basis if needed
-      if (CHANGE_BASIS)
-      {
-         // Transform RHS
-         DGMassBasis<DIM,D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
-         if (IT_MODE)
-         {
-            // Transform initial guess
-            DGMassBasis<DIM,D1D>(e, NE, d2q_B, u, u, d1d);
-         }
-      }
-
-      const int tid = MFEM_THREAD_ID(x) + NB*MFEM_THREAD_ID(y);
-
-      // Compute first residual
-      if (IT_MODE)
-      {
-         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, u, r, d1d, q1d);
-         DGMassAxpy(e, NE, ND, 1.0, b, -1.0, r, r); // r = b - r
-      }
-      else
-      {
-         // if not in iterative mode, use zero initial guess
-         const int BX = MFEM_THREAD_SIZE(x);
-         const int BY = MFEM_THREAD_SIZE(y);
-         const int bxy = BX*BY;
-         const auto B = ConstDeviceMatrix(b, ND, NE);
-         auto U = DeviceMatrix(u, ND, NE);
-         auto R = DeviceMatrix(r, ND, NE);
-         for (int i = tid; i < ND; i += bxy)
-         {
-            U(i, e) = 0.0;
-            R(i, e) = B(i, e);
-         }
-         MFEM_SYNC_THREAD;
-      }
-
-      DGMassPreconditioner(e, NE, ND, dinv, r, z);
-      DGMassAxpy(e, NE, ND, 1.0, z, 0.0, z, d); // d = z
-
-      real_t nom = DGMassDot<NB>(e, NE, ND, d, r);
-      if (nom < 0.0) { return; /* Not positive definite */ }
-      real_t r0 = fmax(nom*RELTOL*RELTOL, ABSTOL*ABSTOL);
-      if (nom <= r0) { return; /* Converged */ }
-
-      DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d);
-      real_t den = DGMassDot<NB>(e, NE, ND, z, d);
-      if (den <= 0.0)
-      {
-         DGMassDot<NB>(e, NE, ND, d, d);
-         // d2 > 0 => not positive definite
-         if (den == 0.0) { return; }
-      }
-
-      // start iteration
-      int i = 1;
-      while (true)
-      {
-         const real_t alpha = nom/den;
-         DGMassAxpy(e, NE, ND, 1.0, u, alpha, d, u); // u = u + alpha*d
-         DGMassAxpy(e, NE, ND, 1.0, r, -alpha, z, r); // r = r - alpha*A*d
-
-         DGMassPreconditioner(e, NE, ND, dinv, r, z);
-
-         real_t betanom = DGMassDot<NB>(e, NE, ND, r, z);
-         if (betanom < 0.0) { return; /* Not positive definite */ }
-         if (betanom <= r0) { break; /* Converged */ }
-
-         if (++i > MAXIT) { break; }
-
-         const real_t beta = betanom/nom;
-         DGMassAxpy(e, NE, ND, 1.0, z, beta, d, d); // d = z + beta*d
-         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d); // z = A d
-         den = DGMassDot<NB>(e, NE, ND, d, z);
-         if (den <= 0.0)
-         {
-            DGMassDot<NB>(e, NE, ND, d, d);
-            // d2 > 0 => not positive definite
-            if (den == 0.0) { break; }
-         }
-         nom = betanom;
-      }
-
-      if (CHANGE_BASIS)
-      {
-         DGMassBasis<DIM,D1D>(e, NE, q2d_B, u, u, d1d);
-      }
-   });
-}
-
 void DGMassInverse::Mult(const Vector &Mu, Vector &u) const
 {
    // Dispatch to templated version based on dim, d1d, and q1d.
@@ -306,23 +160,4 @@ DGMassInvKernels::DGMassInvKernels()
    k::Specialization<3,6,7>::Add();
 }
 
-/// @cond Suppress_Doxygen_warnings
-
-template <int DIM, int D1D, int Q1D>
-DGMassInverse::CGKernelType DGMassInverse::CGKernels::Kernel()
-{
-   return &DGMassInverse::DGMassCGIteration<DIM,D1D,Q1D>;
-}
-
-DGMassInverse::CGKernelType DGMassInverse::CGKernels::Fallback(
-   int dim, int, int)
-{
-   if (dim == 1) { return &DGMassInverse::DGMassCGIteration<1>; }
-   else if (dim == 2) { return &DGMassInverse::DGMassCGIteration<2>; }
-   else if (dim == 3) { return &DGMassInverse::DGMassCGIteration<3>; }
-   else { MFEM_ABORT("Unsupported dimension."); }
-}
-
-/// @endcond
-
 } // namespace mfem

fem/dgmassinv_kernels.hpp

@@ -15,6 +15,7 @@
 #include "../linalg/kernels.hpp"
 #include "kernels.hpp"
 #include "integ/bilininteg_mass_kernels.hpp"
+#include "dgmassinv.hpp"
 
 namespace mfem
 {
@@ -333,6 +334,170 @@ void DGMassBasis(const int e,
 
 } // namespace internal
 
+template<int DIM, int D1D, int Q1D>
+void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
+{
+   using namespace internal; // host/device kernel functions
+
+   const int NE = fes.GetNE();
+   const int d1d = m->dofs1D;
+   const int q1d = m->quad1D;
+
+   const int ND = static_cast<int>(pow(d1d, DIM));
+
+   const auto B = m->maps->B.Read();
+   const auto Bt = m->maps->Bt.Read();
+   const auto pa_data = m->pa_data.Read();
+   const auto dinv = diag_inv.Read();
+   auto r = r_.Write();
+   auto d = d_.Write();
+   auto z = z_.Write();
+   auto u = u_.ReadWrite();
+
+   const real_t RELTOL = rel_tol;
+   const real_t ABSTOL = abs_tol;
+   const int MAXIT = max_iter;
+   const bool IT_MODE = iterative_mode;
+   const bool CHANGE_BASIS = (d2q != nullptr);
+
+   // b is the right-hand side (if no change of basis, this just points to the
+   // incoming RHS vector, if we have to change basis, this points to the
+   // internal b2 vector where we put the transformed RHS)
+   const real_t *b;
+   // the following are non-null if we have to change basis
+   real_t *b2 = nullptr; // non-const access to b2
+   const real_t *b_orig = nullptr; // RHS vector in "original" basis
+   const real_t *d2q_B = nullptr; // matrix to transform initial guess
+   const real_t *q2d_B = nullptr; // matrix to transform solution
+   const real_t *q2d_Bt = nullptr; // matrix to transform RHS
+   if (CHANGE_BASIS)
+   {
+      d2q_B = d2q->B.Read();
+      q2d_B = B_.Read();
+      q2d_Bt = Bt_.Read();
+
+      b2 = b2_.Write();
+      b_orig = b_.Read();
+      b = b2;
+   }
+   else
+   {
+      b = b_.Read();
+   }
+
+   static constexpr int NB = Q1D ? Q1D : 1; // block size
+
+   mfem::forall_2D(NE, NB, NB, [=] MFEM_HOST_DEVICE (int e)
+   {
+      // Perform change of basis if needed
+      if (CHANGE_BASIS)
+      {
+         // Transform RHS
+         DGMassBasis<DIM,D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
+         if (IT_MODE)
+         {
+            // Transform initial guess
+            DGMassBasis<DIM,D1D>(e, NE, d2q_B, u, u, d1d);
+         }
+      }
+
+      const int tid = MFEM_THREAD_ID(x) + NB*MFEM_THREAD_ID(y);
+
+      // Compute first residual
+      if (IT_MODE)
+      {
+         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, u, r, d1d, q1d);
+         DGMassAxpy(e, NE, ND, 1.0, b, -1.0, r, r); // r = b - r
+      }
+      else
+      {
+         // if not in iterative mode, use zero initial guess
+         const int BX = MFEM_THREAD_SIZE(x);
+         const int BY = MFEM_THREAD_SIZE(y);
+         const int bxy = BX*BY;
+         const auto B = ConstDeviceMatrix(b, ND, NE);
+         auto U = DeviceMatrix(u, ND, NE);
+         auto R = DeviceMatrix(r, ND, NE);
+         for (int i = tid; i < ND; i += bxy)
+         {
+            U(i, e) = 0.0;
+            R(i, e) = B(i, e);
+         }
+         MFEM_SYNC_THREAD;
+      }
+
+      DGMassPreconditioner(e, NE, ND, dinv, r, z);
+      DGMassAxpy(e, NE, ND, 1.0, z, 0.0, z, d); // d = z
+
+      real_t nom = DGMassDot<NB>(e, NE, ND, d, r);
+      if (nom < 0.0) { return; /* Not positive definite */ }
+      real_t r0 = fmax(nom*RELTOL*RELTOL, ABSTOL*ABSTOL);
+      if (nom <= r0) { return; /* Converged */ }
+
+      DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d);
+      real_t den = DGMassDot<NB>(e, NE, ND, z, d);
+      if (den <= 0.0)
+      {
+         DGMassDot<NB>(e, NE, ND, d, d);
+         // d2 > 0 => not positive definite
+         if (den == 0.0) { return; }
+      }
+
+      // start iteration
+      int i = 1;
+      while (true)
+      {
+         const real_t alpha = nom/den;
+         DGMassAxpy(e, NE, ND, 1.0, u, alpha, d, u); // u = u + alpha*d
+         DGMassAxpy(e, NE, ND, 1.0, r, -alpha, z, r); // r = r - alpha*A*d
+
+         DGMassPreconditioner(e, NE, ND, dinv, r, z);
+
+         real_t betanom = DGMassDot<NB>(e, NE, ND, r, z);
+         if (betanom < 0.0) { return; /* Not positive definite */ }
+         if (betanom <= r0) { break; /* Converged */ }
+
+         if (++i > MAXIT) { break; }
+
+         const real_t beta = betanom/nom;
+         DGMassAxpy(e, NE, ND, 1.0, z, beta, d, d); // d = z + beta*d
+         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d); // z = A d
+         den = DGMassDot<NB>(e, NE, ND, d, z);
+         if (den <= 0.0)
+         {
+            DGMassDot<NB>(e, NE, ND, d, d);
+            // d2 > 0 => not positive definite
+            if (den == 0.0) { break; }
+         }
+         nom = betanom;
+      }
+
+      if (CHANGE_BASIS)
+      {
+         DGMassBasis<DIM,D1D>(e, NE, q2d_B, u, u, d1d);
+      }
+   });
+}
+
+/// @cond Suppress_Doxygen_warnings
+
+template <int DIM, int D1D, int Q1D>
+inline DGMassInverse::CGKernelType DGMassInverse::CGKernels::Kernel()
+{
+   return &DGMassInverse::DGMassCGIteration<DIM,D1D,Q1D>;
+}
+
+inline DGMassInverse::CGKernelType DGMassInverse::CGKernels::Fallback(
+   int dim, int, int)
+{
+   if (dim == 1) { return &DGMassInverse::DGMassCGIteration<1>; }
+   else if (dim == 2) { return &DGMassInverse::DGMassCGIteration<2>; }
+   else if (dim == 3) { return &DGMassInverse::DGMassCGIteration<3>; }
+   else { MFEM_ABORT("Unsupported dimension."); }
+}
+
+/// @endcond
+
 } // namespace mfem
 
 #endif

fem/fe/fe_base.hpp

@@ -44,7 +44,7 @@ public:
       NumBasisTypes   = 9   /**< Keep track of maximum types to prevent
                                  hard-coding */
    };
-   /** @brief If the input does not represents a valid BasisType, abort with an
+   /** @brief If the input does not represent a valid BasisType, abort with an
        error; otherwise return the input. */
    static int Check(int b_type)
    {
@@ -52,7 +52,7 @@ public:
                   "unknown BasisType: " << b_type);
       return b_type;
    }
-   /** @brief If the input does not represents a valid nodal BasisType, abort
+   /** @brief If the input does not represent a valid nodal BasisType, abort
        with an error; otherwise return the input. */
    static int CheckNodal(int b_type)
    {
@@ -1120,7 +1120,7 @@ public:
       return GetPoints(p, btype, on_device);
    }
 
-   /// Get coordinates of a closed (GaussLegendre) set of points if degree @a p
+   /// Get coordinates of a closed (GaussLobatto) set of points if degree @a p
    const real_t *ClosedPoints(const int p,
                               const int btype = BasisType::GaussLobatto,
                               bool on_device = false)

fem/fe/fe_nurbs.cpp

@@ -81,7 +81,47 @@ void NURBS1DFiniteElement::CalcHessian (const IntegrationPoint &ip,
 
    sum = 1.0/sum;
    add(sum, hess, -2*dsum*sum*sum, grad, hess);
-   add((real_t)1.0, hess, (-d2sum + 2*dsum*dsum*sum)*sum*sum, shape_x, hess);
+   add(1.0, hess, (-d2sum + 2*dsum*dsum*sum)*sum*sum, shape_x, hess);
+}
+
+void NURBS1DFiniteElement::Project(Coefficient &coeff,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   IntegrationPoint ip;
+
+   for (int i = 0; i <= order; i++)
+   {
+      real_t kx = kv[0]->GetBotella(ijk[0] + i);
+      if (!kv[0]->inSpan(kx, ijk[0]+order)) { continue; }
+      ip.x = kv[0]->GetRefPoint(kx, ijk[0]+order);
+
+      Trans.SetIntPoint(&ip);
+      dofs(i) = coeff.Eval(Trans, ip);
+   }
+}
+
+void NURBS1DFiniteElement::Project(VectorCoefficient &vc,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
+   Vector x(vc.GetVDim());
+   IntegrationPoint ip;
+
+   for (int i = 0; i <= order; i++)
+   {
+      real_t kx = kv[0]->GetBotella(ijk[0] + i);
+      if (!kv[0]->inSpan(kx, ijk[0]+order)) { continue; }
+      ip.x = kv[0]->GetRefPoint(kx, ijk[0]+order);
+
+      Trans.SetIntPoint(&ip);
+      vc.Eval(x, Trans, ip);
+      for (int j = 0; j < x.Size(); j++)
+      {
+         dofs(dof*j+i) = x(j);
+      }
+   }
 }
 
 
@@ -215,6 +255,63 @@ void NURBS2DFiniteElement::CalcHessian (const IntegrationPoint &ip,
    }
 }
 
+void NURBS2DFiniteElement::Project(Coefficient &coeff,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   IntegrationPoint ip;
+   for (int o = 0, j = 0; j <= orders[1]; j++)
+   {
+      real_t ky = kv[1]->GetBotella(ijk[1] + j);
+      if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+      {
+         o += orders[0] + 1;
+         continue;
+      }
+      ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         real_t kx = kv[0]->GetBotella(ijk[0] + i);
+         if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+         ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+         Trans.SetIntPoint(&ip);
+         dofs(o) = coeff.Eval(Trans, ip);
+      }
+   }
+}
+
+void NURBS2DFiniteElement::Project(VectorCoefficient &vc,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
+   Vector x(vc.GetVDim());
+   IntegrationPoint ip;
+   for (int o = 0, j = 0; j <= orders[1]; j++)
+   {
+      real_t ky = kv[1]->GetBotella(ijk[1] + j);
+      if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+      {
+         o += orders[0] + 1;
+         continue;
+      }
+      ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         real_t kx = kv[0]->GetBotella(ijk[0] + i);
+         if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+         ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+         for (int v = 0; v < x.Size(); v++)
+         {
+            dofs(dof*v+o) = x(v);
+         }
+      }
+   }
+}
 
 void NURBS3DFiniteElement::SetOrder() const
 {
@@ -401,6 +498,85 @@ void NURBS3DFiniteElement::CalcHessian (const IntegrationPoint &ip,
    }
 }
 
+void NURBS3DFiniteElement::Project(Coefficient &coeff,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   IntegrationPoint ip;
+
+   for (int o = 0, k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 1)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            dofs(o) = coeff.Eval(Trans, ip);
+         }
+      }
+   }
+}
+
+void NURBS3DFiniteElement::Project(VectorCoefficient &vc,
+                                   ElementTransformation &Trans,
+                                   Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == vc.GetVDim()*dof, "");
+   Vector x(vc.GetVDim());
+   IntegrationPoint ip;
+
+   for (int o = 0, k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 1)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+            for (int v = 0; v < x.Size(); v++)
+            {
+               dofs(dof*v+o) = x(v);
+            }
+         }
+      }
+   }
+}
 
 void NURBS_HDiv2DFiniteElement::SetOrder() const
 {
@@ -517,6 +693,63 @@ void NURBS_HDiv2DFiniteElement::CalcDivShape(const IntegrationPoint &ip,
    }
 }
 
+void NURBS_HDiv2DFiniteElement::Project(VectorCoefficient &vc,
+                                        ElementTransformation &Trans,
+                                        Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == dof, "");
+   MFEM_ASSERT(vc.GetVDim() == 2, "");
+   Vector x(2), mx(2);
+   IntegrationPoint ip;
+   int o = 0;
+
+   for (int j = 0; j <= orders[1]; j++)
+   {
+      real_t ky = kv[1]->GetBotella(ijk[1] + j);
+      if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+      {
+         o += orders[0] + 2;
+         continue;
+      }
+      ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+      for (int i = 0; i <= orders[0]+1; i++, o++)
+      {
+         real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+         if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+         ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+
+         Trans.AdjugateJacobian().Mult(x,mx);
+         dofs(o) = mx(0);
+      }
+   }
+
+   for (int j = 0; j <= orders[1]+1; j++)
+   {
+      real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+      if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+      {
+         o += orders[0] + 1;
+         continue;
+      }
+      ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         real_t kx = kv[0]->GetBotella(ijk[0] + i);
+         if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+         ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+
+         Trans.AdjugateJacobian().Mult(x,mx);
+         dofs(o) = mx(1);
+      }
+   }
+}
+
 NURBS_HDiv2DFiniteElement::~NURBS_HDiv2DFiniteElement()
 {
    if (kv1[0]) { delete kv1[0]; }
@@ -696,6 +929,120 @@ void NURBS_HDiv3DFiniteElement::CalcDivShape(const IntegrationPoint &ip,
    }
 }
 
+
+void NURBS_HDiv3DFiniteElement::Project(VectorCoefficient &vc,
+                                        ElementTransformation &Trans,
+                                        Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == dof, "");
+   MFEM_ASSERT(vc.GetVDim() == 3, "");
+   Vector x(2), mx(3);
+   IntegrationPoint ip;
+
+   int o = 0;
+
+   for (int k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 2)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 2;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+            if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+            ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.AdjugateJacobian().Mult(x,mx);
+            dofs(o) = mx(0);
+         }
+      }
+   }
+
+   for (int k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 1)*(orders[1] + 2);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+         if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.AdjugateJacobian().Mult(x,mx);
+            dofs(o) = mx(1);
+         }
+      }
+   }
+
+   for (int k = 0; k <= orders[2]+1; k++)
+   {
+      real_t kz = kv1[2]->GetBotella(ijk[2] + k);
+      if (!kv1[2]->inSpan(kz, ijk[2]+orders[2]+1))
+      {
+         o += (orders[0] + 1)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv1[2]->GetRefPoint(kz, ijk[2]+orders[2]+1);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.AdjugateJacobian().Mult(x,mx);
+            dofs(o) = mx(2);
+         }
+      }
+   }
+
+}
+
+
 NURBS_HDiv3DFiniteElement::~NURBS_HDiv3DFiniteElement()
 {
    if (kv1[0]) { delete kv1[0]; }
@@ -817,13 +1164,68 @@ void NURBS_HCurl2DFiniteElement::CalcCurlShape(const IntegrationPoint &ip,
    }
 }
 
+void NURBS_HCurl2DFiniteElement::Project(VectorCoefficient &vc,
+                                         ElementTransformation &Trans,
+                                         Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == dof, "");
+   MFEM_ASSERT(vc.GetVDim() == 2, "");
+   Vector x(2), xm(2);
+   IntegrationPoint ip;
+   int i, j, o;
+   for (o = 0, j = 0; j <= orders[1]+1; j++)
+   {
+      real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+      if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+      {
+         o += orders[0] + 1;
+         continue;
+      }
+      ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+      for (i = 0; i <= orders[0]; i++, o++)
+      {
+         real_t kx = kv[0]->GetBotella(ijk[0] + i);
+         if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+         ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+
+         Trans.Jacobian().MultTranspose(x,xm);
+         dofs(o) = xm(0);
+      }
+   }
+
+   for (j = 0; j <= orders[1]; j++)
+   {
+      real_t ky = kv[1]->GetBotella(ijk[1] + j);
+      if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+      {
+         o += orders[0] + 2;
+         continue;
+      }
+      ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+      for (i = 0; i <= orders[0]+1; i++, o++)
+      {
+         real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+         if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+         ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+         Trans.SetIntPoint(&ip);
+         vc.Eval(x, Trans, ip);
+
+         Trans.Jacobian().MultTranspose(x,xm);
+         dofs(o) = xm(1);
+      }
+   }
+}
+
 NURBS_HCurl2DFiniteElement::~NURBS_HCurl2DFiniteElement()
 {
    if (kv1[0]) { delete kv1[0]; }
    if (kv1[1]) { delete kv1[1]; }
 }
 
-
 void NURBS_HCurl3DFiniteElement::SetOrder() const
 {
    orders[0] = kv[0]->GetOrder();
@@ -1003,11 +1405,124 @@ void NURBS_HCurl3DFiniteElement::CalcCurlShape(const IntegrationPoint &ip,
             curl_shape(o,0) = shape1_x(i)*dsy1_sz;
             curl_shape(o,1) = -dshape1_x(i)*sy1_sz;
             curl_shape(o,2) = 0.0;
+
          }
       }
    }
 }
 
+void NURBS_HCurl3DFiniteElement::Project(VectorCoefficient &vc,
+                                         ElementTransformation &Trans,
+                                         Vector &dofs) const
+{
+   MFEM_ASSERT(dofs.Size() == dof, "");
+   MFEM_ASSERT(vc.GetVDim() == 3, "");
+   Vector x(3), xm(3);
+   IntegrationPoint ip;
+
+   int o = 0;
+   for (int k = 0; k <= orders[2]+1; k++)
+   {
+      real_t kz = kv1[2]->GetBotella(ijk[2] + k);
+      if (!kv1[2]->inSpan(kz, ijk[2]+orders[2]+1))
+      {
+         o += (orders[0] + 1)*(orders[1] + 2);
+         continue;
+      }
+      ip.z = kv1[2]->GetRefPoint(kz, ijk[2]+orders[2]+1);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+         if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+         {
+            o += orders[0] + 1;
+            continue;
+         }
+         ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            real_t kx = kv[0]->GetBotella(ijk[0] + i);
+            if (!kv[0]->inSpan(kx, ijk[0]+orders[0])) { continue; }
+            ip.x = kv[0]->GetRefPoint(kx, ijk[0]+orders[0]);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.Jacobian().MultTranspose(x,xm);
+            dofs(o) = xm(0);
+         }
+      }
+   }
+
+   for (int k = 0; k <= orders[2]+1; k++)
+   {
+      real_t kz = kv1[2]->GetBotella(ijk[2] + k);
+      if (!kv1[2]->inSpan(kz, ijk[2]+orders[2]+1))
+      {
+         o += (orders[0] + 2)*(orders[1] + 1);
+         continue;
+      }
+      ip.z = kv1[2]->GetRefPoint(kz, ijk[2]+orders[2]+1);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         real_t ky = kv[1]->GetBotella(ijk[1] + j);
+         if (!kv[1]->inSpan(ky, ijk[1]+orders[1]))
+         {
+            o += orders[0] + 2;
+            continue;
+         }
+         ip.y = kv[1]->GetRefPoint(ky, ijk[1]+orders[1]);
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+            if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+            ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.Jacobian().MultTranspose(x,xm);
+            dofs(o) = xm(1);
+         }
+      }
+   }
+
+   for (int k = 0; k <= orders[2]; k++)
+   {
+      real_t kz = kv[2]->GetBotella(ijk[2] + k);
+      if (!kv[2]->inSpan(kz, ijk[2]+orders[2]))
+      {
+         o += (orders[0] + 2)*(orders[1] + 2);
+         continue;
+      }
+      ip.z = kv[2]->GetRefPoint(kz, ijk[2]+orders[2]);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         real_t ky = kv1[1]->GetBotella(ijk[1] + j);
+         if (!kv1[1]->inSpan(ky, ijk[1]+orders[1]+1))
+         {
+            o += orders[0] + 2;
+            continue;
+         }
+         ip.y = kv1[1]->GetRefPoint(ky, ijk[1]+orders[1]+1);
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            real_t kx = kv1[0]->GetBotella(ijk[0] + i);
+            if (!kv1[0]->inSpan(kx, ijk[0]+orders[0]+1)) { continue; }
+            ip.x = kv1[0]->GetRefPoint(kx, ijk[0]+orders[0]+1);
+
+            Trans.SetIntPoint(&ip);
+            vc.Eval(x, Trans, ip);
+
+            Trans.Jacobian().MultTranspose(x,xm);
+            dofs(o) = xm(2);
+         }
+      }
+   }
+
+}
+
+
 NURBS_HCurl3DFiniteElement::~NURBS_HCurl3DFiniteElement()
 {
    if (kv1[0]) { delete kv1[0]; }

fem/fe/fe_nurbs.hpp

@@ -86,6 +86,18 @@ public:
                    DenseMatrix &dshape) const override;
    void CalcHessian (const IntegrationPoint &ip,
                      DenseMatrix &hessian) const override;
+
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(Coefficient &coeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
 };
 
 /// An arbitrary order 2D NURBS element on a square
@@ -121,6 +133,18 @@ public:
                    DenseMatrix &dshape) const override;
    void CalcHessian (const IntegrationPoint &ip,
                      DenseMatrix &hessian) const override;
+
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(Coefficient &coeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
 };
 
 /// An arbitrary order 3D NURBS element on a cube
@@ -161,6 +185,18 @@ public:
                    DenseMatrix &dshape) const override;
    void CalcHessian (const IntegrationPoint &ip,
                      DenseMatrix &hessian) const override;
+
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(Coefficient &coeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
 };
 
 
@@ -242,6 +278,13 @@ public:
    void CalcDivShape(const IntegrationPoint &ip,
                      Vector &divshape) const override;
 
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
    ~NURBS_HDiv2DFiniteElement();
 };
 
@@ -336,6 +379,13 @@ public:
    void CalcDivShape(const IntegrationPoint &ip,
                      Vector &divshape) const override;
 
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
    ~NURBS_HDiv3DFiniteElement();
 };
 
@@ -415,6 +465,13 @@ public:
    void CalcCurlShape(const IntegrationPoint &ip,
                       DenseMatrix &curl_shape) const override;
 
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
    ~NURBS_HCurl2DFiniteElement();
 };
 
@@ -506,6 +563,13 @@ public:
    void CalcCurlShape(const IntegrationPoint &ip,
                       DenseMatrix &curl_shape) const override;
 
+   using FiniteElement::Project;
+
+   /** Evaluate the dofs that are defined on this element.
+       Dofs that can not be evaluated will remain unmodified. */
+   void Project(VectorCoefficient &vcoeff,
+                ElementTransformation &Trans, Vector &dofs) const override;
+
    ~NURBS_HCurl3DFiniteElement();
 };
 

fem/fe/fe_pyramid.cpp

@@ -1574,7 +1574,7 @@ void FuentesPyramid::V_R(int p, Vector s, const DenseMatrix &grad_s,
    {
       // dphi_E_i.GetRow(i, dphi);
       for (int l=0; l<3; l++) { dphi[l] = dphi_E_i(i, l); }
-      add(t * t, dphi, 2 * t * phi_E_i(i), dt3, dphit2);
+      add(t * t, dphi, 2.0 * t * phi_E_i(i), dt3, dphit2);
       dphit2.cross3D(dmu3, dphixdmu);
       // u.SetRow(i, dphixdmu);
       for (int l=0; l<3; l++) { u(i, l) = dphixdmu(l); }

fem/fe_coll.hpp

@@ -111,36 +111,36 @@ public:
    | :------: | :---: | :---: | :-------: | :-----: | :---: |
    | H1_[DIM]_[ORDER] | H1 | * | 1 | VALUE | H1 nodal elements |
    | H1@[BTYPE]_[DIM]_[ORDER] | H1 | * | * | VALUE | H1 nodal elements |
-   | H1Pos_[DIM]_[ORDER] | H1 | * | 1 | VALUE | H1 nodal elements |
+   | H1Pos_[DIM]_[ORDER] | H1 | * | 2 | VALUE | H1 nodal elements |
    | H1Pos_Trace_[DIM]_[ORDER] | H^{1/2} | * | 2 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
    | H1_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
-   | H1_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
+   | H1_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | * | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
    | ND_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | Nedelec vector elements |
    | ND@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | Nedelec vector elements |
-   | ND_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0  | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) |
-   | ND_Trace@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) |
+   | ND_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0  | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces,edges) |
+   | ND_Trace@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H^{1/2} | * | * / * | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces,edges) |
    | ND_R1D_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 1D. |
    | ND_R1D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 1D. |
    | ND_R2D_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 2D. |
    | ND_R2D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 2D. |
    | RT_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | Raviart-Thomas vector elements |
    | RT@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | Raviart-Thomas vector elements |
-   | RT_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
-   | RT_ValTrace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
-   | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
-   | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] |  H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_Trace_[DIM]_[ORDER] | H^{1/2} | * | 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_ValTrace_[DIM]_[ORDER] | H^{1/2} | * | 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | * | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] |  H^{1/2} | * | * | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
    | RT_R1D_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 1D. |
    | RT_R1D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 1D. |
    | RT_R2D_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 2D. |
    | RT_R2D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 2D. |
    | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
-   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
+   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | * | VALUE | Discontinuous L2 elements |
    | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
-   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
+   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | * | INTEGRAL | Discontinuous L2 elements |
    | DG_Iface_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
-   | DG_Iface@[BTYPE]_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
+   | DG_Iface@[BTYPE]_[DIM]_[ORDER] | - | * | * | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
    | DG_IntIface_[DIM]_[ORDER] | - | * | 0 | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
-   | DG_IntIface@[BTYPE]_[DIM]_[ORDER] | - | * | 0 | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
+   | DG_IntIface@[BTYPE]_[DIM]_[ORDER] | - | * | * | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
    | NURBS[ORDER] | - | * | - | VALUE | Non-Uniform Rational B-Splines (NURBS) elements |
    | LinearNonConf3D | - | 1 | 1 | VALUE | Piecewise-linear nonconforming finite elements in 3D |
    | CrouzeixRaviart | - | - | - | - | Crouzeix-Raviart nonconforming elements in 2D |
@@ -172,7 +172,7 @@ public:
    | :------: | :--------: |
    | [DIM]    | Dimension of the elements (1D, 2D, 3D) |
    | [ORDER]  | Approximation order of the elements (P0, P1, P2, ...) |
-   | [BTYPE]  | BasisType of the element (0-GaussLegendre, 1 - GaussLobatto, 2-Bernstein, 3-OpenUniform, 4-CloseUniform, 5-OpenHalfUniform) |
+   | [BTYPE]  | BasisType of the element (0-GaussLegendre, 1-GaussLobatto, 2-Bernstein, 3-OpenUniform, 4-CloseUniform, 5-OpenHalfUniform 6-Serendipity 7-ClosedGL 8-IntegratedGLL) |
    | [OBTYPE] | Open BasisType of the element for elements which have both types |
    | [CBTYPE] | Closed BasisType of the element for elements which have both types |
 

fem/gridfunc.cpp

@@ -42,8 +42,9 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
    // Grid functions are stored on the device
    UseDevice(true);
 
-   fes = new FiniteElementSpace;
-   fec_owned = fes->Load(m, input);
+   owned_fes = std::make_shared<FiniteElementSpace>();
+   fes = owned_fes.get();
+   fec.reset(fes->Load(m, input));
 
    skip_comment_lines(input, '#');
    istream::int_type next_char = input.peek();
@@ -85,10 +86,11 @@ GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
    int vdim, ordering;
 
    fes = gf_array[0]->FESpace();
-   fec_owned = FiniteElementCollection::New(fes->FEColl()->Name());
+   fec.reset(FiniteElementCollection::New(fes->FEColl()->Name()));
    vdim = fes->GetVDim();
    ordering = fes->GetOrdering();
-   fes = new FiniteElementSpace(m, fec_owned, vdim, ordering);
+   owned_fes =
+      std::make_shared<FiniteElementSpace>(m, fec.get(), vdim, ordering);
    SetSize(fes->GetVSize());
 
    if (m->NURBSext)
@@ -155,13 +157,61 @@ GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
    fes_sequence = fes->GetSequence();
 }
 
+GridFunction &GridFunction::operator=(const GridFunction &rhs)
+{
+   if (&rhs != this)
+   {
+      Vector::operator=(rhs);
+      if (fes != rhs.fes)
+      {
+         fes = rhs.fes;
+         owned_fes = rhs.owned_fes;
+         fec = rhs.fec;
+      }
+      else
+      {
+         // ensure we don't accidentally delete if rhs doesn't have shared
+         // ownership
+         if (!owned_fes)
+         {
+            owned_fes = rhs.owned_fes;
+         }
+         if (!fec)
+         {
+            fec = rhs.fec;
+         }
+      }
+      fes_sequence = rhs.fes_sequence;
+   }
+   return *this;
+}
+
 void GridFunction::Destroy()
 {
-   if (fec_owned)
+   owned_fes.reset();
+   fec.reset();
+}
+
+void GridFunction::MakeOwner()
+{
+   if (fec.get() != fes->FEColl())
    {
-      delete fes;
-      delete fec_owned;
-      fec_owned = NULL;
+      fec.reset(const_cast<FiniteElementCollection *>(fes->FEColl()));
+   }
+   if (owned_fes.get() != fes)
+   {
+      owned_fes.reset(fes);
+   }
+}
+
+void GridFunction::MakeOwner(FiniteElementCollection* fec_)
+{
+   if (fec_)
+   {
+      MFEM_VERIFY(fec_ == fes->FEColl(),
+                  "fec_ not associated with fes. If you intended to release "
+                  "ownership, see GridFunction::ShareOwner");
+      MakeOwner();
    }
 }
 
@@ -2352,52 +2402,83 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
    }
 }
 
-void GridFunction::ProjectCoefficient(Coefficient &coeff)
+void GridFunction::ProjectCoefficient(Coefficient &coeff, ProjectType type)
 {
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
    DofTransformation doftrans;
+   Array<int> vdofs;
+   Vector vals;
 
    if (delta_c == NULL)
    {
       if (fes->GetNURBSext() == NULL)
       {
-         Array<int> vdofs;
-         Vector vals;
-
-         for (int i = 0; i < fes->GetNE(); i++)
+         switch (type)
          {
-            fes->GetElementVDofs(i, vdofs, doftrans);
-            vals.SetSize(vdofs.Size());
-            fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
-            doftrans.TransformPrimal(vals);
-            SetSubVector(vdofs, vals);
+            case ProjectType::ELEMENT_L2:
+               ProjectCoefficientElementL2(coeff);
+               return;
+            case ProjectType::GLOBAL_L2:
+               ProjectCoefficientGlobalL2(coeff);
+               return;
+            default:
+               for (int i = 0; i < fes->GetNE(); i++)
+               {
+                  fes->GetElementVDofs(i, vdofs, doftrans);
+                  vals.SetSize(vdofs.Size());
+                  fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
+                  doftrans.TransformPrimal(vals);
+                  SetSubVector(vdofs, vals);
+               }
          }
       }
       else
       {
-         // Define and assemble linear form
-         LinearForm b(fes);
-         b.AddDomainIntegrator(new DomainLFIntegrator(coeff));
-         b.Assemble();
+         switch (type)
+         {
+            case ProjectType::DEFAULT:
+            case ProjectType::ELEMENT_L2:
+               ProjectCoefficientElementL2(coeff);
+               return;
+            case ProjectType::GLOBAL_L2:
+               ProjectCoefficientGlobalL2(coeff);
+               return;
+            case ProjectType::ELEMENT:
+               constexpr real_t signal = std::numeric_limits<real_t>::min();
 
-         // Define and assemble bilinear form
-         BilinearForm a(fes);
-         a.AddDomainIntegrator(new MassIntegrator());
-         a.Assemble();
+               for (int i = 0; i < fes->GetNE(); i++)
+               {
+                  fes->GetElementVDofs(i, vdofs, doftrans);
+                  vals.SetSize(vdofs.Size());
+                  vals = signal;
 
-         // Set solver and preconditioner
-         SparseMatrix A(a.SpMat());
-         GSSmoother  prec(A);
-         CGSolver cg;
-         cg.SetOperator(A);
-         cg.SetPreconditioner(prec);
-         cg.SetRelTol(1e-12);
-         cg.SetMaxIter(1000);
-         cg.SetPrintLevel(0);
+                  fes->GetFE(i)->Project(coeff,
+                                         *fes->GetElementTransformation(i),
+                                         vals);
+                  doftrans.TransformPrimal(vals);
 
-         // Solve and get solution
-         *this = 0.0;
-         cg.Mult(b,*this);
+                  // Remove undefined dofs
+                  // The knot location (either Botella, Demko or Greville point)
+                  // where the NURBS dof are evaluated might fall outside of the
+                  // domain of the element. In that case the value is not set, and
+                  // the value remains the signal value.
+                  int s = 0;
+                  for (int ii = 0; ii < vals.Size(); ii++)
+                  {
+                     if (vals[ii] != signal)
+                     {
+                        vdofs[s] = vdofs[ii];
+                        vals(s) = vals(ii);
+                        s++;
+                     }
+                  }
+                  vdofs.SetSize(s);
+                  vals.SetSize(s);
+
+                  // Add reduced dofs to global vector
+                  SetSubVector(vdofs, vals);
+               }
+         }
       }
    }
    else
@@ -2410,6 +2491,167 @@ void GridFunction::ProjectCoefficient(Coefficient &coeff)
    }
 }
 
+void GridFunction::ProjectCoefficientGlobalL2(Coefficient &coeff, real_t rtol,
+                                              int iter)
+{
+   // Define and assemble linear form
+   LinearForm b(fes);
+   b.AddDomainIntegrator(new DomainLFIntegrator(coeff));
+   b.Assemble();
+
+   // Define and assemble bilinear form
+   BilinearForm a(fes);
+   a.AddDomainIntegrator(new MassIntegrator());
+   a.Assemble();
+
+   // Set solver and preconditioner
+   SparseMatrix A(a.SpMat());
+   GSSmoother  prec(A);
+   CGSolver cg;
+   cg.SetOperator(A);
+   cg.SetPreconditioner(prec);
+   cg.SetRelTol(rtol);
+   cg.SetMaxIter(iter);
+   cg.SetPrintLevel(0);
+
+   // Solve and get solution
+   *this = 0.0;
+   cg.Mult(b,*this);
+}
+
+void GridFunction::ProjectCoefficientElementL2(Coefficient &coeff)
+{
+   Vector Va;
+   ProjectCoefficientElementL2_(coeff, *this, Va);
+   (*this) /= Va;
+}
+
+void GridFunction::ProjectCoefficientElementL2_(Coefficient &coeff,
+                                                Vector &x, Vector &Va)
+{
+   DofTransformation doftrans;
+   Array<int> vdofs;
+   Vector shape,shape2, elvect, elwght;
+   DenseMatrix elmat;
+   Va.SetSize(fes->GetNDofs() );
+   x.SetSize(fes->GetNDofs() );
+   Va = 0.0;
+   x = 0.0;
+
+   if (fes->GetNURBSext() == NULL)
+   {
+      for (int e = 0; e < fes->GetNE(); e++)
+      {
+         fes->GetElementDofs (e, vdofs, doftrans);
+         ElementTransformation &tr = *fes -> GetElementTransformation (e);
+         const FiniteElement &el = *fes->GetFE(e);
+         int dof = el.GetDof();
+         shape.SetSize(dof);
+         elvect.SetSize(dof);
+         elwght.SetSize(dof);
+         elmat.SetSize(dof,dof);
+         elvect = 0.0;
+         elwght = 0.0;
+         elmat = 0.0;
+
+         const IntegrationRule &ir = IntRules.Get(el.GetGeomType(),
+                                                  2 * el.GetOrder() + 1);
+
+         // Element vector & weight
+         for (int i = 0; i < ir.GetNPoints(); i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+
+            tr.SetIntPoint (&ip);
+            real_t wght = ip.weight*tr.Weight();
+            real_t val = coeff.Eval(tr, ip);
+
+            el.CalcPhysShape(tr, shape);
+
+            elvect.Add(wght * val, shape);
+            elwght.Add(wght, shape);
+            AddMult_a_VVt(wght, shape, elmat);
+         }
+
+         // Solve
+         if (!LinearSolve(elmat, elvect.GetData(),1e-12))
+         {
+            MFEM_WARNING("Error in inverting element local matrix");
+         }
+
+         // Scale
+         elvect *= elwght;
+
+         // Add reduced dofs to global vector
+         x.AddElementVector(vdofs, elvect);
+         Va.AddElementVector(vdofs, elwght);
+      }
+   }
+   else
+   {
+      for (int e = 0; e < fes->GetNE(); e++)
+      {
+         fes->GetElementDofs (e, vdofs, doftrans);
+         ElementTransformation &tr = *fes -> GetElementTransformation (e);
+         const FiniteElement &el = *fes->GetFE(e);
+         int dof = el.GetDof();
+         int dim = el.GetDim();
+         int p = el.GetOrder();
+         L2_FECollection fe_coll(p,  dim);
+         //H1_FECollection fe_coll(p,  dim, BasisType::Positive);
+         const FiniteElement &el2 = *fe_coll.FiniteElementForGeometry(el.GetGeomType());
+         MFEM_ASSERT(el2.GetDof() == dof, "Element dofs do not match.");
+
+         shape.SetSize(dof);
+         shape2.SetSize(dof);
+         elvect.SetSize(dof);
+         elwght.SetSize(dof);
+         elmat.SetSize(dof,dof);
+         elvect = 0.0;
+         elwght = 0.0;
+         elmat = 0.0;
+
+         const IntegrationRule &ir = IntRules.Get(el.GetGeomType(),
+                                                  2 * el.GetOrder() + 1);
+
+         // Element vector & weight
+         for (int i = 0; i < ir.GetNPoints(); i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+
+            tr.SetIntPoint (&ip);
+            real_t wght = ip.weight*tr.Weight();
+            real_t val = coeff.Eval(tr, ip);
+            el.CalcPhysShape(tr, shape);
+            el2.CalcPhysShape(tr, shape2);
+
+            elvect.Add(wght * val, shape2);
+            elwght.Add(wght, shape);
+            AddMult_a_VVt(wght, shape2, elmat);
+         }
+         // Solve
+         if (!LinearSolve(elmat, elvect.GetData(),1e-12))
+         {
+            MFEM_WARNING("Error in inverting element local matrix 2");
+         }
+         // Map to NURBS
+         DenseMatrix I;
+         el2.Project(el,tr,I);
+         if (!LinearSolve(I, elvect.GetData(),1e-32))
+         {
+            MFEM_WARNING("Error in inverting element local matrix 3");
+         }
+
+         // Scale
+         elvect *= elwght;
+
+         // Add reduced dofs to global vector
+         x.AddElementVector(vdofs, elvect);
+         Va.AddElementVector(vdofs, elwght);
+      }
+   }
+}
+
 void GridFunction::ProjectCoefficient(
    Coefficient &coeff, Array<int> &dofs, int vd)
 {
@@ -2434,49 +2676,318 @@ void GridFunction::ProjectCoefficient(
    }
 }
 
-void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
+void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff,
+                                      ProjectType type)
 {
+   Array<int> vdofs;
+   Vector vals;
    DofTransformation doftrans;
+
    if (fes->GetNURBSext() == NULL)
    {
-      int i;
-      Array<int> vdofs;
-      Vector vals;
-
-      for (i = 0; i < fes->GetNE(); i++)
+      switch (type)
       {
-         fes->GetElementVDofs(i, vdofs, doftrans);
-         vals.SetSize(vdofs.Size());
-         fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
-         doftrans.TransformPrimal(vals);
-         SetSubVector(vdofs, vals);
+         case ProjectType::ELEMENT_L2:
+            ProjectCoefficientElementL2(vcoeff);
+            return;
+         case ProjectType::GLOBAL_L2:
+            ProjectCoefficientGlobalL2(vcoeff);
+            return;
+         default:
+            for (int i = 0; i < fes->GetNE(); i++)
+            {
+               fes->GetElementVDofs(i, vdofs, doftrans);
+               vals.SetSize(vdofs.Size());
+               fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
+               doftrans.TransformPrimal(vals);
+               SetSubVector(vdofs, vals);
+            }
       }
    }
    else
    {
-      // Define and assemble linear form
-      LinearForm b(fes);
+      switch (type)
+      {
+         case ProjectType::DEFAULT:
+         case ProjectType::ELEMENT_L2:
+            ProjectCoefficientElementL2(vcoeff);
+            return;
+         case ProjectType::GLOBAL_L2:
+            ProjectCoefficientGlobalL2(vcoeff);
+            return;
+         case ProjectType::ELEMENT:
+            constexpr real_t signal = std::numeric_limits<real_t>::min();
+            for (int i = 0; i < fes->GetNE(); i++)
+            {
+               fes->GetElementVDofs(i, vdofs, doftrans);
+               vals.SetSize(vdofs.Size());
+               vals = signal;
+               fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
+               doftrans.TransformPrimal(vals);
+               // Remove undefined dofs
+               // The knot location (either Botella, Demko or Greville point)
+               // where the NURBS dof are evaluated might fall outside of the
+               // domain of the element. In that case the value is not set, and
+               // the value remains the signal value.
+               int s = 0;
+               for (int ii = 0; ii < vals.Size(); ii++)
+               {
+                  if (vals[ii] != signal)
+                  {
+                     vdofs[s] = vdofs[ii];
+                     vals(s) = vals(ii);
+                     s++;
+                  }
+               }
+               vdofs.SetSize(s);
+               vals.SetSize(s);
+
+               // Add reduced dofs to global vector
+               SetSubVector(vdofs, vals);
+            }
+      }
+   }
+}
+
+void GridFunction::ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
+                                              real_t rtol, int iter)
+{
+   // Define and assemble linear form
+   LinearForm b(fes);
+   BilinearForm a(fes);
+
+   if (fes->GetTypicalFE()->GetRangeType() == mfem::FiniteElement::VECTOR)
+   {
       b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(vcoeff));
-      b.Assemble();
-
-      // Define and assemble bilinear form
-      BilinearForm a(fes);
       a.AddDomainIntegrator(new VectorFEMassIntegrator());
-      a.Assemble();
+   }
+   else
+   {
+      b.AddDomainIntegrator(new VectorDomainLFIntegrator(vcoeff));
+      a.AddDomainIntegrator(new VectorMassIntegrator());
+   }
+   a.Assemble();
+   b.Assemble();
 
-      // Set solver and preconditioner
-      SparseMatrix A(a.SpMat());
-      GSSmoother  prec(A);
-      CGSolver cg;
-      cg.SetOperator(A);
-      cg.SetPreconditioner(prec);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(1000);
-      cg.SetPrintLevel(0);
+   // Set solver and preconditioner
+   SparseMatrix A(a.SpMat());
+   GSSmoother  prec(A);
+   CGSolver cg;
+   cg.SetOperator(A);
+   cg.SetPreconditioner(prec);
+   cg.SetRelTol(rtol);
+   cg.SetMaxIter(iter);
+   cg.SetPrintLevel(0);
 
-      // Solve and get solution
-      *this = 0.0;
-      cg.Mult(b,*this);
+   // Solve and get solution
+   *this = 0.0;
+   cg.Mult(b,*this);
+}
+
+void GridFunction::ProjectCoefficientElementL2_(VectorCoefficient &vcoeff,
+                                                Vector &x, Vector &Va)
+{
+   DofTransformation doftrans;
+   Array<int> vdofs;
+   Vector shapel2, elvect, elwght, val;
+   DenseMatrix  shape, elmat;
+   Va.SetSize(Size());
+   x.SetSize(Size());
+   Va = 0.0;
+   x = 0.0;
+
+   if (fes->GetNURBSext() == NULL)
+   {
+      for (int e = 0; e < fes->GetNE(); e++)
+      {
+         fes->GetElementVDofs (e, vdofs, doftrans);
+         ElementTransformation &tr = *fes -> GetElementTransformation (e);
+         const FiniteElement &el = *fes->GetFE(e);
+         int dof = el.GetDof();
+         int dim = el.GetRangeDim();
+         shape.SetSize(dof,dim);
+         shapel2.SetSize(dof);
+         elvect.SetSize(dof);
+         elwght.SetSize(dof);
+         elmat.SetSize(dof,dof);
+         elvect = 0.0;
+         elwght = 0.0;
+         elmat = 0.0;
+
+         const IntegrationRule &ir = IntRules.Get(el.GetGeomType(),
+                                                  2 * el.GetOrder() + 1);
+
+         // Element vector & weight
+         for (int i = 0; i < ir.GetNPoints(); i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+
+            tr.SetIntPoint (&ip);
+            real_t wght = ip.weight*tr.Weight();
+            vcoeff.Eval(val, tr, ip);
+            val *= wght;
+
+            el.CalcPhysVShape(tr, shape);
+
+            shape.AddMult (val, elvect);
+            AddMult_a_AAt(wght, shape, elmat);
+
+            shape.GetRowl2(shapel2);
+            elwght.Add(wght, shapel2);
+         }
+
+         // Solve
+         if (!LinearSolve(elmat, elvect.GetData(),1e-12))
+         {
+            MFEM_WARNING("Error in inverting element local matrix");
+         }
+
+         // Scale
+         elvect *= elwght;
+
+         // Add to global vector
+         x.AddElementVector(vdofs, elvect);
+
+         // Add to weight vector -- no need for an orientation
+         for (int i = 0; i < vdofs.Size(); i++)
+         {
+            vdofs[i] = FiniteElementSpace::DecodeDof(vdofs[i]);
+         }
+         Va.AddElementVector(vdofs, elwght);
+      }
+   }
+   else
+   {
+      DenseMatrix partelmat;
+      Vector shape2;
+
+      if (fes->GetTypicalFE()->GetOrder() >= 6 )
+      {
+         MFEM_WARNING("This project is not stable for"
+                      "NURBS VectorFE with order >= 5");
+      }
+      for (int e = 0; e < fes->GetNE(); e++)
+      {
+         fes->GetElementVDofs (e, vdofs, doftrans);
+         ElementTransformation &tr = *fes -> GetElementTransformation (e);
+         const FiniteElement &el = *fes->GetFE(e);
+         int dof = el.GetDof();
+         int dim = el.GetRangeDim();
+         int p = el.GetOrder();
+         L2_FECollection fe_coll(p,  dim);
+         const FiniteElement &el2 = *fe_coll.FiniteElementForGeometry(el.GetGeomType());
+         int dof2 = el2.GetDof();
+         MFEM_ASSERT(dof2*dim >= dof, "Element dofs do not match.");
+         shape2.SetSize(dof2);
+         shape.SetSize(dof,dim);
+         shapel2.SetSize(dof);
+         elvect.SetSize(dof2*dim);
+         elwght.SetSize(dof);
+         elmat.SetSize(dof2*dim,dof2*dim);
+         partelmat.SetSize(dof2,dof2);
+         elvect = 0.0;
+         elwght = 0.0;
+         elmat = 0.0;
+
+         const IntegrationRule &ir = IntRules.Get(el.GetGeomType(),
+                                                  2 * el.GetOrder() + 1);
+
+         // Element vector & weight
+         for (int i = 0; i < ir.GetNPoints(); i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+
+            tr.SetIntPoint (&ip);
+            real_t wght = ip.weight*tr.Weight();
+            vcoeff.Eval(val, tr, ip);
+            val *= wght;
+
+            el2.CalcPhysShape(tr, shape2);
+            el.CalcPhysVShape(tr, shape);
+
+            for (int k = 0; k < dim; k++)
+            {
+               for (int s = 0; s < dof2; s++)
+               {
+                  elvect(dof2*k+s) += val(k) * shape2(s);
+               }
+            }
+
+            MultVVt(shape2, partelmat);
+            partelmat *= wght;
+            for (int k = 0; k < dim; k++)
+            {
+               elmat.AddMatrix(partelmat, dof2*k, dof2*k);
+            }
+
+            shape.GetRowl2(shapel2);
+            elwght.Add(wght, shapel2);
+         }
+
+         // Solve
+         if (!LinearSolve(elmat, elvect.GetData()))
+         {
+            MFEM_WARNING("Error in inverting element local matrix");
+         }
+
+         // Map to NURBS
+         DenseMatrix I;
+         el2.Project(el,tr,I);
+
+         // LSQ solve
+         // For higher order NURBS solving this non-square matrix causes issues.
+         // For Order <=4 the routine seems to work fine.
+         Vector vec(dof);
+         DenseMatrix mat(dof, dof);
+         I.Transpose();
+         I.Mult(elvect, vec);
+         MultAAt(I, mat);
+         if (!LinearSolve(mat, vec.GetData(), 1e-24))
+         {
+            mat.TestInversion();
+            MFEM_WARNING("Error in inverting element local matrix");
+         }
+         elvect = vec;
+
+         // Scale
+         elvect *= elwght;
+
+         // Add to global vector
+         x.AddElementVector(vdofs, elvect);
+
+         // Add to weight vector -- no need for an orientation
+         for (int i = 0; i < vdofs.Size(); i++)
+         {
+            vdofs[i] = FiniteElementSpace::DecodeDof(vdofs[i]);
+         }
+         Va.AddElementVector(vdofs, elwght);
+      }
+   }
+}
+
+void GridFunction::ProjectCoefficientElementL2(VectorCoefficient &vcoeff)
+{
+   if (fes->GetTypicalFE()->GetRangeType() == mfem::FiniteElement::VECTOR)
+   {
+      Vector Va;
+      ProjectCoefficientElementL2_(vcoeff, *this, Va);
+      (*this) /= Va;
+   }
+   else
+   {
+      Array<int> vdofs(fes->GetNDofs());
+      Vector x, Va;
+      VectorComponentCoefficient coeff(vcoeff,
+                                       0);  // 0  to ensure we have a valid object
+
+      for (int v = 0; v < VectorDim(); v++)
+      {
+         coeff.SetComponent(v);
+         ProjectCoefficientElementL2_(coeff, x, Va);
+         x /= Va;
+         fes->GetVDofs(v, vdofs);
+         SetSubVector(vdofs, x);
+      }
    }
 }
 
@@ -4025,7 +4536,7 @@ std::unique_ptr<GridFunction> GridFunction::ProlongateToMaxOrder() const
    PRefinementTransferOperator P(*fes, *fesMax);
    P.Mult(*this, *xMax);
 
-   xMax->MakeOwner(fecMax);
+   xMax->MakeOwner();
    return std::unique_ptr<GridFunction>(xMax);
 }
 
@@ -4598,7 +5109,7 @@ GridFunction *Extrude1DGridFunction(Mesh *mesh, Mesh *mesh2d,
    // assuming sol is scalar
    solfes2d = new FiniteElementSpace(mesh2d, solfec2d);
    sol2d = new GridFunction(solfes2d);
-   sol2d->MakeOwner(solfec2d);
+   sol2d->MakeOwner();
    {
       GridFunctionCoefficient csol(sol);
       ExtrudeCoefficient c2d(mesh, csol, ny);

fem/gridfunc.hpp

@@ -27,20 +27,37 @@
 namespace mfem
 {
 
+/** This enumerated type describes the three main projection types:
+    - ELEMENT, assigns the degree of freedom per element, as specified in the
+      specific element
+    - GLOBAL_L2, solves a global L2 projection
+    - ELEMENT_L2, solves a element level L2 projection. Inter element
+      connectivity is dealt with similar as in:
+      Bezier-Projection : A unified approach for local projection and
+      quadrature-free refinement and coarsening of NURBS and T-splines with
+      particular application to isogeometric design and analysis
+      [CMAME (284) 2015 pg 55-105]
+    - DEFAULT, for NURBS spaces this is ELEMENT_L2, while for all other spaces
+      this ELEMENT.
+   Note 1: ELEMENT_L2 also works for non NURBS elements
+   Note 2: For NURBS elements the ELEMENT projection gives results without
+   over and undershoots. However, the gradient near the boundary does not
+   converge.*/
+enum class ProjectType { DEFAULT, ELEMENT, GLOBAL_L2, ELEMENT_L2 };
+
 /// Class for grid function - Vector with associated FE space.
 class GridFunction : public Vector
 {
 protected:
-   /// FE space on which the grid function lives. Owned if #fec_owned is not NULL.
-   FiniteElementSpace *fes;
+   /// FE space on which the grid function lives.
+   FiniteElementSpace *fes = nullptr;
 
    /** @brief Used when the grid function is read from a file. It can also be
-       set explicitly, see MakeOwner().
+       set explicitly, see MakeOwner(). */
+   std::shared_ptr<FiniteElementCollection> fec;
+   std::shared_ptr<FiniteElementSpace> owned_fes;
 
-       If not NULL, this pointer is owned by the GridFunction. */
-   FiniteElementCollection *fec_owned;
-
-   long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence
+   long fes_sequence = 0; // see FiniteElementSpace::sequence, Mesh::sequence
 
    /** Optional, internal true-dof vector: if the FiniteElementSpace #fes has a
        non-trivial (i.e. not NULL) prolongation operator, this Vector may hold
@@ -66,6 +83,11 @@ protected:
        degree of freedom. */
    void ProjectDiscCoefficient(VectorCoefficient &coeff, Array<int> &dof_attr);
 
+   /** Helper function for ProjectCoefficientElementL2 */
+   void ProjectCoefficientElementL2_(Coefficient &coeff, Vector &sol, Vector &Va);
+   void ProjectCoefficientElementL2_(VectorCoefficient &vcoeff, Vector &sol,
+                                     Vector &Va);
+
    /// Loading helper.
    void LegacyNCReorder();
 
@@ -73,16 +95,25 @@ protected:
 
 public:
 
-   GridFunction() { fes = NULL; fec_owned = NULL; fes_sequence = 0; UseDevice(true); }
+   GridFunction() { UseDevice(true); }
 
    /// Copy constructor. The internal true-dof vector #t_vec is not copied.
    GridFunction(const GridFunction &orig)
-      : Vector(orig), fes(orig.fes), fec_owned(NULL), fes_sequence(orig.fes_sequence)
+      : Vector(orig), fes(orig.fes), fec(orig.fec), owned_fes(orig.owned_fes),
+        fes_sequence(orig.fes_sequence)
    { UseDevice(true); }
 
+   GridFunction(GridFunction &&orig) = default;
+
    /// Construct a GridFunction associated with the FiniteElementSpace @a *f.
-   GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+   GridFunction(FiniteElementSpace *f)
+      : Vector(f->GetVSize()), fes(f), fes_sequence(f->GetSequence())
+   { UseDevice(true); }
+
+   /// Same as above but specify the memory type
+   GridFunction(FiniteElementSpace *f, MemoryType mt)
+      : Vector(f->GetVSize(), mt), fes(f), fes_sequence(f->GetSequence())
+   { UseDevice(true); }
 
    /// Construct a GridFunction using previously allocated array @a data.
    /** The GridFunction does not assume ownership of @a data which is assumed to
@@ -91,14 +122,15 @@ public:
        array can be replaced later using the method SetData().
     */
    GridFunction(FiniteElementSpace *f, real_t *data)
-      : Vector(data, f->GetVSize())
-   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+      : Vector(data, f->GetVSize()), fes(f), fes_sequence(f->GetSequence())
+   { UseDevice(true); }
 
    /** @brief Construct a GridFunction using previously allocated Vector @a base
        starting at the given offset, @a base_offset. */
    GridFunction(FiniteElementSpace *f, Vector &base, int base_offset = 0)
-      : Vector(base, base_offset, f->GetVSize())
-   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+      : Vector(base, base_offset, f->GetVSize()), fes(f),
+        fes_sequence(f->GetSequence())
+   { UseDevice(true); }
 
    /// Construct a GridFunction on the given Mesh, using the data from @a input.
    /** The content of @a input should be in the format created by the method
@@ -108,21 +140,25 @@ public:
 
    GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces);
 
-   /// Copy assignment. Only the data of the base class Vector is copied.
-   /** It is assumed that this object and @a rhs use FiniteElementSpace%s that
-       have the same size.
+   /// Copy assignment. Temporary data is not copied.
+   GridFunction &operator=(const GridFunction &rhs);
 
-       @note Defining this method overwrites the implicitly defined copy
-       assignment operator. */
-   GridFunction &operator=(const GridFunction &rhs)
-   { return operator=((const Vector &)rhs); }
+   GridFunction &operator=(GridFunction &&gf) = default;
 
-   /// Make the GridFunction the owner of #fec_owned and #fes.
-   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership of #fec_owned
-       and #fes is taken away. */
-   void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; }
+   /// Make the GridFunction a shared owner of #fec and #fes.
+   void MakeOwner();
+   [[deprecated("Use MakeOwner() instead")]]
+   void MakeOwner(FiniteElementCollection* fec_);
+   /// Gets a shared ownership of #owned_fes and #fec if this GridFunction has
+   /// shared ownership.
+   void ShareOwner(std::shared_ptr<FiniteElementSpace> &fes_,
+                   std::shared_ptr<FiniteElementCollection> &fec_)
+   {
+      fes_ = owned_fes;
+      fec_ = fec;
+   }
 
-   FiniteElementCollection *OwnFEC() { return fec_owned; }
+   FiniteElementCollection* OwnFEC() { return fec.get(); }
 
    /// Shortcut for calling FiniteElementSpace::GetVectorDim() on the underlying #fes
    int VectorDim() const;
@@ -420,9 +456,30 @@ public:
    /** @brief Project @a coeff Coefficient to @a this GridFunction. The
        projection computation depends on the choice of the FiniteElementSpace
        #fes. Note that this is usually interpolation at the degrees of freedom
-       in each element (not L2 projection). For NURBS spaces these degrees of
-       freedom are not available and L2 projection is resorted to as fallback. */
-   virtual void ProjectCoefficient(Coefficient &coeff);
+       in each element (not L2 projection). For elements without a projection
+       member function one could use ProjectCoefficientGlobalL2 instead.
+       NOTE: For parallel simulations with NURBS elements some dofs might
+       not be defined, if the evaluation point does not reside on this rank.
+       If that is the case it is defined on another rank, and the issue is
+       rectified with the appropriate communication, see in ParGridFunction.
+       */
+   virtual void ProjectCoefficient(Coefficient &coeff,
+                                   ProjectType type = ProjectType::DEFAULT);
+
+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection is a global L2 projection. This routine can be used a
+       fallback for elements without a projection member function.*/
+   virtual void ProjectCoefficientGlobalL2(Coefficient &coeff,
+                                           real_t rtol = 1e-12,
+                                           int iter = 1000);
+
+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection is an element local L2 projection, with an appropriate
+       weighting for Dofs that are shared between elements. Inspired on
+       Bezier-Projection [CMAME (284) 2015 pg 55-105]
+       This routine can be used a fallback for elements without a projection
+       member function.*/
+   virtual void ProjectCoefficientElementL2(Coefficient &coeff);
 
    /** @brief Project @a coeff Coefficient to @a this GridFunction, using one
        element for each degree of freedom in @a dofs and nodal interpolation on
@@ -432,9 +489,26 @@ public:
    /** @brief Project @a vcoeff VectorCoefficient to @a this GridFunction. The
        projection computation depends on the choice of the FiniteElementSpace
        #fes. Note that this is usually interpolation at the degrees of freedom
-       in each element (not L2 projection). For NURBS spaces these degrees of
-       freedom are not available and L2 projection is resorted to as fallback. */
-   void ProjectCoefficient(VectorCoefficient &vcoeff);
+       in each element (not L2 projection). For elements without a projection
+       member function one could use ProjectCoefficientGlobalL2 instead.
+       NOTE: For parallel simulations with NURBS elements some dofs might
+       not be defined, if the evaluation point does not reside on this rank.
+       If that is the case it is defined on another rank, and the issue is
+       rectified with the appropriate communication, see in ParGridFunction.*/
+   virtual void ProjectCoefficient(VectorCoefficient &vcoeff,
+                                   ProjectType type = ProjectType::DEFAULT);
+
+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection is a global L2 projection. This routine can be used a
+       fallback for elements without a projection member function.*/
+   virtual void ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
+                                           real_t rtol = 1e-12,
+                                           int iter = 1000);
+
+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection is a global L2 projection. This routine can be used a
+       fallback for elements without a projection member function.*/
+   virtual void ProjectCoefficientElementL2(VectorCoefficient &vcoeff);
 
    /** @brief Project @a vcoeff VectorCoefficient to @a this GridFunction, using
        one element for each degree of freedom in @a dofs and nodal interpolation

fem/gslib.cpp

@@ -234,7 +234,7 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
 }
 
 void FindPointsGSLIB::FindPoints(const Vector &point_pos,
-                                 int point_pos_ordering)
+                                 const int point_pos_ordering)
 {
    MFEM_VERIFY(setupflag, "Use FindPointsGSLIB::Setup before finding points.");
    bool dev_mode = (point_pos.UseDevice() && Device::IsEnabled());
@@ -482,7 +482,7 @@ void FindPointsGSLIB::SetupDevice()
 }
 
 void FindPointsGSLIB::FindPointsOnDevice(const Vector &point_pos,
-                                         int point_pos_ordering)
+                                         const int point_pos_ordering)
 {
    if (!DEV.setup_device)
    {
@@ -505,13 +505,13 @@ void FindPointsGSLIB::FindPointsOnDevice(const Vector &point_pos,
 
    if (dim == 2)
    {
-      FindPointsLocal2(point_pos, point_pos_ordering, gsl_code, gsl_elem, gsl_ref,
-                       gsl_dist, points_cnt);
+      FindPointsLocal2(point_pos, point_pos_ordering, gsl_code, gsl_elem,
+                       gsl_ref, gsl_dist, points_cnt);
    }
    else
    {
-      FindPointsLocal3(point_pos, point_pos_ordering, gsl_code, gsl_elem, gsl_ref,
-                       gsl_dist, points_cnt);
+      FindPointsLocal3(point_pos, point_pos_ordering, gsl_code, gsl_elem,
+                       gsl_ref, gsl_dist, points_cnt);
    }
 
    // Sync from device to host
@@ -1085,7 +1085,7 @@ void FindPointsGSLIB::InterpolateOnDevice(const Vector &field_in_evec,
 #else
 void FindPointsGSLIB::SetupDevice() {};
 void FindPointsGSLIB::FindPointsOnDevice(const Vector &point_pos,
-                                         int point_pos_ordering) {};
+                                         const int point_pos_ordering) {};
 void FindPointsGSLIB::InterpolateOnDevice(const Vector &field_in_evec,
                                           Vector &field_out,
                                           const int nel, const int ncomp,
@@ -1094,7 +1094,8 @@ void FindPointsGSLIB::InterpolateOnDevice(const Vector &field_in_evec,
 #endif
 
 void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
-                                 int point_pos_ordering, const double bb_t,
+                                 const int point_pos_ordering,
+                                 const double bb_t,
                                  const double newt_tol, const int npt_max)
 {
    if (!setupflag || (mesh != &m) )
@@ -1105,16 +1106,28 @@ void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
 }
 
 void FindPointsGSLIB::Interpolate(const Vector &point_pos,
-                                  const GridFunction &field_in, Vector &field_out,
-                                  int point_pos_ordering)
+                                  const GridFunction &field_in,
+                                  Vector &field_out,
+                                  const int point_pos_ordering)
 {
    FindPoints(point_pos, point_pos_ordering);
    Interpolate(field_in, field_out);
 }
 
+void FindPointsGSLIB::Interpolate(const Vector &point_pos,
+                                  const GridFunction &field_in,
+                                  Vector &field_out,
+                                  const int point_pos_ordering,
+                                  const int field_out_ordering)
+{
+   FindPoints(point_pos, point_pos_ordering);
+   Interpolate(field_in, field_out, field_out_ordering);
+}
+
 void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
-                                  const GridFunction &field_in, Vector &field_out,
-                                  int point_pos_ordering)
+                                  const GridFunction &field_in,
+                                  Vector &field_out,
+                                  const int point_pos_ordering)
 {
    FindPoints(m, point_pos, point_pos_ordering);
    Interpolate(field_in, field_out);
@@ -1470,7 +1483,7 @@ void FindPointsGSLIB::SetupSplitMeshesAndIntegrationRules(const int order)
 }
 
 void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
-                                     Vector &node_vals)
+                                     Vector &node_vals) const
 {
    const GridFunction *nodes     = gf_in;
    const FiniteElementSpace *fes = nodes->FESpace();
@@ -1758,6 +1771,13 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
 
 void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
                                   Vector &field_out)
+{
+   Interpolate(field_in, field_out, field_in.FESpace()->GetOrdering());
+}
+
+void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
+                                  Vector &field_out,
+                                  const int field_out_ordering)
 {
    const int  gf_order   = field_in.FESpace()->GetMaxElementOrder(),
               mesh_order = mesh->GetNodalFESpace()->GetMaxElementOrder();
@@ -1800,7 +1820,7 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
       const int maxOrder = field_in.FESpace()->GetMaxElementOrder();
 
       InterpolateOnDevice(node_vals, field_out, NE_split_total, ncomp,
-                          maxOrder+1, field_in.FESpace()->GetOrdering());
+                          maxOrder+1, field_out_ordering);
       return;
 #endif
    }
@@ -1812,12 +1832,13 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
        field_in.FESpace()->IsVariableOrder() ==
        mesh->GetNodalFESpace()->IsVariableOrder())
    {
-      InterpolateH1(field_in, field_out);
+      InterpolateH1(field_in, field_out, field_out_ordering);
       return;
    }
    else
    {
-      InterpolateGeneral(field_in, field_out);
+      InterpolateGeneral(field_in, field_out,
+                         field_out_ordering);
       if (!fec_l2 || avgtype == AvgType::NONE) { return; }
    }
 
@@ -1861,11 +1882,11 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
 
       if (gf_order_h1 == mesh_order) // basis is GaussLobatto by default
       {
-         InterpolateH1(field_in_h1, field_out_l2);
+         InterpolateH1(field_in_h1, field_out_l2, field_out_ordering);
       }
       else
       {
-         InterpolateGeneral(field_in_h1, field_out_l2);
+         InterpolateGeneral(field_in_h1, field_out_l2, field_out_ordering);
       }
 
       // Copy interpolated values for the points on element border
@@ -1873,7 +1894,7 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
       {
          for (int i = 0; i < indl2.Size(); i++)
          {
-            int idx = field_in_h1.FESpace()->GetOrdering() == Ordering::byNODES?
+            int idx = field_out_ordering == Ordering::byNODES?
                       indl2[i] + j*points_cnt:
                       indl2[i]*ncomp + j;
             field_out(idx) = field_out_l2(idx);
@@ -1883,7 +1904,8 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
 }
 
 void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
-                                    Vector &field_out)
+                                    Vector &field_out,
+                                    const int field_out_ordering)
 {
    FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
    if (field_in.FESpace()->IsVariableOrder())
@@ -1913,7 +1935,8 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
                 dataptrout = i*points_cnt;
       if (field_in.FESpace()->GetOrdering() == Ordering::byNODES)
       {
-         field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
+         field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin,
+                                        points_fld);
       }
       else
       {
@@ -1945,7 +1968,7 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
                         (gslib::findpts_data_3 *)this->fdataD);
       }
    }
-   if (field_in.FESpace()->GetOrdering() == Ordering::byVDIM)
+   if (field_out_ordering == Ordering::byVDIM)
    {
       Vector field_out_temp = field_out;
       for (int i = 0; i < ncomp; i++)
@@ -1959,7 +1982,8 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
 }
 
 void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
-                                         Vector &field_out)
+                                         Vector &field_out,
+                                         const int field_out_ordering)
 {
    int ncomp   = field_in.VectorDim(),
        nptorig = points_cnt,
@@ -1979,7 +2003,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
          if (dim == 3) { ip.z = gsl_mfem_ref(index*dim + 2); }
          Vector localval(ncomp);
          field_in.GetVectorValue(gsl_mfem_elem[index], ip, localval);
-         if (field_in.FESpace()->GetOrdering() == Ordering::byNODES)
+         if (field_out_ordering == Ordering::byNODES)
          {
             for (int i = 0; i < ncomp; i++)
             {
@@ -2014,7 +2038,10 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
       for (int index = 0; index < npt; index++)
       {
          if (gsl_code[index] == 2) { continue; }
-         for (int d = 0; d < dim; ++d) { pt->r[d]= gsl_mfem_ref(index*dim + d); }
+         for (int d = 0; d < dim; ++d)
+         {
+            pt->r[d]= gsl_mfem_ref(index*dim + d);
+         }
          pt->index = index;
          pt->proc  = gsl_proc[index];
          pt->el    = gsl_mfem_elem[index];
@@ -2104,7 +2131,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
             sdpt = (struct send_pt *)sendpt->ptr;
             for (int index = 0; index < static_cast<int>(sendpt->n); index++)
             {
-               int idx = field_in.FESpace()->GetOrdering() == Ordering::byNODES ?
+               int idx = field_out_ordering == Ordering::byNODES ?
                          sdpt->index + j*nptorig :
                          sdpt->index*ncomp + j;
                field_out(idx) = sdpt->ival;
@@ -2246,7 +2273,7 @@ void FindPointsGSLIB::DistributeInterpolatedValues(const Vector &int_vals,
    }
 }
 
-void FindPointsGSLIB::GetAxisAlignedBoundingBoxes(Vector &aabb)
+void FindPointsGSLIB::GetAxisAlignedBoundingBoxes(Vector &aabb) const
 {
    MFEM_VERIFY(setupflag, "Call FindPointsGSLIB::Setup method first");
    auto *findptsData3 = (gslib::findpts_data_3 *)this->fdataD;
@@ -2317,7 +2344,7 @@ void FindPointsGSLIB::GetAxisAlignedBoundingBoxes(Vector &aabb)
 }
 
 void FindPointsGSLIB::GetOrientedBoundingBoxes(DenseTensor &obbA, Vector &obbC,
-                                               Vector &obbV)
+                                               Vector &obbV) const
 {
    MFEM_VERIFY(setupflag, "Call FindPointsGSLIB::Setup method first");
    auto *findptsData3 = (gslib::findpts_data_3 *)this->fdataD;
@@ -2502,8 +2529,8 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
 }
 
 void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
-                                        Array<unsigned int> &point_id,
-                                        int point_pos_ordering)
+                                        const Array<unsigned int> &point_id,
+                                        const int point_pos_ordering)
 {
    MFEM_VERIFY(setupflag, "Use OversetFindPointsGSLIB::Setup before "
                "finding points.");
@@ -2582,10 +2609,10 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
 }
 
 void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
-                                         Array<unsigned int> &point_id,
+                                         const Array<unsigned int> &point_id,
                                          const GridFunction &field_in,
                                          Vector &field_out,
-                                         int point_pos_ordering)
+                                         const int point_pos_ordering)
 {
    FindPoints(point_pos, point_id, point_pos_ordering);
    Interpolate(field_in, field_out);

fem/gslib.hpp

@@ -119,11 +119,13 @@ protected:
    } DEV;
 
    /// Use GSLIB for communication and interpolation
-   virtual void InterpolateH1(const GridFunction &field_in, Vector &field_out);
+   virtual void InterpolateH1(const GridFunction &field_in, Vector &field_out,
+                              const int field_out_ordering);
    /// Uses GSLIB Crystal Router for communication followed by MFEM's
    /// interpolation functions
    virtual void InterpolateGeneral(const GridFunction &field_in,
-                                   Vector &field_out);
+                                   Vector &field_out,
+                                   const int field_out_ordering);
 
    /// Since GSLIB is designed to work with quads/hexes, we split every
    /// triangle/tet/prism/pyramid element into quads/hexes.
@@ -140,7 +142,7 @@ protected:
    virtual void SetupSplitMeshesAndIntegrationRules(const int order);
 
    /// Get GridFunction value at the points expected by GSLIB.
-   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals);
+   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals) const;
 
    /// Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For simplices,
    /// find the original element number (that was split into micro quads/hexes)
@@ -182,7 +184,7 @@ protected:
        These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
        byVDim: (XYZ,XYZ,....XYZ) specified by @a point_pos_ordering. */
    void FindPointsOnDevice(const Vector &point_pos,
-                           int point_pos_ordering = Ordering::byNODES);
+                           const int point_pos_ordering = Ordering::byNODES);
 
    /** Interpolation of field values at prescribed reference space positions.
        @param[in] field_in_evec E-vector of grid function to be interpolated.
@@ -253,10 +255,15 @@ public:
        #gsl_dist        Distance between the sought and the found point
                         in physical space. */
    void FindPoints(const Vector &point_pos,
-                   int point_pos_ordering = Ordering::byNODES);
+                   const int point_pos_ordering = Ordering::byNODES);
+   /// Convenience function when point positions are in a ParticleVector
+   void FindPoints(const ParticleVector &point_pos)
+   {
+      FindPoints(point_pos, point_pos.GetOrdering());
+   }
    /// Setup FindPoints and search positions
    void FindPoints(Mesh &m, const Vector &point_pos,
-                   int point_pos_ordering = Ordering::byNODES,
+                   const int point_pos_ordering = Ordering::byNODES,
                    const double bb_t = 0.1, const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
@@ -268,18 +275,31 @@ public:
        @param[out] field_out  Interpolated values. For points that are not found
                               the value is set to #default_interp_value. */
    virtual void Interpolate(const GridFunction &field_in, Vector &field_out);
+   /// Interpolation of field values, with output ordering specification.
+   virtual void Interpolate(const GridFunction &field_in, Vector &field_out,
+                            const int field_out_ordering);
+   /// Interpolation of field values, with output ordering from ParticleVector
+   virtual void Interpolate(const GridFunction &field_in,
+                            ParticleVector &field_out)
+   {
+      Interpolate(field_in, field_out, field_out.GetOrdering());
+   }
    /** Search positions and interpolate. The ordering (byNODES or byVDIM) of
        the output values in \p field_out corresponds to the ordering used
        in the input GridFunction \p field_in. */
    void Interpolate(const Vector &point_pos, const GridFunction &field_in,
                     Vector &field_out,
-                    int point_pos_ordering = Ordering::byNODES);
+                    const int point_pos_ordering = Ordering::byNODES);
+   /// Search positions and interpolate with given point and output ordering.
+   void Interpolate(const Vector &point_pos, const GridFunction &field_in,
+                    Vector &field_out, const int point_pos_ordering,
+                    const int field_out_ordering);
    /** Setup FindPoints, search positions and interpolate. The ordering (byNODES
        or byVDIM) of the output values in \p field_out corresponds to the
        ordering used in the input GridFunction \p field_in. */
    void Interpolate(Mesh &m, const Vector &point_pos,
                     const GridFunction &field_in, Vector &field_out,
-                    int point_pos_ordering = Ordering::byNODES);
+                    const int point_pos_ordering = Ordering::byNODES);
 
    /// Average type to be used for L2 functions in-case a point is located at
    /// an element boundary where the function might be multi-valued.
@@ -376,7 +396,7 @@ public:
    /// The size of the returned vector is (nel x nverts x dim), where nel is the
    /// number of elements (after splitting for simplcies), nverts is number of
    /// vertices (4 in 2D, 8 in 3D), and dim is the spatial dimension.
-   void GetAxisAlignedBoundingBoxes(Vector &aabb);
+   void GetAxisAlignedBoundingBoxes(Vector &aabb) const;
 
    /// Return the oriented bounding boxes (OBB) computed during \ref Setup.
    /// Each OBB is represented using the inverse transformation (A^{-1}) and
@@ -386,7 +406,8 @@ public:
    /// size (dim x dim x nel), and the OBB centers are returned in \p obbC,
    /// a vector of size (nel x dim). The vertices of the OBBs are returned in
    /// \p obbV, a vector of size (nel x nverts x dim) .
-   void GetOrientedBoundingBoxes(DenseTensor &obbA, Vector &obbC, Vector &obbV);
+   void GetOrientedBoundingBoxes(DenseTensor &obbA, Vector &obbC,
+                                 Vector &obbV) const;
 };
 
 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
@@ -446,13 +467,14 @@ public:
                                        byNodes: (XXX...,YYY...,ZZZ) or
                                        byVDim: (XYZ,XYZ,....XYZ) */
    void FindPoints(const Vector &point_pos,
-                   Array<unsigned int> &point_id,
-                   int point_pos_ordering = Ordering::byNODES);
+                   const Array<unsigned int> &point_id,
+                   const int point_pos_ordering = Ordering::byNODES);
 
    /** Search positions and interpolate */
-   void Interpolate(const Vector &point_pos, Array<unsigned int> &point_id,
+   void Interpolate(const Vector &point_pos,
+                    const Array<unsigned int> &point_id,
                     const GridFunction &field_in, Vector &field_out,
-                    int point_pos_ordering = Ordering::byNODES);
+                    const int point_pos_ordering = Ordering::byNODES);
    using FindPointsGSLIB::Interpolate;
 };
 

fem/linearform_ext.hpp

@@ -18,6 +18,7 @@
 namespace mfem
 {
 
+class Operator;
 class LinearForm;
 
 /// Class extending the LinearForm class to support assembly on devices.

fem/lor/lor_dg.hpp

@@ -23,6 +23,8 @@ class BatchedLOR_DG : BatchedLORKernel
 {
    IntegrationRule ir_face; ///< Collocated Gauss-Lobatto face quadrature rule.
    real_t kappa; ///< DG penalty parameter.
+   bool has_bdr_integ; ///< Is there a boundary integrator?
+   const Array<int> *bdr_markers; ///< Boundary integrator markers.
 public:
    template <int ORDER, int SDIM> void Assemble2D();
    template <int ORDER> void Assemble3D();
@@ -38,8 +40,7 @@ public:
       ProjectLORCoefficient<MassIntegrator>(a, c1);
       ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
 
-      auto *integ = GetInteriorFaceIntegrator<DGDiffusionIntegrator>(a);
-      if (integ)
+      if (auto *integ = GetInteriorFaceIntegrator<DGDiffusionIntegrator>(a))
       {
          kappa = integ->GetPenaltyParameter();
       }
@@ -47,6 +48,19 @@ public:
       {
          kappa = 0.0;
       }
+
+      has_bdr_integ = false;
+      auto *bdr_face_integs = a.GetBFBFI();
+      for (int i = 0; i < bdr_face_integs->Size(); ++i)
+      {
+         if (auto *integ = dynamic_cast<DGDiffusionIntegrator*>((*bdr_face_integs)[i]))
+         {
+            kappa = integ->GetPenaltyParameter();
+            bdr_markers = (*a.GetBFBFI_Marker())[i];
+            has_bdr_integ = true;
+            break;
+         }
+      }
    }
 
    /// @brief Compute and return the face info array.

fem/lor/lor_dg_impl.hpp

@@ -22,9 +22,13 @@ namespace mfem
 Array<int> BatchedLOR_DG::GetFaceInfo() const
 {
    Mesh &mesh = *fes_ho.GetMesh();
+   const Array<int> &bdr_face_attrs = mesh.GetBdrFaceAttributes();
    const int nf = mesh.GetNumFaces();
    Array<int> face_info(nf * 6); // (e0, f0, o0, e1, f1, o1)
    auto h_face_info = Reshape(face_info.HostWrite(), 6, nf);
+
+   int bdr_face_counter = 0;
+
    for (int f = 0; f < nf; ++f)
    {
       auto finfo = mesh.GetFaceInformation(f);
@@ -43,6 +47,19 @@ Array<int> BatchedLOR_DG::GetFaceInfo() const
          h_face_info(4, f) = -1;
          h_face_info(5, f) = -1;
       }
+
+      if (finfo.IsBoundary())
+      {
+         // Check if Neumann boundary; skip these when adding boundary penalties
+         const int bdr_attr = bdr_face_attrs[bdr_face_counter];
+         if (!has_bdr_integ || (bdr_markers && !(*bdr_markers)[bdr_attr - 1]))
+         {
+            h_face_info(0, f) = -1;
+            h_face_info(1, f) = -1;
+            h_face_info(2, f) = -1;
+         }
+         bdr_face_counter += 1;
+      }
    }
    return face_info;
 }
@@ -144,6 +161,7 @@ void BatchedLOR_DG::AssembleFaceTerms()
    {
       const int f_0 = d_face_info(1, f);
       const int f_1 = d_face_info(4, f);
+      if (f_0 < 0) { return; } // Skip Neumann boundary faces
       const int nsides = (f_1 >= 0) ? 2 : 1;
       for (int el_i = 0; el_i < nsides; ++el_i)
       {

fem/moonolith/mortarassembler.cpp

@@ -78,10 +78,7 @@ template <int Dim>
 void BuildBoxes(const Mesh &mesh,
                 std::vector<::moonolith::AABB<Dim, double>> &element_boxes)
 {
-#ifndef NDEBUG
-   const int dim = mesh.Dimension();
-   assert(dim == Dim);
-#endif
+   MFEM_ASSERT(mesh.Dimension() == Dim, "Mesh and box dimensions mismatched");
    element_boxes.resize(mesh.GetNE());
 
    DenseMatrix pts;

fem/pgridfunc.cpp

@@ -39,10 +39,13 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, const GridFunction *gf,
 {
    const FiniteElementSpace *glob_fes = gf->FESpace();
    // duplicate the FiniteElementCollection from 'gf'
-   fec_owned = FiniteElementCollection::New(glob_fes->FEColl()->Name());
+   fec.reset(FiniteElementCollection::New(glob_fes->FEColl()->Name()));
+
    // create a local ParFiniteElementSpace from the global one:
-   fes = pfes = new ParFiniteElementSpace(pmesh, glob_fes, partitioning,
-                                          fec_owned);
+   owned_fes = std::make_shared<ParFiniteElementSpace>(pmesh, glob_fes,
+                                                       partitioning, fec.get());
+   fes = owned_fes.get();
+   pfes = static_cast<ParFiniteElementSpace *>(fes);
    SetSize(pfes->GetVSize());
 
    if (partitioning)
@@ -76,10 +79,17 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, std::istream &input)
    : GridFunction(pmesh, input)
 {
    // Convert the FiniteElementSpace, fes, to a ParFiniteElementSpace:
-   pfes = new ParFiniteElementSpace(pmesh, fec_owned, fes->GetVDim(),
-                                    fes->GetOrdering());
-   delete fes;
-   fes = pfes;
+   owned_fes = std::make_shared<ParFiniteElementSpace>(
+                  pmesh, fec.get(), fes->GetVDim(), fes->GetOrdering());
+   fes = owned_fes.get();
+   pfes = static_cast<ParFiniteElementSpace*>(fes);
+}
+
+ParGridFunction& ParGridFunction::operator=(const ParGridFunction &rhs)
+{
+   operator=((const GridFunction &)rhs);
+   pfes = rhs.pfes;
+   return *this;
 }
 
 void ParGridFunction::Update()
@@ -543,13 +553,22 @@ void ParGridFunction::GetElementDofValues(int el, Vector &dof_vals) const
    }
 }
 
-void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
+void ParGridFunction::ProjectCoefficient(Coefficient &coeff, ProjectType type)
 {
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
 
    if (delta_c == NULL)
    {
-      GridFunction::ProjectCoefficient(coeff);
+      (*this) = std::numeric_limits<real_t>::min();
+      GridFunction::ProjectCoefficient(coeff,type);
+
+      // Accumulate for all vdofs.
+      if (pfes->GetNURBSext())
+      {
+         GroupCommunicator &gcomm = pfes->GroupComm();
+         gcomm.Reduce<real_t>(data, GroupCommunicator::Max);
+         gcomm.Bcast<real_t>(data);
+      }
    }
    else
    {
@@ -565,6 +584,147 @@ void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
    }
 }
 
+void ParGridFunction::ProjectCoefficient(VectorCoefficient &vcoeff,
+                                         ProjectType type)
+{
+   GridFunction::ProjectCoefficient(vcoeff, type);
+
+   // Accumulate for all vdofs.
+   if (pfes->GetNURBSext())
+   {
+      GroupCommunicator &gcomm = pfes->GroupComm();
+      gcomm.Reduce<real_t>(data, GroupCommunicator::Max);
+      gcomm.Bcast<real_t>(data);
+   }
+}
+
+void ParGridFunction::ProjectCoefficientGlobalL2(Coefficient &coeff,
+                                                 real_t rtol,
+                                                 int iter)
+{
+   // Define and assemble linear form
+   ParLinearForm b(pfes);
+   b.AddDomainIntegrator(new DomainLFIntegrator(coeff));
+   b.Assemble();
+
+   // Define and assemble bilinear form
+   ParBilinearForm a(pfes);
+   a.AddDomainIntegrator(new MassIntegrator());
+   a.Assemble();
+
+   // Configure solver
+   OperatorPtr A;
+   Vector B, X, x(*this);
+   Array<int> ess_tdof_list;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   Solver *prec = new HypreBoomerAMG;
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(rtol);
+   cg.SetMaxIter(iter);
+   cg.SetPrintLevel(0);
+   cg.SetPreconditioner(*prec);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   a.RecoverFEMSolution(X, b, x);
+   delete prec;
+}
+
+void ParGridFunction::ProjectCoefficientElementL2(Coefficient &coeff)
+{
+   Vector Va;
+   ProjectCoefficientElementL2_(coeff, *this, Va);
+
+   GroupCommunicator &gcomm = pfes->GroupComm();
+   gcomm.Reduce<real_t>(GetData(), GroupCommunicator::Sum);
+   gcomm.Bcast<real_t>(GetData());
+
+   gcomm.Reduce<real_t>(Va.GetData(), GroupCommunicator::Sum);
+   gcomm.Bcast<real_t>(Va.GetData());
+   (*this)/=Va;
+}
+
+void ParGridFunction::ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
+                                                 real_t rtol, int iter)
+{
+   // Define and assemble linear form
+   ParLinearForm b(pfes);
+   ParBilinearForm a(pfes);
+
+   // Dimension argument to GetRangeType is arbitrary to be 3, could also be 2.
+   if (fes->FEColl()->GetRangeType(3)  == mfem::FiniteElement::VECTOR)
+   {
+      b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(vcoeff));
+      a.AddDomainIntegrator(new VectorFEMassIntegrator());
+   }
+   else
+   {
+      b.AddDomainIntegrator(new VectorDomainLFIntegrator(vcoeff));
+      a.AddDomainIntegrator(new VectorMassIntegrator());
+   }
+   b.Assemble();
+   a.Assemble();
+
+   // Configure solver
+   OperatorPtr A;
+   Vector B, X, x(*this);
+   x = 0.0;
+   Array<int> ess_tdof_list;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   Solver *prec = new HypreBoomerAMG;
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(rtol);
+   cg.SetMaxIter(iter);
+   cg.SetPrintLevel(0);
+   cg.SetPreconditioner(*prec);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   a.RecoverFEMSolution(X, b, x);
+   x.Print();
+   delete prec;
+}
+
+void ParGridFunction::ProjectCoefficientElementL2(VectorCoefficient &vcoeff)
+{
+   if (fes->GetTypicalFE()->GetRangeType() == mfem::FiniteElement::VECTOR)
+   {
+      Vector Va;
+      ProjectCoefficientElementL2_(vcoeff, *this, Va);
+
+      GroupCommunicator &gcomm = pfes->GroupComm();
+      gcomm.Reduce<real_t>(GetData(), GroupCommunicator::Sum);
+      gcomm.Bcast<real_t>(GetData());
+
+      gcomm.Reduce<real_t>(Va.GetData(), GroupCommunicator::Sum);
+      gcomm.Bcast<real_t>(Va.GetData());
+      (*this)/=Va;
+   }
+   else
+   {
+      Array<int> vdofs(fes->GetNDofs());
+      Vector x, Va, gVa(Size());
+      VectorComponentCoefficient coeff(vcoeff,0);
+      *this = 0.0;
+      gVa = 0.0;
+      for (int v = 0; v < VectorDim(); v++)
+      {
+         coeff.SetComponent(v);
+         ProjectCoefficientElementL2_(coeff, x, Va);
+         fes->GetVDofs(v, vdofs);
+         SetSubVector(vdofs, x);
+         gVa.SetSubVector(vdofs, Va);
+      }
+
+      GroupCommunicator &gcomm = pfes->GroupComm();
+      gcomm.Reduce<real_t>(GetData(), GroupCommunicator::Sum);
+      gcomm.Bcast<real_t>(GetData());
+
+      gcomm.Reduce<real_t>(gVa.GetData(), GroupCommunicator::Sum);
+      gcomm.Bcast<real_t>(gVa.GetData());
+      *this /= gVa;
+   }
+}
+
+
 void ParGridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff)
 {
    // local maximal element attribute for each dof
@@ -1056,7 +1216,7 @@ GridFunction ParGridFunction::GetSerialGridFunction(int save_rank,
                                              pfes->GetVDim(),
                                              pfes->GetOrdering());
    GridFunction serial_gf = GetSerialGridFunction(save_rank, *serial_fes);
-   serial_gf.MakeOwner(serial_fec); // Also assumes ownership of serial_fes
+   serial_gf.MakeOwner(); // Also assumes ownership of serial_fes
    return serial_gf;
 }
 
@@ -1296,7 +1456,7 @@ std::unique_ptr<ParGridFunction> ParGridFunction::ProlongateToMaxOrder() const
    PRefinementTransferOperator P(*pfes, *pfesMax);
    P.Mult(*this, *xMax);
 
-   xMax->MakeOwner(fecMax);
+   xMax->MakeOwner();
    return std::unique_ptr<ParGridFunction>(xMax);
 }
 

fem/pgridfunc.hpp

@@ -70,8 +70,14 @@ public:
    ParGridFunction(const ParGridFunction &orig)
       : GridFunction(orig), pfes(orig.pfes) { }
 
+   ParGridFunction(ParGridFunction &&orig) = default;
+
    ParGridFunction(ParFiniteElementSpace *pf) : GridFunction(pf), pfes(pf) { }
 
+   /// Same as above but specify the device memory type
+   ParGridFunction(ParFiniteElementSpace *pf, MemoryType mt) :
+      GridFunction(pf, mt), pfes(pf) { }
+
    /// Construct a ParGridFunction using previously allocated array @a data.
    /** The ParGridFunction does not assume ownership of @a data which is assumed
        to be of size at least `pf->GetVSize()`. Similar to the GridFunction and
@@ -110,14 +116,8 @@ public:
        constructed. The new ParGridFunction assumes ownership of both. */
    ParGridFunction(ParMesh *pmesh, std::istream &input);
 
-   /// Copy assignment. Only the data of the base class Vector is copied.
-   /** It is assumed that this object and @a rhs use ParFiniteElementSpace%s
-       that have the same size.
-
-       @note Defining this method overwrites the implicitly defined copy
-       assignment operator. */
-   ParGridFunction &operator=(const ParGridFunction &rhs)
-   { return operator=((const Vector &)rhs); }
+   ParGridFunction &operator=(const ParGridFunction &rhs);
+   ParGridFunction &operator=(ParGridFunction &&orig) = default;
 
    /// Assign constant values to the ParGridFunction data.
    ParGridFunction &operator=(real_t value)
@@ -257,7 +257,11 @@ public:
    void GetElementDofValues(int el, Vector &dof_vals) const override;
 
    using GridFunction::ProjectCoefficient;
-   void ProjectCoefficient(Coefficient &coeff) override;
+   void ProjectCoefficient(Coefficient &coeff,
+                           ProjectType type = ProjectType::DEFAULT) override;
+
+   void ProjectCoefficient(VectorCoefficient &vcoeff,
+                           ProjectType type = ProjectType::DEFAULT) override;
 
    using GridFunction::ProjectDiscCoefficient;
    /** @brief Project a discontinuous vector coefficient as a grid function on
@@ -282,6 +286,18 @@ public:
    void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                      const Array<int> &bdr_attr) override;
 
+   void ProjectCoefficientGlobalL2(Coefficient &coeff,
+                                   real_t rtol = 1e-12,
+                                   int iter = 1000) override;
+
+   void ProjectCoefficientElementL2(Coefficient &coeff) override;
+
+   void ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
+                                   real_t rtol = 1e-12,
+                                   int iter = 1000) override;
+
+   void ProjectCoefficientElementL2(VectorCoefficient &vcoeff) override;
+
    /// @brief Returns ||u_ex - u_h||_L1 in parallel for H1 or L2 elements
    ///
    /// @see GridFunction::ComputeL1Error(Coefficient *exsol[],

fem/tmop.cpp

@@ -3938,7 +3938,7 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
                                                                dim);
    // Initial gradients.
    surf_fit_grad = new GridFunction(fes_grad);
-   surf_fit_grad->MakeOwner(fec_grad);
+   surf_fit_grad->MakeOwner();
    for (int d = 0; d < dim; d++)
    {
       ParGridFunction surf_fit_grad_comp(fes, surf_fit_grad->GetData()+d*s0.Size());
@@ -3956,7 +3956,7 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
                                                                dim*dim);
    // Initial Hessians.
    surf_fit_hess = new GridFunction(fes_hess);
-   surf_fit_hess->MakeOwner(fec_hess);
+   surf_fit_hess->MakeOwner();
    int id = 0;
    for (int d = 0; d < dim; d++)
    {

general/device.cpp

@@ -22,6 +22,8 @@
 
 #include <unordered_map>
 #include <map>
+#include <sstream>
+#include <iomanip>
 
 namespace mfem
 {
@@ -716,6 +718,29 @@ void Device::DeviceMem(size_t *free, size_t *total)
 #endif
 }
 
+std::string Device::GetUUID(const int device_id)
+{
+   std::stringstream res;
+#if defined(MFEM_USE_CUDA)
+   cudaDeviceProp prop;
+   MFEM_GPU_CHECK(cudaGetDeviceProperties(&prop, device_id));
+   for (int i = 0; i < 16; ++i)
+   {
+      res << std::setfill('0') << std::setw(2) << std::hex
+          << static_cast<unsigned>(prop.uuid.bytes[i]);
+   }
+#elif defined(MFEM_USE_HIP)
+   hipUUID uuid;
+   MFEM_GPU_CHECK(hipDeviceGetUuid(&uuid, device_id));
+   for (int i = 0; i < 16; ++i)
+   {
+      res << std::setfill('0') << std::setw(2) << std::hex
+          << static_cast<unsigned>(uuid.bytes[i]);
+   }
+#endif
+   return res.str();
+}
+
 int Device::NumMultiprocessors(int dev)
 {
 #if defined(MFEM_USE_CUDA)

general/device.hpp

@@ -255,6 +255,10 @@ public:
    /// Get the number of available devices (may be called before configuration).
    static int GetDeviceCount();
 
+   /// Gets a string representation of the GPU UUID.
+   /// 0 <= @a device_id < GetDeviceCount()
+   static std::string GetUUID(const int device_id = 0);
+
    /** @brief Return true if any of the backends in the backend mask, @a b_mask,
        are allowed. */
    /** This method can be used with any of the Backend::Id constants, the

general/optparser.hpp

@@ -14,11 +14,12 @@
 
 #include "../config/config.hpp"
 #include "array.hpp"
-#include "../linalg/vector.hpp"
 
 namespace mfem
 {
 
+class Vector;
+
 /** Class for parsing command-line options.
 
     The class is initialized with argc and argv, and new options are added with

general/table.hpp

@@ -146,7 +146,7 @@ public:
    int *ReadWriteJ(bool on_dev = true) { return J.ReadWrite(on_dev); }
    const int *HostReadJ() const { return J.HostRead(); }
    int *HostWriteJ() { return J.HostWrite(); }
-   int *ReadWriteJ() { return J.HostReadWrite(); }
+   int *HostReadWriteJ() { return J.HostReadWrite(); }
 
    /// Sort the column (TYPE II) indices in each row.
    void SortRows();

linalg/densemat.cpp

@@ -1370,6 +1370,35 @@ void DenseMatrix::Getl1Diag(Vector &l) const
       }
 }
 
+void DenseMatrix::GetRowl1(Vector &l) const
+{
+   l.SetSize(height);
+   l = 0.0;
+
+   for (int j = 0; j < width; ++j)
+      for (int i = 0; i < height; ++i)
+      {
+         l(i) += fabs((*this)(i,j));
+      }
+}
+
+void DenseMatrix::GetRowl2(Vector &l) const
+{
+   l.SetSize(height);
+   l = 0.0;
+
+   for (int j = 0; j < width; ++j)
+      for (int i = 0; i < height; ++i)
+      {
+         l[i] += operator()(i,j)*operator()(i,j);
+      }
+
+   for (int i = 0; i < height; ++i)
+   {
+      l[i] = sqrt(l[i]);
+   }
+}
+
 void DenseMatrix::GetRowSums(Vector &l) const
 {
    l.SetSize(height);

linalg/densemat.hpp

@@ -346,8 +346,12 @@ public:
    /// Returns the diagonal of the matrix
    void GetDiag(Vector &d) const;
    /// Returns the l1 norm of the rows of the matrix v_i = sum_j |a_ij|
-   void Getl1Diag(Vector &l) const;
-   /// Compute the row sums of the DenseMatrix
+   MFEM_DEPRECATED void Getl1Diag(Vector &l) const;
+   /// Returns the l1 norm of the rows of the matrix v_i = sum_j |a_ij|
+   void GetRowl1(Vector &l) const;
+   /// Returns the l2norm of the rows of the DenseMatrix
+   void GetRowl2(Vector &l) const;
+   /// Returns the row sums of the DenseMatrix
    void GetRowSums(Vector &l) const;
 
    /// Creates n x n diagonal matrix with diagonal elements c

linalg/hypre.cpp

@@ -1681,6 +1681,13 @@ void HypreParMatrix::GetOffd(SparseMatrix &offd, HYPRE_BigInt* &cmap) const
    cmap = A->col_map_offd;
 }
 
+void HypreParMatrix::GetOffdColMap(HYPRE_BigInt* &cmap,
+                                   HYPRE_Int &num_cols) const
+{
+   cmap = A->col_map_offd;
+   num_cols = hypre_CSRMatrixNumCols(A->offd);
+}
+
 void HypreParMatrix::MergeDiagAndOffd(SparseMatrix &merged)
 {
    HostRead();

linalg/hypre.hpp

@@ -665,6 +665,8 @@ public:
    void GetDiag(SparseMatrix &diag) const;
    /// Get the local off-diagonal block. NOTE: 'offd' will not own any data.
    void GetOffd(SparseMatrix &offd, HYPRE_BigInt* &cmap) const;
+   /// Get the global column mapping for the local off-diagonal block.
+   void GetOffdColMap(HYPRE_BigInt* &cmap, HYPRE_Int &num_cols) const;
    /** @brief Get a single SparseMatrix containing all rows from this processor,
        merged from the diagonal and off-diagonal blocks stored by the
        HypreParMatrix. */
@@ -959,6 +961,14 @@ public:
    const Memory<HYPRE_Int> &GetDiagMemoryJ() const { return mem_diag.J; }
    const Memory<real_t> &GetDiagMemoryData() const { return mem_diag.data; }
 
+   Memory<HYPRE_Int> &GetOffdMemoryI() { return mem_offd.I; }
+   Memory<HYPRE_Int> &GetOffdMemoryJ() { return mem_offd.J; }
+   Memory<real_t> &GetOffdMemoryData() { return mem_offd.data; }
+
+   const Memory<HYPRE_Int> &GetOffdMemoryI() const { return mem_offd.I; }
+   const Memory<HYPRE_Int> &GetOffdMemoryJ() const { return mem_offd.J; }
+   const Memory<real_t> &GetOffdMemoryData() const { return mem_offd.data; }
+
    /// @brief Prints the locally owned rows in parallel. The resulting files can
    /// be read with Read_IJMatrix().
    void Print(const std::string &fname, HYPRE_Int offi = 0,

linalg/matrix.cpp

@@ -19,8 +19,7 @@
 namespace mfem
 {
 
-template <class T>
-void MatrixMP<T>::Print(std::ostream & os, int width_) const
+void Matrix::Print (std::ostream & os, int width_) const
 {
    using namespace std;
    // output flags = scientific + show sign
@@ -41,10 +40,4 @@ void MatrixMP<T>::Print(std::ostream & os, int width_) const
    os << '\n';
 }
 
-template class MatrixMP<float>;
-template class MatrixMP<double>;
-
-template class AbstractSparseMatrixMP<float>;
-template class AbstractSparseMatrixMP<double>;
-
 }

linalg/matrix.hpp

@@ -21,39 +21,31 @@ namespace mfem
 
 // Abstract data types matrix, inverse matrix
 
-template <class T>
-class MatrixInverseMP;
+class MatrixInverse;
 
 /// Abstract data type matrix
-
-template <class T>
-class MatrixMP : public OperatorMP<T>
+class Matrix : public Operator
 {
-   friend class MatrixInverseMP<T>;
-
-protected:
-   using OperatorBase::height;
-   using OperatorBase::width;
-
+   friend class MatrixInverse;
 public:
 
    /// Creates a square matrix of size s.
-   explicit MatrixMP(int s) : OperatorMP<T>(s) { }
+   explicit Matrix(int s) : Operator(s) { }
 
    /// Creates a matrix of the given height and width.
-   explicit MatrixMP(int h, int w) : OperatorMP<T>(h, w) { }
+   explicit Matrix(int h, int w) : Operator(h, w) { }
 
    /// Returns whether the matrix is a square matrix.
    bool IsSquare() const { return (height == width); }
 
    /// Returns reference to a_{ij}.
-   virtual T &Elem(int i, int j) = 0;
+   virtual real_t &Elem(int i, int j) = 0;
 
    /// Returns constant reference to a_{ij}.
-   virtual const T &Elem(int i, int j) const = 0;
+   virtual const real_t &Elem(int i, int j) const = 0;
 
    /// Returns a pointer to (an approximation) of the matrix inverse.
-   virtual MatrixInverseMP<T> *Inverse() const = 0;
+   virtual MatrixInverse *Inverse() const = 0;
 
    /// Finalizes the matrix initialization.
    virtual void Finalize(int) { }
@@ -62,35 +54,30 @@ public:
    virtual void Print(std::ostream & os = mfem::out, int width_ = 4) const;
 
    /// Destroys matrix.
-   virtual ~MatrixMP() { }
+   virtual ~Matrix() { }
 };
 
-using Matrix = MatrixMP<real_t>;
 
 /// Abstract data type for matrix inverse
-template <class T>
-class MatrixInverseMP : public SolverMP<T>
+class MatrixInverse : public Solver
 {
 public:
-   MatrixInverseMP() { }
+   MatrixInverse() { }
 
    /// Creates approximation of the inverse of square matrix
-   MatrixInverseMP(const MatrixMP<T> &mat)
-      : SolverMP<T>(mat.height, mat.width) { }
+   MatrixInverse(const Matrix &mat)
+      : Solver(mat.height, mat.width) { }
 };
 
-using MatrixInverse = MatrixInverseMP<real_t>;
-
 /// Abstract data type for sparse matrices
-template <class T>
-class AbstractSparseMatrixMP : public MatrixMP<T>
+class AbstractSparseMatrix : public Matrix
 {
 public:
    /// Creates a square matrix of the given size.
-   explicit AbstractSparseMatrixMP(int s = 0) : MatrixMP<T>(s) { }
+   explicit AbstractSparseMatrix(int s = 0) : Matrix(s) { }
 
    /// Creates a matrix of the given height and width.
-   explicit AbstractSparseMatrixMP(int h, int w) : MatrixMP<T>(h, w) { }
+   explicit AbstractSparseMatrix(int h, int w) : Matrix(h, w) { }
 
    /// Returns the number of non-zeros in a matrix
    virtual int NumNonZeroElems() const = 0;
@@ -99,33 +86,30 @@ public:
    /** Returns:
        - 0 if @a cols and @a srow are copies of the values in the matrix.
        - 1 if @a cols and @a srow are views of the values in the matrix. */
-   virtual int GetRow(const int row, Array<int> &cols,
-                      VectorMP<T> &srow) const = 0;
+   virtual int GetRow(const int row, Array<int> &cols, Vector &srow) const = 0;
 
    /** @brief If the matrix is square, this method will place 1 on the diagonal
        (i,i) if row i has "almost" zero l1-norm.
 
        If entry (i,i) does not belong to the sparsity pattern of A, then an
        error will occur. */
-   virtual void EliminateZeroRows(const T threshold = 1e-12) = 0;
+   virtual void EliminateZeroRows(const real_t threshold = 1e-12) = 0;
 
    /// Matrix-Vector Multiplication y = A*x
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override = 0;
+   void Mult(const Vector &x, Vector &y) const override = 0;
    /// Matrix-Vector Multiplication y = y + val*A*x
-   void AddMult(const VectorMP<T> &x, VectorMP<T> &y,
-                const T val = 1.) const override = 0;
+   void AddMult(const Vector &x, Vector &y,
+                const real_t val = 1.) const override = 0;
    /// MatrixTranspose-Vector Multiplication y = A'*x
-   void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override = 0;
+   void MultTranspose(const Vector &x, Vector &y) const override = 0;
    /// MatrixTranspose-Vector Multiplication y = y + val*A'*x
-   void AddMultTranspose(const VectorMP<T> &x, VectorMP<T> &y,
-                         const T val = 1.) const override = 0;
+   void AddMultTranspose(const Vector &x, Vector &y,
+                         const real_t val = 1.) const override = 0;
 
    /// Destroys AbstractSparseMatrix.
-   virtual ~AbstractSparseMatrixMP() { }
+   virtual ~AbstractSparseMatrix() { }
 };
 
-using AbstractSparseMatrix = AbstractSparseMatrixMP<real_t>;
-
 }
 
 #endif

linalg/mma.hpp

@@ -23,11 +23,7 @@
 namespace mfem
 {
 // forward declaration
-template <class T>
-class VectorMP;
-
-using Vector = VectorMP<real_t>;
-
+class Vector;
 
 /** \brief MMA (Method of Moving Asymptotes) solves a nonlinear optimization
  *         problem involving an objective function, inequality constraints,

linalg/ode.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "../general/communication.hpp"
+#include "../general/forall.hpp"
 #include "operator.hpp"
 #include "ode.hpp"
 
@@ -184,6 +185,23 @@ void ODESolver::Init(TimeDependentOperator &f_)
    mem_type = GetMemoryType(f_.GetMemoryClass());
 }
 
+void ODESolver::ComputeSlopeFromState(const real_t dt, const Vector &u,
+                                      Vector &k)
+{
+   // k currently holds state u(t+dt),
+   // convert to slope k = du/dt ~= (u(t+dt)-u(t))/dt
+   const int usz = u.Size();
+   real_t fac = 1.0/dt;
+   auto d_u = u.Read();
+   auto d_k = k.ReadWrite();
+
+   mfem::forall(usz, [=] MFEM_HOST_DEVICE (int i)
+   {
+      d_k[i] -= d_u[i];
+      d_k[i] *= fac;
+   });
+}
+
 void ForwardEulerSolver::Init(TimeDependentOperator &f_)
 {
    ODESolver::Init(f_);
@@ -629,6 +647,10 @@ void AdamsMoultonSolver::Step(Vector &x, real_t &t, real_t &dt)
       }
       state.ShiftStages();
       f->ImplicitSolve(a[0]*dt, x, state[0]);
+      if (f->ImplicitVarTypeIsState())
+      {
+         ComputeSlopeFromState(a[0]*dt, x, state[0]);
+      }
       x.Add(a[0]*dt, state[0]);
       t += dt;
    }
@@ -661,7 +683,15 @@ void BackwardEulerSolver::Step(Vector &x, real_t &t, real_t &dt)
 {
    f->SetTime(t + dt);
    f->ImplicitSolve(dt, x, k); // solve for k: k = f(x + dt*k, t + dt)
-   x.Add(dt, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      x = k; // x = u_{i+1}
+   }
+   else
+   {
+      x.Add(dt, k);
+   }
+
    t += dt;
 }
 
@@ -676,7 +706,16 @@ void ImplicitMidpointSolver::Step(Vector &x, real_t &t, real_t &dt)
 {
    f->SetTime(t + dt/2);
    f->ImplicitSolve(dt/2, x, k);
-   x.Add(dt, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      x.Neg();
+      x.Add(2.0, k);
+   }
+   else
+   {
+      x.Add(dt, k);
+   }
+
    t += dt;
 }
 
@@ -718,11 +757,19 @@ void SDIRK23Solver::Step(Vector &x, real_t &t, real_t &dt)
    // note: with gamma_opt=3, both solve are outside [t,t+dt] since a>1
    f->SetTime(t + gamma*dt);
    f->ImplicitSolve(gamma*dt, x, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(gamma*dt, x, k);
+   }
    add(x, (1.-2.*gamma)*dt, k, y); // y = x + (1-2*gamma)*dt*k
    x.Add(dt/2, k);
 
    f->SetTime(t + (1.-gamma)*dt);
    f->ImplicitSolve(gamma*dt, y, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(gamma*dt, y, k);
+   }
    x.Add(dt/2, k);
    t += dt;
 }
@@ -749,17 +796,29 @@ void SDIRK34Solver::Step(Vector &x, real_t &t, real_t &dt)
 
    f->SetTime(t + a*dt);
    f->ImplicitSolve(a*dt, x, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, x, k);
+   }
    add(x, (0.5-a)*dt, k, y);
    add(x,  (2.*a)*dt, k, z);
    x.Add(b*dt, k);
 
    f->SetTime(t + dt/2);
    f->ImplicitSolve(a*dt, y, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, y, k);
+   }
    z.Add((1.-4.*a)*dt, k);
    x.Add((1.-2.*b)*dt, k);
 
    f->SetTime(t + (1.-a)*dt);
    f->ImplicitSolve(a*dt, z, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, z, k);
+   }
    x.Add(b*dt, k);
    t += dt;
 }
@@ -785,15 +844,27 @@ void SDIRK33Solver::Step(Vector &x, real_t &t, real_t &dt)
 
    f->SetTime(t + a*dt);
    f->ImplicitSolve(a*dt, x, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, x, k);
+   }
    add(x, (c-a)*dt, k, y);
    x.Add(b*dt, k);
 
    f->SetTime(t + c*dt);
    f->ImplicitSolve(a*dt, y, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, y, k);
+   }
    x.Add((1.0-a-b)*dt, k);
 
    f->SetTime(t + dt);
    f->ImplicitSolve(a*dt, x, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, x, k);
+   }
    x.Add(a*dt, k);
    t += dt;
 }
@@ -818,6 +889,10 @@ void TrapezoidalRuleSolver::Step(Vector &x, real_t &t, real_t &dt)
 
    f->SetTime(t + dt);
    f->ImplicitSolve(dt/2.0, y, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(0.5*dt, y, k);
+   }
    x.Add(dt/2.0, k);
    t += dt;
 }
@@ -848,11 +923,19 @@ void ESDIRK32Solver::Step(Vector &x, real_t &t, real_t &dt)
 
    f->SetTime(t + (2.0*a)*dt);
    f->ImplicitSolve(a*dt, y, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, y, k);
+   }
    z.Add(b*dt, k);
    x.Add(b*dt, k);
 
    f->SetTime(t + dt);
    f->ImplicitSolve(a*dt, z, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, z, k);
+   }
    x.Add(a*dt, k);
    t += dt;
 }
@@ -885,11 +968,19 @@ void ESDIRK33Solver::Step(Vector &x, real_t &t, real_t &dt)
 
    f->SetTime(t + (2.0*a)*dt);
    f->ImplicitSolve(a*dt, y, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, y, k);
+   }
    z.Add(b*dt, k);
    x.Add(b_2*dt, k);
 
    f->SetTime(t + dt);
    f->ImplicitSolve(a*dt, z, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(a*dt, z, k);
+   }
    x.Add(b_3*dt, k);
    t += dt;
 }
@@ -955,6 +1046,10 @@ void GeneralizedAlphaSolver::Step(Vector &x, real_t &t, real_t &dt)
    real_t dt_eff = (gamma*alpha_f/alpha_m)*dt;
    f->SetTime(t + alpha_f*dt);
    f->ImplicitSolve(dt_eff, y, k);
+   if (f->ImplicitVarTypeIsState())
+   {
+      ComputeSlopeFromState(dt_eff, y, k);
+   }
 
    // Update x and xdot
    x.Add((1.0 - (gamma/alpha_m))*dt, state[0]);
@@ -1116,8 +1211,8 @@ void SecondOrderODESolver::EulerStep(Vector &x, Vector &dxdt, real_t &t,
    f->SetTime(t + dt);
    f->ImplicitSolve(0.5*dt*dt, dt, x, dxdt, state[0]);
 
-   x   .Add(0.5*dt*dt, state[0]);
-   dxdt.Add(dt,    state[0]);
+   x.Add(0.5*dt*dt, state[0]);
+   dxdt.Add(dt, state[0]);
    t += dt;
 }
 
@@ -1203,8 +1298,8 @@ void NewmarkSolver::Step(Vector &x, Vector &dxdt, real_t &t, real_t &dt)
    f->SetTime(t + dt);
    f->ImplicitSolve(fac3*dt*dt, fac4*dt, x, dxdt, state[0]);
 
-   x   .Add(fac3*dt*dt, state[0]);
-   dxdt.Add(fac4*dt,    state[0]);
+   x.Add(fac3*dt*dt, state[0]);
+   dxdt.Add(fac4*dt, state[0]);
    t += dt;
 }
 

linalg/ode.hpp

@@ -120,6 +120,7 @@ public:
 class ODESolver
 {
 protected:
+   using ImplicitVariableType = TimeDependentOperator::ImplicitVariableType;
    /// Pointer to the associated TimeDependentOperator.
    TimeDependentOperator *f;  // f(.,t) : R^n --> R^n
    MemoryType mem_type;
@@ -192,6 +193,22 @@ public:
    /// Returns how many State vectors the ODE requires
    virtual int GetStateSize() { return 0; };
 
+   ///@brief Returns @a true if the ODESolver supports the given
+   /// #ImplicitVariableType, @a var, and returns @a false otherwise.
+   ///@note Should be overriden in ODESolver that calls TimeDependentOperator::ImplicitSolve().
+   virtual bool SupportsImplicitVariableType(ImplicitVariableType var) const
+   { return false; };
+
+   /** @brief Compute the finite-difference slope, @a $\frac{du}{dt} \approx \frac{u(t+dt)-u(t)}{dt}$,
+    * and store it in @a k.
+    * @param [in] dt Finite difference step size.
+    * @param [in] u  state vector, @a u(t).
+    * @param [in,out] k   On input, @a k contains the state vector, @a u( @a t+ @a dt).
+    * On output, @a k contains the computed slope, @a du/dt.
+    * */
+   virtual void ComputeSlopeFromState(const real_t dt, const Vector &u,
+                                      Vector &k);
+
    // Help info for ODESolver options
    static MFEM_EXPORT std::string ExplicitTypes;
    static MFEM_EXPORT std::string ImplicitTypes;
@@ -361,6 +378,12 @@ public:
    void Init(TimeDependentOperator &f_) override;
 
    void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -374,6 +397,12 @@ public:
    void Init(TimeDependentOperator &f_) override;
 
    void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -395,6 +424,12 @@ public:
    void Init(TimeDependentOperator &f_) override;
 
    void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -409,6 +444,12 @@ public:
    void Init(TimeDependentOperator &f_) override;
 
    void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -423,6 +464,12 @@ public:
    void Init(TimeDependentOperator &f_) override;
 
    void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -437,6 +484,12 @@ public:
    void Init(TimeDependentOperator &f_) override;
 
    void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -451,6 +504,12 @@ public:
    void Init(TimeDependentOperator &f_) override;
 
    void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -465,6 +524,12 @@ public:
    void Init(TimeDependentOperator &f_) override;
 
    void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -490,6 +555,12 @@ public:
 
    ODEStateData& GetState() override { return state; }
    const ODEStateData& GetState() const override { return state; }
+
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 
@@ -606,6 +677,11 @@ public:
 
    ODEStateData& GetState() override { return state; }
    const ODEStateData& GetState() const override { return state; }
+   bool SupportsImplicitVariableType(ImplicitVariableType var) const override
+   {
+      return (var == ImplicitVariableType::STATE ||
+              var == ImplicitVariableType::SLOPE);
+   }
 };
 
 /** A 1-stage, 2nd order AM method. */

linalg/operator.cpp

@@ -19,12 +19,10 @@
 namespace mfem
 {
 
-template <class T>
-void OperatorMP<T>::InitTVectors(const OperatorMP<T> *Po,
-                                 const OperatorMP<T> *Ri,
-                                 const OperatorMP<T> *Pi,
-                                 VectorMP<T> &x, VectorMP<T> &b,
-                                 VectorMP<T> &X, VectorMP<T> &B) const
+void Operator::InitTVectors(const Operator *Po, const Operator *Ri,
+                            const Operator *Pi,
+                            Vector &x, Vector &b,
+                            Vector &X, Vector &B) const
 {
    if (!IsIdentityProlongation(Po))
    {
@@ -50,27 +48,23 @@ void OperatorMP<T>::InitTVectors(const OperatorMP<T> *Po,
    }
 }
 
-template <class T>
-void OperatorMP<T>::AddMult(const VectorMP<T> &x, VectorMP<T> &y,
-                            const T a) const
+void Operator::AddMult(const Vector &x, Vector &y, const real_t a) const
 {
-   mfem::VectorMP<T> z(y.Size());
+   mfem::Vector z(y.Size());
    Mult(x, z);
    y.Add(a, z);
 }
 
-template <class T>
-void OperatorMP<T>::AddMultTranspose(const VectorMP<T> &x, VectorMP<T> &y,
-                                     const T a) const
+void Operator::AddMultTranspose(const Vector &x, Vector &y,
+                                const real_t a) const
 {
-   mfem::VectorMP<T> z(y.Size());
+   mfem::Vector z(y.Size());
    MultTranspose(x, z);
    y.Add(a, z);
 }
 
-template <class T>
-void OperatorMP<T>::ArrayMult(const Array<const VectorMP<T> *> &X,
-                              Array<VectorMP<T> *> &Y) const
+void Operator::ArrayMult(const Array<const Vector *> &X,
+                         Array<Vector *> &Y) const
 {
    MFEM_ASSERT(X.Size() == Y.Size(),
                "Number of columns mismatch in Operator::Mult!");
@@ -81,9 +75,8 @@ void OperatorMP<T>::ArrayMult(const Array<const VectorMP<T> *> &X,
    }
 }
 
-template <class T>
-void OperatorMP<T>::ArrayMultTranspose(const Array<const VectorMP<T> *> &X,
-                                       Array<VectorMP<T> *> &Y) const
+void Operator::ArrayMultTranspose(const Array<const Vector *> &X,
+                                  Array<Vector *> &Y) const
 {
    MFEM_ASSERT(X.Size() == Y.Size(),
                "Number of columns mismatch in Operator::MultTranspose!");
@@ -94,10 +87,8 @@ void OperatorMP<T>::ArrayMultTranspose(const Array<const VectorMP<T> *> &X,
    }
 }
 
-template <class T>
-void OperatorMP<T>::ArrayAddMult(const Array<const VectorMP<T> *> &X,
-                                 Array<VectorMP<T> *> &Y,
-                                 const T a) const
+void Operator::ArrayAddMult(const Array<const Vector *> &X, Array<Vector *> &Y,
+                            const real_t a) const
 {
    MFEM_ASSERT(X.Size() == Y.Size(),
                "Number of columns mismatch in Operator::AddMult!");
@@ -108,9 +99,8 @@ void OperatorMP<T>::ArrayAddMult(const Array<const VectorMP<T> *> &X,
    }
 }
 
-template <class T>
-void OperatorMP<T>::ArrayAddMultTranspose(const Array<const VectorMP<T> *> &X,
-                                          Array<VectorMP<T> *> &Y, const T a) const
+void Operator::ArrayAddMultTranspose(const Array<const Vector *> &X,
+                                     Array<Vector *> &Y, const real_t a) const
 {
    MFEM_ASSERT(X.Size() == Y.Size(),
                "Number of columns mismatch in Operator::AddMultTranspose!");
@@ -121,48 +111,44 @@ void OperatorMP<T>::ArrayAddMultTranspose(const Array<const VectorMP<T> *> &X,
    }
 }
 
-template <class T>
-void OperatorMP<T>::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                     VectorMP<T> &x, VectorMP<T> &b,
-                                     OperatorMP<T>* &Aout, VectorMP<T> &X, VectorMP<T> &B,
-                                     int copy_interior)
+void Operator::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                Vector &x, Vector &b,
+                                Operator* &Aout, Vector &X, Vector &B,
+                                int copy_interior)
 {
-   const OperatorMP<T> *P = this->GetProlongation();
-   const OperatorMP<T> *R = this->GetRestriction();
+   const Operator *P = this->GetProlongation();
+   const Operator *R = this->GetRestriction();
    InitTVectors(P, R, P, x, b, X, B);
 
    if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
 
-   ConstrainedOperatorMP<T> *constrainedA;
+   ConstrainedOperator *constrainedA;
    FormConstrainedSystemOperator(ess_tdof_list, constrainedA);
    constrainedA->EliminateRHS(X, B);
    Aout = constrainedA;
 }
 
-template <class T>
-void OperatorMP<T>::FormRectangularLinearSystem(
+void Operator::FormRectangularLinearSystem(
    const Array<int> &trial_tdof_list,
-   const Array<int> &test_tdof_list, VectorMP<T> &x, VectorMP<T> &b,
-   OperatorMP<T>* &Aout, VectorMP<T> &X, VectorMP<T> &B)
+   const Array<int> &test_tdof_list, Vector &x, Vector &b,
+   Operator* &Aout, Vector &X, Vector &B)
 {
-   const OperatorMP<T> *Pi = this->GetProlongation();
-   const OperatorMP<T> *Po = this->GetOutputProlongation();
-   const OperatorMP<T> *Ri = this->GetRestriction();
+   const Operator *Pi = this->GetProlongation();
+   const Operator *Po = this->GetOutputProlongation();
+   const Operator *Ri = this->GetRestriction();
    InitTVectors(Po, Ri, Pi, x, b, X, B);
 
-   RectangularConstrainedOperatorMP<T> *constrainedA;
+   RectangularConstrainedOperator *constrainedA;
    FormRectangularConstrainedSystemOperator(trial_tdof_list, test_tdof_list,
                                             constrainedA);
    constrainedA->EliminateRHS(X, B);
    Aout = constrainedA;
 }
 
-template <class T>
-void OperatorMP<T>::RecoverFEMSolution(const VectorMP<T> &X,
-                                       const VectorMP<T> &b, VectorMP<T> &x)
+void Operator::RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x)
 {
    // Same for Rectangular and Square operators
-   const OperatorMP<T> *P = this->GetProlongation();
+   const Operator *P = this->GetProlongation();
    if (!IsIdentityProlongation(P))
    {
       // Apply conforming prolongation
@@ -179,28 +165,26 @@ void OperatorMP<T>::RecoverFEMSolution(const VectorMP<T> &X,
    }
 }
 
-template <class T>
-OperatorMP<T> * OperatorMP<T>::SetupRAP(const OperatorMP<T> *Pi,
-                                        const OperatorMP<T> *Po)
+Operator * Operator::SetupRAP(const Operator *Pi, const Operator *Po)
 {
-   OperatorMP<T> *rap;
+   Operator *rap;
    if (!IsIdentityProlongation(Pi))
    {
       if (!IsIdentityProlongation(Po))
       {
-         rap = new RAPOperatorMP<T>(*Po, *this, *Pi);
+         rap = new RAPOperator(*Po, *this, *Pi);
       }
       else
       {
-         rap = new ProductOperatorMP<T>(this, Pi, false, false);
+         rap = new ProductOperator(this, Pi, false,false);
       }
    }
    else
    {
       if (!IsIdentityProlongation(Po))
       {
-         TransposeOperatorMP<T> * PoT = new TransposeOperatorMP<T>(Po);
-         rap = new ProductOperatorMP<T>(PoT, this, true, false);
+         TransposeOperator * PoT = new TransposeOperator(Po);
+         rap = new ProductOperator(PoT, this, true,false);
       }
       else
       {
@@ -210,74 +194,67 @@ OperatorMP<T> * OperatorMP<T>::SetupRAP(const OperatorMP<T> *Pi,
    return rap;
 }
 
-template <class T>
-void OperatorMP<T>::FormConstrainedSystemOperator(
-   const Array<int> &ess_tdof_list, ConstrainedOperatorMP<T>* &Aout)
+void Operator::FormConstrainedSystemOperator(
+   const Array<int> &ess_tdof_list, ConstrainedOperator* &Aout)
 {
-   const OperatorMP<T> *P = this->GetProlongation();
-   OperatorMP<T> *rap = SetupRAP(P, P);
+   const Operator *P = this->GetProlongation();
+   Operator *rap = SetupRAP(P, P);
 
    // Impose the boundary conditions through a ConstrainedOperator, which owns
    // the rap operator when P and R are non-trivial
-   ConstrainedOperatorMP<T> *A = new ConstrainedOperatorMP<T>(rap, ess_tdof_list,
-                                                              rap != this);
+   ConstrainedOperator *A = new ConstrainedOperator(rap, ess_tdof_list,
+                                                    rap != this);
    Aout = A;
 }
 
-template <class T>
-void OperatorMP<T>::FormRectangularConstrainedSystemOperator(
+void Operator::FormRectangularConstrainedSystemOperator(
    const Array<int> &trial_tdof_list, const Array<int> &test_tdof_list,
-   RectangularConstrainedOperatorMP<T>* &Aout)
+   RectangularConstrainedOperator* &Aout)
 {
-   const OperatorMP<T> *Pi = this->GetProlongation();
-   const OperatorMP<T> *Po = this->GetOutputProlongation();
-   OperatorMP<T> *rap = SetupRAP(Pi, Po);
+   const Operator *Pi = this->GetProlongation();
+   const Operator *Po = this->GetOutputProlongation();
+   Operator *rap = SetupRAP(Pi, Po);
 
    // Impose the boundary conditions through a RectangularConstrainedOperator,
    // which owns the rap operator when P and R are non-trivial
-   RectangularConstrainedOperatorMP<T> *A
-      = new RectangularConstrainedOperatorMP<T>(rap,
-                                                trial_tdof_list, test_tdof_list,
-                                                rap != this);
+   RectangularConstrainedOperator *A
+      = new RectangularConstrainedOperator(rap,
+                                           trial_tdof_list, test_tdof_list,
+                                           rap != this);
    Aout = A;
 }
 
-template <class T>
-void OperatorMP<T>::FormSystemOperator(const Array<int> &ess_tdof_list,
-                                       OperatorMP<T>* &Aout)
+void Operator::FormSystemOperator(const Array<int> &ess_tdof_list,
+                                  Operator* &Aout)
 {
-   ConstrainedOperatorMP<T> *A;
+   ConstrainedOperator *A;
    FormConstrainedSystemOperator(ess_tdof_list, A);
    Aout = A;
 }
 
-template <class T>
-void OperatorMP<T>::FormRectangularSystemOperator(const Array<int>
-                                                  &trial_tdof_list,
-                                                  const Array<int> &test_tdof_list,
-                                                  OperatorMP<T>* &Aout)
+void Operator::FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
+                                             const Array<int> &test_tdof_list,
+                                             Operator* &Aout)
 {
-   RectangularConstrainedOperatorMP<T> *A;
+   RectangularConstrainedOperator *A;
    FormRectangularConstrainedSystemOperator(trial_tdof_list, test_tdof_list, A);
    Aout = A;
 }
 
-template <class T>
-void OperatorMP<T>::FormDiscreteOperator(OperatorMP<T>* &Aout)
+void Operator::FormDiscreteOperator(Operator* &Aout)
 {
-   const OperatorMP<T> *Pin  = this->GetProlongation();
-   const OperatorMP<T> *Rout = this->GetOutputRestriction();
-   Aout = new TripleProductOperatorMP<T>(Rout, this, Pin, false, false, false);
+   const Operator *Pin  = this->GetProlongation();
+   const Operator *Rout = this->GetOutputRestriction();
+   Aout = new TripleProductOperator(Rout, this, Pin,false, false, false);
 }
 
-template <class T>
-void OperatorMP<T>::PrintMatlab(std::ostream & os, int n, int m) const
+void Operator::PrintMatlab(std::ostream & os, int n, int m) const
 {
    using namespace std;
    if (n == 0) { n = width; }
    if (m == 0) { m = height; }
 
-   VectorMP<T> x(n), y(m);
+   Vector x(n), y(m);
    x = 0.0;
 
    os << setiosflags(ios::scientific | ios::showpos);
@@ -296,8 +273,7 @@ void OperatorMP<T>::PrintMatlab(std::ostream & os, int n, int m) const
    }
 }
 
-template <class T>
-void OperatorMP<T>::PrintMatlab(std::ostream &os) const
+void Operator::PrintMatlab(std::ostream &os) const
 {
    PrintMatlab(os, width, height);
 }
@@ -428,11 +404,9 @@ SumOperator::~SumOperator()
    if (ownB) { delete B; }
 }
 
-template <class T>
-ProductOperatorMP<T>::ProductOperatorMP(const OperatorMP<T> *A,
-                                        const OperatorMP<T> *B,
-                                        bool ownA, bool ownB)
-   : OperatorMP<T>(A->Height(), B->Width()),
+ProductOperator::ProductOperator(const Operator *A, const Operator *B,
+                                 bool ownA, bool ownB)
+   : Operator(A->Height(), B->Width()),
      A(A), B(B), ownA(ownA), ownB(ownB), z(A->Width())
 {
    MFEM_VERIFY(A->Width() == B->Height(),
@@ -449,18 +423,16 @@ ProductOperatorMP<T>::ProductOperatorMP(const OperatorMP<T> *A,
    }
 }
 
-template <class T>
-ProductOperatorMP<T>::~ProductOperatorMP()
+ProductOperator::~ProductOperator()
 {
    if (ownA) { delete A; }
    if (ownB) { delete B; }
 }
 
-template <class T>
-RAPOperatorMP<T>::RAPOperatorMP(const OperatorMP<T> &Rt_,
-                                const OperatorMP<T> &A_,
-                                const OperatorMP<T> &P_)
-   : OperatorMP<T>(Rt_.Width(), P_.Width()), Rt(Rt_), A(A_), P(P_)
+
+RAPOperator::RAPOperator(const Operator &Rt_, const Operator &A_,
+                         const Operator &P_)
+   : Operator(Rt_.Width(), P_.Width()), Rt(Rt_), A(A_), P(P_)
 {
    MFEM_VERIFY(Rt.Height() == A.Height(),
                "incompatible Operators: Rt.Height() = " << Rt.Height()
@@ -491,11 +463,11 @@ RAPOperatorMP<T>::RAPOperatorMP(const OperatorMP<T> &Rt_,
    APx.SetSize(A.Height(), mem_type);
 }
 
-template <class T>
-TripleProductOperatorMP<T>::TripleProductOperatorMP(
-   const OperatorMP<T> *A, const OperatorMP<T> *B, const OperatorMP<T> *C,
+
+TripleProductOperator::TripleProductOperator(
+   const Operator *A, const Operator *B, const Operator *C,
    bool ownA, bool ownB, bool ownC)
-   : OperatorMP<T>(A->Height(), C->Width())
+   : Operator(A->Height(), C->Width())
    , A(A), B(B), C(C)
    , ownA(ownA), ownB(ownB), ownC(ownC)
 {
@@ -528,20 +500,18 @@ TripleProductOperatorMP<T>::TripleProductOperatorMP(
    t2.SetSize(B->Height(), mem_type);
 }
 
-template <class T>
-TripleProductOperatorMP<T>::~TripleProductOperatorMP()
+TripleProductOperator::~TripleProductOperator()
 {
    if (ownA) { delete A; }
    if (ownB) { delete B; }
    if (ownC) { delete C; }
 }
 
-template <class T>
-ConstrainedOperatorMP<T>::ConstrainedOperatorMP(OperatorMP<T> *A,
-                                                const Array<int> &list,
-                                                bool own_A_,
-                                                DiagonalPolicy diag_policy_)
-   : OperatorMP<T>(A->Height(), A->Width()), A(A), own_A(own_A_),
+
+ConstrainedOperator::ConstrainedOperator(Operator *A, const Array<int> &list,
+                                         bool own_A_,
+                                         DiagonalPolicy diag_policy_)
+   : Operator(A->Height(), A->Width()), A(A), own_A(own_A_),
      diag_policy(diag_policy_)
 {
    // 'mem_class' should work with A->Mult() and mfem::forall():
@@ -551,12 +521,11 @@ ConstrainedOperatorMP<T>::ConstrainedOperatorMP(OperatorMP<T> *A,
    constraint_list.MakeRef(list);
    // typically z and w are large vectors, so use the device (GPU) to perform
    // operations on them
-   z.SetSize(this->height, mem_type); z.UseDevice(true);
-   w.SetSize(this->height, mem_type); w.UseDevice(true);
+   z.SetSize(height, mem_type); z.UseDevice(true);
+   w.SetSize(height, mem_type); w.UseDevice(true);
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::AssembleDiagonal(VectorMP<T> &diag) const
+void ConstrainedOperator::AssembleDiagonal(Vector &diag) const
 {
    A->AssembleDiagonal(diag);
 
@@ -587,9 +556,7 @@ void ConstrainedOperatorMP<T>::AssembleDiagonal(VectorMP<T> &diag) const
    }
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::EliminateRHS(const VectorMP<T> &x,
-                                            VectorMP<T> &b) const
+void ConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
 {
    w = 0.0;
    const int csz = constraint_list.Size();
@@ -616,10 +583,8 @@ void ConstrainedOperatorMP<T>::EliminateRHS(const VectorMP<T> &x,
    });
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::ConstrainedMult(const VectorMP<T> &x,
-                                               VectorMP<T> &y,
-                                               const bool transpose) const
+void ConstrainedOperator::ConstrainedMult(const Vector &x, Vector &y,
+                                          const bool transpose) const
 {
    const int csz = constraint_list.Size();
    if (csz == 0)
@@ -680,10 +645,8 @@ void ConstrainedOperatorMP<T>::ConstrainedMult(const VectorMP<T> &x,
    }
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::ConstrainedAbsMult(const VectorMP<T> &x,
-                                                  VectorMP<T> &y,
-                                                  const bool transpose) const
+void ConstrainedOperator::ConstrainedAbsMult(const Vector &x, Vector &y,
+                                             const bool transpose) const
 {
    const int csz = constraint_list.Size();
    if (csz == 0)
@@ -744,52 +707,43 @@ void ConstrainedOperatorMP<T>::ConstrainedAbsMult(const VectorMP<T> &x,
    }
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::Mult(const VectorMP<T> &x, VectorMP<T> &y) const
+void ConstrainedOperator::Mult(const Vector &x, Vector &y) const
 {
    constexpr bool transpose = false;
    ConstrainedMult(x, y, transpose);
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::AbsMult(const VectorMP<T> &x,
-                                       VectorMP<T> &y) const
+void ConstrainedOperator::AbsMult(const Vector &x, Vector &y) const
 {
    constexpr bool transpose = false;
    ConstrainedAbsMult(x, y, transpose);
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::MultTranspose(const VectorMP<T> &x,
-                                             VectorMP<T> &y) const
+void ConstrainedOperator::MultTranspose(const Vector &x, Vector &y) const
 {
    constexpr bool transpose = true;
    ConstrainedMult(x, y, transpose);
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::AbsMultTranspose(const VectorMP<T> &x,
-                                                VectorMP<T> &y) const
+void ConstrainedOperator::AbsMultTranspose(const Vector &x, Vector &y) const
 {
    constexpr bool transpose = true;
    ConstrainedAbsMult(x, y, transpose);
 }
 
-template <class T>
-void ConstrainedOperatorMP<T>::AddMult(const VectorMP<T> &x, VectorMP<T> &y,
-                                       const T a) const
+void ConstrainedOperator::AddMult(const Vector &x, Vector &y,
+                                  const real_t a) const
 {
    Mult(x, w);
    y.Add(a, w);
 }
 
-template <class T>
-RectangularConstrainedOperatorMP<T>::RectangularConstrainedOperatorMP(
-   OperatorMP<T> *A,
+RectangularConstrainedOperator::RectangularConstrainedOperator(
+   Operator *A,
    const Array<int> &trial_list,
    const Array<int> &test_list,
    bool own_A_)
-   : OperatorMP<T>(A->Height(), A->Width()), A(A), own_A(own_A_)
+   : Operator(A->Height(), A->Width()), A(A), own_A(own_A_)
 {
    // 'mem_class' should work with A->Mult() and mfem::forall():
    mem_class = A->GetMemoryClass()*Device::GetMemoryClass();
@@ -799,13 +753,12 @@ RectangularConstrainedOperatorMP<T>::RectangularConstrainedOperatorMP(
    trial_constraints.MakeRef(trial_list);
    test_constraints.MakeRef(test_list);
    // typically z and w are large vectors, so store them on the device
-   z.SetSize(this->height, mem_type); z.UseDevice(true);
-   w.SetSize(this->width, mem_type); w.UseDevice(true);
+   z.SetSize(height, mem_type); z.UseDevice(true);
+   w.SetSize(width, mem_type); w.UseDevice(true);
 }
 
-template <class T>
-void RectangularConstrainedOperatorMP<T>::EliminateRHS(const VectorMP<T> &x,
-                                                       VectorMP<T> &b) const
+void RectangularConstrainedOperator::EliminateRHS(const Vector &x,
+                                                  Vector &b) const
 {
    w = 0.0;
    const int trial_csz = trial_constraints.Size();
@@ -830,9 +783,7 @@ void RectangularConstrainedOperatorMP<T>::EliminateRHS(const VectorMP<T> &x,
    });
 }
 
-template <class T>
-void RectangularConstrainedOperatorMP<T>::Mult(const VectorMP<T> &x,
-                                               VectorMP<T> &y) const
+void RectangularConstrainedOperator::Mult(const Vector &x, Vector &y) const
 {
    const int trial_csz = trial_constraints.Size();
    const int test_csz = test_constraints.Size();
@@ -866,9 +817,8 @@ void RectangularConstrainedOperatorMP<T>::Mult(const VectorMP<T> &x,
    }
 }
 
-template <class T>
-void RectangularConstrainedOperatorMP<T>::MultTranspose(const VectorMP<T> &x,
-                                                        VectorMP<T> &y) const
+void RectangularConstrainedOperator::MultTranspose(const Vector &x,
+                                                   Vector &y) const
 {
    const int trial_csz = trial_constraints.Size();
    const int test_csz = test_constraints.Size();
@@ -902,9 +852,7 @@ void RectangularConstrainedOperatorMP<T>::MultTranspose(const VectorMP<T> &x,
    }
 }
 
-template <class T>
-T InnerProductOperatorMP<T>::Dot(const VectorMP<T> &x,
-                                 const VectorMP<T> &y) const
+real_t InnerProductOperator::Dot(const Vector &x, const Vector &y) const
 {
 #ifndef MFEM_USE_MPI
    return (x * y);
@@ -979,24 +927,4 @@ real_t PowerMethod::EstimateLargestEigenvalue(Operator& opr, Vector& v0,
    return eigenvalue;
 }
 
-template class OperatorMP<float>;
-template class OperatorMP<double>;
-
-template class ConstrainedOperatorMP<float>;
-template class ConstrainedOperatorMP<double>;
-
-template class RectangularConstrainedOperatorMP<float>;
-template class RectangularConstrainedOperatorMP<double>;
-
-template class RAPOperatorMP<float>;
-template class RAPOperatorMP<double>;
-
-template class ProductOperatorMP<float>;
-template class ProductOperatorMP<double>;
-
-template class TripleProductOperatorMP<float>;
-template class TripleProductOperatorMP<double>;
-
-template class InnerProductOperatorMP<float>;
-template class InnerProductOperatorMP<double>;
 }

linalg/operator.hpp

@@ -17,44 +17,32 @@
 namespace mfem
 {
 
-template <class T>
-class ConstrainedOperatorMP;
+class ConstrainedOperator;
+class RectangularConstrainedOperator;
 
-template <class T>
-class RectangularConstrainedOperatorMP;
-
-class OperatorBase
+/// Abstract operator
+class Operator
 {
 protected:
    int height; ///< Dimension of the output / number of rows in the matrix.
    int width;  ///< Dimension of the input / number of columns in the matrix.
 
+   /// see FormSystemOperator()
+   /** @note Uses DiagonalPolicy::DIAG_ONE. */
+   void FormConstrainedSystemOperator(
+      const Array<int> &ess_tdof_list, ConstrainedOperator* &Aout);
+
+   /// see FormRectangularSystemOperator()
+   void FormRectangularConstrainedSystemOperator(
+      const Array<int> &trial_tdof_list,
+      const Array<int> &test_tdof_list,
+      RectangularConstrainedOperator* &Aout);
+
+   /** @brief Returns RAP Operator of this, using input/output Prolongation matrices
+       @a Pi corresponds to "P", @a Po corresponds to "Rt" */
+   Operator *SetupRAP(const Operator *Pi, const Operator *Po);
+
 public:
-   /// Get the height (size of output) of the Operator. Synonym with NumRows().
-   inline int Height() const { return height; }
-
-   /// Get the width (size of input) of the Operator. Synonym with NumCols().
-   inline int Width() const { return width; }
-
-   enum Type
-   {
-      ANY_TYPE,         ///< ID for the base class Operator, i.e. any type.
-      MFEM_SPARSEMAT,   ///< ID for class SparseMatrix.
-      Hypre_ParCSR,     ///< ID for class HypreParMatrix.
-      PETSC_MATAIJ,     ///< ID for class PetscParMatrix, MATAIJ format.
-      PETSC_MATIS,      ///< ID for class PetscParMatrix, MATIS format.
-      PETSC_MATSHELL,   ///< ID for class PetscParMatrix, MATSHELL format.
-      PETSC_MATNEST,    ///< ID for class PetscParMatrix, MATNEST format.
-      PETSC_MATHYPRE,   ///< ID for class PetscParMatrix, MATHYPRE format.
-      PETSC_MATGENERIC, ///< ID for class PetscParMatrix, unspecified format.
-      Complex_Operator, ///< ID for class ComplexOperator.
-      MFEM_ComplexSparseMat, ///< ID for class ComplexSparseMatrix.
-      Complex_Hypre_ParCSR,   ///< ID for class ComplexHypreParMatrix.
-      Complex_DenseMat,  ///< ID for class ComplexDenseMatrix
-      MFEM_Block_Matrix,     ///< ID for class BlockMatrix.
-      MFEM_Block_Operator   ///< ID for the base class BlockOperator.
-   };
-
    /// Defines operator diagonal policy upon elimination of rows and/or columns.
    enum DiagonalPolicy
    {
@@ -62,45 +50,26 @@ public:
       DIAG_ONE,  ///< Set the diagonal value to one
       DIAG_KEEP  ///< Keep the diagonal value
    };
-};
 
-/// Abstract operator
-template <class T>
-class OperatorMP : public OperatorBase
-{
-protected:
-   /// see FormSystemOperator()
-   /** @note Uses DiagonalPolicy::DIAG_ONE. */
-   void FormConstrainedSystemOperator(
-      const Array<int> &ess_tdof_list, ConstrainedOperatorMP<T>* &Aout);
-
-   /// see FormRectangularSystemOperator()
-   void FormRectangularConstrainedSystemOperator(
-      const Array<int> &trial_tdof_list,
-      const Array<int> &test_tdof_list,
-      RectangularConstrainedOperatorMP<T>* &Aout);
-
-   /** @brief Returns RAP Operator of this, using input/output Prolongation matrices
-       @a Pi corresponds to "P", @a Po corresponds to "Rt" */
-   OperatorMP *SetupRAP(const OperatorMP<T> *Pi, const OperatorMP<T> *Po);
-
-public:
    /// Initializes memory for true vectors of linear system
-   void InitTVectors(const OperatorMP<T> *Po, const OperatorMP<T> *Ri,
-                     const OperatorMP<T> *Pi,
-                     VectorMP<T> &x, VectorMP<T> &b, VectorMP<T> &X, VectorMP<T> &B) const;
+   void InitTVectors(const Operator *Po, const Operator *Ri, const Operator *Pi,
+                     Vector &x, Vector &b, Vector &X, Vector &B) const;
 
    /// Construct a square Operator with given size s (default 0).
-   explicit OperatorMP(int s = 0) { height = width = s; }
+   explicit Operator(int s = 0) { height = width = s; }
 
    /** @brief Construct an Operator with the given height (output size) and
        width (input size). */
-   OperatorMP(int h, int w) { height = h; width = w; }
+   Operator(int h, int w) { height = h; width = w; }
 
+   /// Get the height (size of output) of the Operator. Synonym with NumRows().
+   inline int Height() const { return height; }
    /** @brief Get the number of rows (size of output) of the Operator. Synonym
        with Height(). */
    inline int NumRows() const { return height; }
 
+   /// Get the width (size of input) of the Operator. Synonym with NumCols().
+   inline int Width() const { return width; }
    /** @brief Get the number of columns (size of input) of the Operator. Synonym
        with Width(). */
    inline int NumCols() const { return width; }
@@ -117,63 +86,61 @@ public:
    virtual MemoryClass GetMemoryClass() const { return MemoryClass::HOST; }
 
    /// Operator application: `y=A(x)`.
-   virtual void Mult(const VectorMP<T> &x, VectorMP<T> &y) const = 0;
+   virtual void Mult(const Vector &x, Vector &y) const = 0;
 
    /** @brief Action of the absolute-value operator: `y=|A|(x)`. The default
      behavior in class Operator is to generate an error. If the Operator is a
      composition of several operators, the composition unfold into a product
      of absolute-value operators too. */
-   virtual void AbsMult(const VectorMP<T> &x, VectorMP<T> &y) const
+   virtual void AbsMult(const Vector &x, Vector &y) const
    { MFEM_ABORT("Operator::AbsMult() is not overridden!"); }
 
    /** @brief Action of the transpose operator: `y=A^t(x)`. The default behavior
        in class Operator is to generate an error. */
-   virtual void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const
+   virtual void MultTranspose(const Vector &x, Vector &y) const
    { MFEM_ABORT("Operator::MultTranspose() is not overridden!"); }
 
    /** @brief Action of the transpose absolute-value operator: `y=|A|^t(x)`.
       The default behavior in class Operator is to generate an error. */
-   virtual void AbsMultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const
+   virtual void AbsMultTranspose(const Vector &x, Vector &y) const
    { MFEM_ABORT("Operator::AbsMultTranspose() is not overridden!"); }
 
    /// Operator application: `y+=A(x)` (default) or `y+=a*A(x)`.
-   virtual void AddMult(const VectorMP<T> &x, VectorMP<T> &y,
-                        const T a = 1.0) const;
+   virtual void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const;
 
    /// Operator transpose application: `y+=A^t(x)` (default) or `y+=a*A^t(x)`.
-   virtual void AddMultTranspose(const VectorMP<T> &x, VectorMP<T> &y,
-                                 const T a = 1.0) const;
+   virtual void AddMultTranspose(const Vector &x, Vector &y,
+                                 const real_t a = 1.0) const;
 
    /// Operator application on a matrix: `Y=A(X)`.
-   virtual void ArrayMult(const Array<const VectorMP<T> *> &X,
-                          Array<VectorMP<T> *> &Y) const;
+   virtual void ArrayMult(const Array<const Vector *> &X,
+                          Array<Vector *> &Y) const;
 
    /// Action of the transpose operator on a matrix: `Y=A^t(X)`.
-   virtual void ArrayMultTranspose(const Array<const VectorMP<T> *> &X,
-                                   Array<VectorMP<T> *> &Y) const;
+   virtual void ArrayMultTranspose(const Array<const Vector *> &X,
+                                   Array<Vector *> &Y) const;
 
    /// Operator application on a matrix: `Y+=A(X)` (default) or `Y+=a*A(X)`.
-   virtual void ArrayAddMult(const Array<const VectorMP<T> *> &X,
-                             Array<VectorMP<T> *> &Y,
-                             const T a = 1.0) const;
+   virtual void ArrayAddMult(const Array<const Vector *> &X, Array<Vector *> &Y,
+                             const real_t a = 1.0) const;
 
    /** @brief Operator transpose application on a matrix: `Y+=A^t(X)` (default)
        or `Y+=a*A^t(X)`. */
-   virtual void ArrayAddMultTranspose(const Array<const VectorMP<T> *> &X,
-                                      Array<VectorMP<T> *> &Y, const T a = 1.0) const;
+   virtual void ArrayAddMultTranspose(const Array<const Vector *> &X,
+                                      Array<Vector *> &Y, const real_t a = 1.0) const;
 
    /** @brief Evaluate the gradient operator at the point @a x. The default
        behavior in class Operator is to generate an error. */
-   virtual OperatorMP<T> &GetGradient(const VectorMP<T> &x) const
+   virtual Operator &GetGradient(const Vector &x) const
    {
       MFEM_ABORT("Operator::GetGradient() is not overridden!");
-      return const_cast<OperatorMP<T> &>(*this);
+      return const_cast<Operator &>(*this);
    }
 
    /** @brief Computes the diagonal entries into @a diag. Typically, this
        operation only makes sense for linear Operator%s. In some cases, only an
        approximation of the diagonal is computed. */
-   virtual void AssembleDiagonal(VectorMP<T> &diag) const
+   virtual void AssembleDiagonal(Vector &diag) const
    {
       MFEM_CONTRACT_VAR(diag);
       MFEM_ABORT("Not relevant or not implemented for this Operator.");
@@ -181,15 +148,15 @@ public:
 
    /** @brief Prolongation operator from linear algebra (linear system) vectors,
        to input vectors for the operator. `NULL` means identity. */
-   virtual const OperatorMP<T> *GetProlongation() const { return NULL; }
+   virtual const Operator *GetProlongation() const { return NULL; }
 
    /** @brief Restriction operator from input vectors for the operator to linear
        algebra (linear system) vectors. `NULL` means identity. */
-   virtual const OperatorMP<T> *GetRestriction() const  { return NULL; }
+   virtual const Operator *GetRestriction() const  { return NULL; }
 
    /** @brief Prolongation operator from linear algebra (linear system) vectors,
        to output vectors for the operator. `NULL` means identity. */
-   virtual const OperatorMP<T> *GetOutputProlongation() const
+   virtual const Operator *GetOutputProlongation() const
    {
       return GetProlongation(); // Assume square unless specialized
    }
@@ -198,11 +165,11 @@ public:
        form to facilitate matrix-free RAP-type operators.
 
        `NULL` means identity. */
-   virtual const OperatorMP<T> *GetOutputRestrictionTranspose() const { return NULL; }
+   virtual const Operator *GetOutputRestrictionTranspose() const { return NULL; }
 
    /** @brief Restriction operator from output vectors for the operator to linear
        algebra (linear system) vectors. `NULL` means identity. */
-   virtual const OperatorMP<T> *GetOutputRestriction() const
+   virtual const Operator *GetOutputRestriction() const
    {
       return GetRestriction(); // Assume square unless specialized
    }
@@ -238,8 +205,8 @@ public:
        @note If there are no transformations, @a X simply reuses the data of @a
        x. */
    void FormLinearSystem(const Array<int> &ess_tdof_list,
-                         VectorMP<T> &x, VectorMP<T> &b,
-                         OperatorMP<T>* &A, VectorMP<T> &X, VectorMP<T> &B,
+                         Vector &x, Vector &b,
+                         Operator* &A, Vector &X, Vector &B,
                          int copy_interior = 0);
 
    /** @brief Form a column-constrained linear system using a matrix-free approach.
@@ -270,8 +237,8 @@ public:
        x. */
    void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
                                     const Array<int> &test_tdof_list,
-                                    VectorMP<T> &x, VectorMP<T> &b,
-                                    OperatorMP<T>* &A, VectorMP<T> &X, VectorMP<T> &B);
+                                    Vector &x, Vector &b,
+                                    Operator* &A, Vector &X, Vector &B);
 
    /** @brief Reconstruct a solution vector @a x (e.g. a GridFunction) from the
        solution @a X of a constrained linear system obtained from
@@ -282,8 +249,7 @@ public:
        @a x, for this Operator (presumably a finite element grid function). This
        method has identical signature to the analogous method for bilinear
        forms, though currently @a b is not used in the implementation. */
-   virtual void RecoverFEMSolution(const VectorMP<T> &X, const VectorMP<T> &b,
-                                   VectorMP<T> &x);
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
 
    /** @brief Return in @a A a parallel (on truedofs) version of this square
        operator.
@@ -291,7 +257,7 @@ public:
        This returns the same operator as FormLinearSystem(), but does without
        the transformations of the right-hand side and initial guess. */
    void FormSystemOperator(const Array<int> &ess_tdof_list,
-                           OperatorMP<T>* &A);
+                           Operator* &A);
 
    /** @brief Return in @a A a parallel (on truedofs) version of this
        rectangular operator (including constraints).
@@ -300,7 +266,7 @@ public:
        without the transformations of the right-hand side. */
    void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
                                       const Array<int> &test_tdof_list,
-                                      OperatorMP<T>* &A);
+                                      Operator* &A);
 
    /** @brief Return in @a A a parallel (on truedofs) version of this
        rectangular operator.
@@ -313,7 +279,7 @@ public:
        Operator maps between. These are e.g. available through the (parallel)
        finite element space of any (parallel) bilinear form operator. We have:
        `A(X)=[Rout (*this) Pin](X)`. */
-   void FormDiscreteOperator(OperatorMP<T>* &A);
+   void FormDiscreteOperator(Operator* &A);
 
    /// Prints operator with input size n and output size m in Matlab format.
    void PrintMatlab(std::ostream & out, int n, int m = 0) const;
@@ -322,7 +288,28 @@ public:
    virtual void PrintMatlab(std::ostream & out) const;
 
    /// Virtual destructor.
-   virtual ~OperatorMP() { }
+   virtual ~Operator() { }
+
+   /// Enumeration defining IDs for some classes derived from Operator.
+   /** This enumeration is primarily used with class OperatorHandle. */
+   enum Type
+   {
+      ANY_TYPE,         ///< ID for the base class Operator, i.e. any type.
+      MFEM_SPARSEMAT,   ///< ID for class SparseMatrix.
+      Hypre_ParCSR,     ///< ID for class HypreParMatrix.
+      PETSC_MATAIJ,     ///< ID for class PetscParMatrix, MATAIJ format.
+      PETSC_MATIS,      ///< ID for class PetscParMatrix, MATIS format.
+      PETSC_MATSHELL,   ///< ID for class PetscParMatrix, MATSHELL format.
+      PETSC_MATNEST,    ///< ID for class PetscParMatrix, MATNEST format.
+      PETSC_MATHYPRE,   ///< ID for class PetscParMatrix, MATHYPRE format.
+      PETSC_MATGENERIC, ///< ID for class PetscParMatrix, unspecified format.
+      Complex_Operator, ///< ID for class ComplexOperator.
+      MFEM_ComplexSparseMat, ///< ID for class ComplexSparseMatrix.
+      Complex_Hypre_ParCSR,   ///< ID for class ComplexHypreParMatrix.
+      Complex_DenseMat,  ///< ID for class ComplexDenseMatrix
+      MFEM_Block_Matrix,     ///< ID for class BlockMatrix.
+      MFEM_Block_Operator   ///< ID for the base class BlockOperator.
+   };
 
    /// Return the type ID of the Operator class.
    /** This method is intentionally non-virtual, so that it returns the ID of
@@ -332,7 +319,6 @@ public:
    Type GetType() const { return ANY_TYPE; }
 };
 
-using Operator = OperatorMP<real_t>;
 
 /// Base abstract class for first order time dependent operators.
 /** Operator of the form: (u,t) -> k(u,t), where k generally solves the
@@ -395,11 +381,24 @@ public:
       ADDITIVE_TERM_2
    };
 
+   /** Used to specify the variable being returned by ImplicitSolve(). This can
+    * be queried by ODESolver to identify the variable being solved for.
+    * @warning Not all ODESolver may support all options. See ODESolver::SupportsImplicitVariableType() */
+   enum ImplicitVariableType
+   {
+      SLOPE, ///< stage slope, $k = \frac{du}{dt}$.
+      STATE  ///< stage state, $k = u$.
+   };
+
 protected:
    real_t t;  ///< Current time.
    Type type; /**< @brief Describes the form of the TimeDependentOperator, see
                    the documentation of #Type. */
    EvalMode eval_mode; ///< Current evaluation mode.
+   ImplicitVariableType implicit_variable_type =
+      ImplicitVariableType::SLOPE; /**< @brief
+                                                        Return variable for
+                                                        ImplicitSolve()*/
 
 public:
    /** @brief Construct a "square" TimeDependentOperator (u,t) -> k(u,t), where
@@ -443,6 +442,24 @@ public:
    virtual void SetEvalMode(const EvalMode new_eval_mode)
    { eval_mode = new_eval_mode; }
 
+   /** @brief Sets the #ImplicitVariableType for ImplicitSolve()*/
+   virtual void SetImplicitVariableType(const ImplicitVariableType variable_type)
+   { implicit_variable_type = variable_type; }
+
+   /** @brief Returns the #ImplicitVariableType for ImplicitSolve(). */
+   virtual ImplicitVariableType GetImplicitVariableType() const
+   { return implicit_variable_type; }
+
+   /** @brief Returns @a true if implicit variable is #STATE and @a false otherwise.
+    * Used by ODESolver to identify the stage variable returned by ImplicitSolve() */
+   virtual bool ImplicitVarTypeIsState() const
+   { return (implicit_variable_type == ImplicitVariableType::STATE); }
+
+   /** @brief Returns @a true if implicit variable is #SLOPE and @a false otherwise.
+    * Used by ODESolver to identify the stage variable returned by ImplicitSolve() */
+   virtual bool ImplicitVarTypeIsSlope() const
+   { return (implicit_variable_type == ImplicitVariableType::SLOPE); }
+
    /** @brief Perform the action of the explicit part of the operator, G:
        @a v = G(@a u, t) where t is the current time.
 
@@ -476,7 +493,8 @@ public:
 
    /** @brief Solve for the unknown @a k, at the current time t, the following
        equation:
-       F(@a u + @a gamma @a k, @a k, t) = G(@a u + @a gamma @a k, t).
+        1. $F( u + \gamma k, k, t) = G( u +  \gamma k, t)$, if solving for stage-slope (default)
+        2. $F( u , \frac{k-u}{\gamma}, t) = G(k, t)$, if solving for stage-state
 
        For solving an ordinary differential equation of the form
        $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
@@ -486,8 +504,9 @@ public:
          2. F(u,k,t) = M k and G(u,t) = g(u,t)
          3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
 
-       Regardless of the choice of F and G, this function should solve for @a k
-       in M @a k = g(@a u + @a gamma @a k, t).
+        Regardless of the choice of F and G, this function should solve for @a k:
+        - $~Mk = g( u + \gamma k, t)~$, if solving for stage-slope.
+        - $~Mk  = \gamma g(k, t) + Mu~$, if solving for stage-state
 
        To see how @a k can be useful, consider the backward Euler method defined
        by $ y(t + \Delta t) = y(t) + \Delta t k_0 $ where
@@ -505,6 +524,7 @@ public:
        $ y(t) + \Delta t \sum_{j=1}^{i-1} a_{ij} k_j $ and @a gamma set to
        $ a_{ii} \Delta t $, for $ k_i $. For example, see class SDIRK33Solver.
 
+       See SetImplicitVariableType() to switch between different variable modes.
        If not re-implemented, this method simply generates an error. */
    virtual void ImplicitSolve(const real_t gamma, const Vector &u, Vector &k);
 
@@ -802,8 +822,7 @@ public:
 
 
 /// Base class for solvers
-template <class T>
-class SolverMP : public OperatorMP<T>
+class Solver : public Operator
 {
 public:
    /// If true, use the second argument of Mult() as an initial guess.
@@ -813,42 +832,37 @@ public:
 
        @warning Use a Boolean expression for the second parameter (not an int)
        to distinguish this call from the general rectangular constructor. */
-   explicit SolverMP(int s = 0, bool iter_mode = false)
-      : OperatorMP<T>(s) { iterative_mode = iter_mode; }
+   explicit Solver(int s = 0, bool iter_mode = false)
+      : Operator(s) { iterative_mode = iter_mode; }
 
    /// Initialize a Solver with height @a h and width @a w.
-   SolverMP(int h, int w, bool iter_mode = false)
-      : OperatorMP<T>(h, w) { iterative_mode = iter_mode; }
+   Solver(int h, int w, bool iter_mode = false)
+      : Operator(h, w) { iterative_mode = iter_mode; }
 
    /// Set/update the solver for the given operator.
-   virtual void SetOperator(const OperatorMP<T> &op) = 0;
+   virtual void SetOperator(const Operator &op) = 0;
 };
 
-using Solver = SolverMP<real_t>;
 
 /// Identity Operator I: x -> x.
-template <class T>
-class IdentityOperatorMP : public OperatorMP<T>
+class IdentityOperator : public Operator
 {
 public:
    /// Create an identity operator of size @a n.
-   explicit IdentityOperatorMP(int n) : OperatorMP<T>(n) { }
+   explicit IdentityOperator(int n) : Operator(n) { }
 
    /// Operator application
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override { y = x; }
+   void Mult(const Vector &x, Vector &y) const override { y = x; }
 
    /// Application of the transpose
-   void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override { y = x; }
+   void MultTranspose(const Vector &x, Vector &y) const override { y = x; }
 };
 
-using IdentityOperator = IdentityOperatorMP<real_t>;
-
 /// Returns true if P is the identity prolongation, i.e. if it is either NULL or
 /// an IdentityOperator.
-template <class T>
-inline bool IsIdentityProlongation(const OperatorMP<T> *P)
+inline bool IsIdentityProlongation(const Operator *P)
 {
-   return !P || dynamic_cast<const IdentityOperatorMP<T>*>(P);
+   return !P || dynamic_cast<const IdentityOperator*>(P);
 }
 
 /// Scaled Operator B: x -> a A(x).
@@ -875,32 +889,29 @@ public:
 
 /** @brief The transpose of a given operator. Switches the roles of the methods
     Mult() and MultTranspose(). */
-template <class T>
-class TransposeOperatorMP : public OperatorMP<T>
+class TransposeOperator : public Operator
 {
 private:
-   const OperatorMP<T> &A;
+   const Operator &A;
 
 public:
    /// Construct the transpose of a given operator @a *a.
-   TransposeOperatorMP(const OperatorMP<T> *a)
-      : OperatorMP<T>(a->Width(), a->Height()), A(*a) { }
+   TransposeOperator(const Operator *a)
+      : Operator(a->Width(), a->Height()), A(*a) { }
 
    /// Construct the transpose of a given operator @a a.
-   TransposeOperatorMP(const OperatorMP<T> &a)
-      : OperatorMP<T>(a.Width(), a.Height()), A(a) { }
+   TransposeOperator(const Operator &a)
+      : Operator(a.Width(), a.Height()), A(a) { }
 
    /// Operator application. Apply the transpose of the original Operator.
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override
+   void Mult(const Vector &x, Vector &y) const override
    { A.MultTranspose(x, y); }
 
    /// Application of the transpose. Apply the original Operator.
-   void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override
+   void MultTranspose(const Vector &x, Vector &y) const override
    { A.Mult(x, y); }
 };
 
-using TransposeOperator = TransposeOperatorMP<real_t>;
-
 /// General linear combination operator: x -> a A(x) + b B(x).
 class SumOperator : public Operator
 {
@@ -925,53 +936,48 @@ public:
 };
 
 /// General product operator: x -> (A*B)(x) = A(B(x)).
-template <class T>
-class ProductOperatorMP : public OperatorMP<T>
+class ProductOperator : public Operator
 {
-   const OperatorMP<T> *A, *B;
+   const Operator *A, *B;
    bool ownA, ownB;
-   mutable VectorMP<T> z;
+   mutable Vector z;
 
 public:
-   ProductOperatorMP(const OperatorMP<T> *A, const OperatorMP<T> *B, bool ownA,
-                     bool ownB);
+   ProductOperator(const Operator *A, const Operator *B, bool ownA, bool ownB);
 
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override
+   void Mult(const Vector &x, Vector &y) const override
    { B->Mult(x, z); A->Mult(z, y); }
 
-   void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override
+   void MultTranspose(const Vector &x, Vector &y) const override
    { A->MultTranspose(x, z); B->MultTranspose(z, y); }
 
-   virtual ~ProductOperatorMP<T>();
+   virtual ~ProductOperator();
 };
 
-using ProductOperator = ProductOperatorMP<real_t>;
 
 /// The operator x -> R*A*P*x constructed through the actions of R^T, A and P
-template <class T>
-class RAPOperatorMP : public OperatorMP<T>
+class RAPOperator : public Operator
 {
 private:
-   const OperatorMP<T> & Rt;
-   const OperatorMP<T> & A;
-   const OperatorMP<T> & P;
-   mutable VectorMP<T> Px;
-   mutable VectorMP<T> APx;
+   const Operator & Rt;
+   const Operator & A;
+   const Operator & P;
+   mutable Vector Px;
+   mutable Vector APx;
    MemoryClass mem_class;
 
 public:
    /// Construct the RAP operator given R^T, A and P.
-   RAPOperatorMP<T>(const OperatorMP<T> &Rt_, const OperatorMP<T> &A_,
-                    const OperatorMP<T> &P_);
+   RAPOperator(const Operator &Rt_, const Operator &A_, const Operator &P_);
 
    MemoryClass GetMemoryClass() const override { return mem_class; }
 
    /// Operator application.
-   void Mult(const VectorMP<T> & x, VectorMP<T> & y) const override
+   void Mult(const Vector & x, Vector & y) const override
    { P.Mult(x, Px); A.Mult(Px, APx); Rt.MultTranspose(APx, y); }
 
    /// Operator-wise absolute-value application.
-   void AbsMult(const VectorMP<T> & x, VectorMP<T> & y) const override
+   void AbsMult(const Vector & x, Vector & y) const override
    { P.AbsMult(x, Px); A.AbsMult(Px, APx); Rt.AbsMultTranspose(APx, y); }
 
    /// Approximate diagonal of the RAP Operator.
@@ -981,7 +987,7 @@ public:
        When P is the FE space prolongation operator on a mesh without hanging
        nodes and Rt = P, the returned diagonal is exact, as long as the diagonal
        of A is also exact. */
-   void AssembleDiagonal(VectorMP<T> &diag) const override
+   void AssembleDiagonal(Vector &diag) const override
    {
       A.AssembleDiagonal(APx);
       P.MultTranspose(APx, diag);
@@ -992,11 +998,11 @@ public:
    }
 
    /// Application of the transpose.
-   void MultTranspose(const VectorMP<T> & x, VectorMP<T> & y) const override
+   void MultTranspose(const Vector & x, Vector & y) const override
    { Rt.Mult(x, APx); A.MultTranspose(APx, Px); P.MultTranspose(Px, y); }
 
    /// Operator-wise absolute-value application of the transpose
-   void AbsMultTranspose(const VectorMP<T> & x, VectorMP<T> & y) const override
+   void AbsMultTranspose(const Vector & x, Vector & y) const override
    {
       Rt.AbsMult(x, APx);
       A.AbsMultTranspose(APx, Px);
@@ -1004,35 +1010,32 @@ public:
    }
 };
 
-using RAPOperator = RAPOperatorMP<real_t>;
 
 /// General triple product operator x -> A*B*C*x, with ownership of the factors.
-template <class T>
-class TripleProductOperatorMP : public OperatorMP<T>
+class TripleProductOperator : public Operator
 {
-   const OperatorMP<T> *A;
-   const OperatorMP<T> *B;
-   const OperatorMP<T> *C;
+   const Operator *A;
+   const Operator *B;
+   const Operator *C;
    bool ownA, ownB, ownC;
-   mutable VectorMP<T> t1, t2;
+   mutable Vector t1, t2;
    MemoryClass mem_class;
 
 public:
-   TripleProductOperatorMP(const OperatorMP<T> *A, const OperatorMP<T> *B,
-                           const OperatorMP<T> *C, bool ownA, bool ownB, bool ownC);
+   TripleProductOperator(const Operator *A, const Operator *B,
+                         const Operator *C, bool ownA, bool ownB, bool ownC);
 
    MemoryClass GetMemoryClass() const override { return mem_class; }
 
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override
+   void Mult(const Vector &x, Vector &y) const override
    { C->Mult(x, t1); B->Mult(t1, t2); A->Mult(t2, y); }
 
-   void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override
+   void MultTranspose(const Vector &x, Vector &y) const override
    { A->MultTranspose(x, t2); B->MultTranspose(t2, t1); C->MultTranspose(t1, y); }
 
-   virtual ~TripleProductOperatorMP<T>();
+   virtual ~TripleProductOperator();
 };
 
-using TripleProductOperator = TripleProductOperatorMP<real_t>;
 
 /** @brief Square Operator for imposing essential boundary conditions using only
     the action, Mult(), of a given unconstrained Operator.
@@ -1043,19 +1046,13 @@ using TripleProductOperator = TripleProductOperatorMP<real_t>;
 
     Do not confuse with ConstrainedSolver, which despite the name has very
     different functionality. */
-template <class T>
-class ConstrainedOperatorMP : public OperatorMP<T>
+class ConstrainedOperator : public Operator
 {
-   using DiagonalPolicy = OperatorBase::DiagonalPolicy;
-   using OperatorBase::DIAG_ONE;
-   using OperatorBase::DIAG_KEEP;
-   using OperatorBase::DIAG_ZERO;
-
 protected:
    Array<int> constraint_list;  ///< List of constrained indices/dofs.
-   OperatorMP<T> *A;                 ///< The unconstrained Operator.
+   Operator *A;                 ///< The unconstrained Operator.
    bool own_A;                  ///< Ownership flag for A.
-   mutable VectorMP<T> z, w;         ///< Auxiliary vectors.
+   mutable Vector z, w;         ///< Auxiliary vectors.
    MemoryClass mem_class;
    DiagonalPolicy diag_policy;  ///< Diagonal policy for constrained dofs
 
@@ -1068,9 +1065,8 @@ public:
        ownership flag @a own_A is true, the operator @a *A will be destroyed
        when this object is destroyed. The @a diag_policy determines how the
        operator sets entries corresponding to essential dofs. */
-   ConstrainedOperatorMP(OperatorMP<T> *A, const Array<int> &list,
-                         bool own_A = false,
-                         DiagonalPolicy diag_policy = DIAG_ONE);
+   ConstrainedOperator(Operator *A, const Array<int> &list, bool own_A = false,
+                       DiagonalPolicy diag_policy = DIAG_ONE);
 
    /// Returns the type of memory in which the solution and temporaries are stored.
    MemoryClass GetMemoryClass() const override { return mem_class; }
@@ -1080,7 +1076,7 @@ public:
    { diag_policy = diag_policy_; }
 
    /// Diagonal of A, modified according to the used DiagonalPolicy.
-   void AssembleDiagonal(VectorMP<T> &diag) const override;
+   void AssembleDiagonal(Vector &diag) const override;
 
    /** @brief Eliminate "essential boundary condition" values specified in @a x
        from the given right-hand side @a b.
@@ -1093,7 +1089,7 @@ public:
        the vectors, and "_i" -- the rest of the entries.
 
        @note This method is consistent with `DiagonalPolicy::DIAG_ONE`. */
-   void EliminateRHS(const VectorMP<T> &x, VectorMP<T> &b) const;
+   void EliminateRHS(const Vector &x, Vector &b) const;
 
    /** @brief Constrained operator action.
 
@@ -1103,33 +1099,29 @@ public:
 
        where the "_b" subscripts denote the essential (boundary) indices/dofs of
        the vectors, and "_i" -- the rest of the entries. */
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   void Mult(const Vector &x, Vector &y) const override;
 
-   void AddMult(const VectorMP<T> &x, VectorMP<T> &y,
-                const T a = 1.0) const override;
+   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override;
 
-   void AbsMult(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   void AbsMult(const Vector &x, Vector &y) const override;
 
-   void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   void MultTranspose(const Vector &x, Vector &y) const override;
 
-   void AbsMultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   void AbsMultTranspose(const Vector &x, Vector &y) const override;
 
    /** @brief Implementation of Mult or MultTranspose.
-    *  TODO - Generalize to allow constraining rows and columns differently. */
-   void ConstrainedMult(const VectorMP<T> &x, VectorMP<T> &y,
-                        const bool transpose) const;
+       TODO - Generalize to allow constraining rows and columns differently. */
+   void ConstrainedMult(const Vector &x, Vector &y, const bool transpose) const;
 
    /** @brief Implementation of AbsMult or AbsMultTranspose.
        TODO - Generalize to allow constraining rows and columns differently. */
-   void ConstrainedAbsMult(const VectorMP<T> &x, VectorMP<T> &y,
+   void ConstrainedAbsMult(const Vector &x, Vector &y,
                            const bool transpose) const;
 
    /// Destructor: destroys the unconstrained Operator, if owned.
-   ~ConstrainedOperatorMP<T>() override { if (own_A) { delete A; } }
+   ~ConstrainedOperator() override { if (own_A) { delete A; } }
 };
 
-using ConstrainedOperator = ConstrainedOperatorMP<real_t>;
-
 /** @brief Rectangular Operator for imposing essential boundary conditions on
     the input space using only the action, Mult(), of a given unconstrained
     Operator.
@@ -1137,14 +1129,13 @@ using ConstrainedOperator = ConstrainedOperatorMP<real_t>;
     Rectangular operator constrained by fixing certain entries in the solution
     to given "essential boundary condition" values. This class is used by the
     general matrix-free formulation of Operator::FormRectangularLinearSystem. */
-template <class T>
-class RectangularConstrainedOperatorMP : public OperatorMP<T>
+class RectangularConstrainedOperator : public Operator
 {
 protected:
    Array<int> trial_constraints, test_constraints;
-   OperatorMP<T> *A;
+   Operator *A;
    bool own_A;
-   mutable VectorMP<T> z, w;
+   mutable Vector z, w;
    MemoryClass mem_class;
 
 public:
@@ -1155,8 +1146,8 @@ public:
        constrain, i.e. each entry @a trial_list[i] represents an essential trial
        dof. If the ownership flag @a own_A is true, the operator @a *A will be
        destroyed when this object is destroyed. */
-   RectangularConstrainedOperatorMP(OperatorMP<T> *A, const Array<int> &trial_list,
-                                    const Array<int> &test_list, bool own_A = false);
+   RectangularConstrainedOperator(Operator *A, const Array<int> &trial_list,
+                                  const Array<int> &test_list, bool own_A = false);
    /// Returns the type of memory in which the solution and temporaries are stored.
    MemoryClass GetMemoryClass() const override { return mem_class; }
    /** @brief Eliminate columns corresponding to "essential boundary condition"
@@ -1169,7 +1160,7 @@ public:
 
        where the "_b" subscripts denote the essential (boundary) indices and the
        "_j" subscript denotes the essential test indices */
-   void EliminateRHS(const VectorMP<T> &x, VectorMP<T> &b) const;
+   void EliminateRHS(const Vector &x, Vector &b) const;
    /** @brief Rectangular-constrained operator action.
 
        Performs the following steps:
@@ -1179,19 +1170,16 @@ public:
 
        where the "_i" subscripts denote all the nonessential (boundary) trial
        indices and the "_j" subscript denotes the essential test indices */
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override;
-   void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override;
-   virtual ~RectangularConstrainedOperatorMP<T>() { if (own_A) { delete A; } }
+   void Mult(const Vector &x, Vector &y) const override;
+   void MultTranspose(const Vector &x, Vector &y) const override;
+   virtual ~RectangularConstrainedOperator() { if (own_A) { delete A; } }
 };
 
-using RectangularConstrainedOperator = RectangularConstrainedOperatorMP<real_t>;
-
 /** @brief Abstract class for defining inner products. The method Eval()
     must be implemented in derived classes to compute the inner product
     of two vectors according to a specific inner product definition.
 */
-template <class T>
-class InnerProductOperatorMP : public OperatorMP<T>
+class InnerProductOperator : public Operator
 {
 #ifdef MFEM_USE_MPI
 private:
@@ -1199,16 +1187,16 @@ private:
    int dot_prod_type = 0; // 0: local, 1: global
 
 public:
-   InnerProductOperatorMP(MPI_Comm comm_) : OperatorMP<T>(1)
+   InnerProductOperator(MPI_Comm comm_) : Operator(1)
    { comm = comm_; dot_prod_type = 1; }
 #endif
 protected:
    /// @brief Standard global/local $\ell_2$ inner product.
-   virtual T Dot(const VectorMP<T> &x, const VectorMP<T> &y) const;
+   virtual real_t Dot(const Vector &x, const Vector &y) const;
 
 public:
    /// Create an operator of size 1 (scalar).
-   InnerProductOperatorMP() : OperatorMP<T>(1)
+   InnerProductOperator() : Operator(1)
    {
 #ifdef MFEM_USE_MPI
       dot_prod_type = 0;
@@ -1217,7 +1205,7 @@ public:
 
    /// Operator application - not always needed/used but added
    /// to satisfy the abstract base class interface.
-   virtual void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override
+   virtual void Mult(const Vector &x, Vector &y) const override
    {
       MFEM_ABORT("Mult is not implemented.");
    }
@@ -1225,11 +1213,9 @@ public:
    /** @brief Compute the inner product (x,y) of vectors x and y.
               This is an abstract method that must be
               implemented in derived classes. */
-   virtual real_t Eval(const VectorMP<T> &x, const VectorMP<T> &y) = 0;
+   virtual real_t Eval(const Vector &x, const Vector &y) = 0;
 };
 
-using InnerProductOperator = InnerProductOperatorMP<real_t>;
-
 /** @brief PowerMethod helper class to estimate the largest eigenvalue of an
            operator using the iterative power method. */
 class PowerMethod

linalg/petsc.cpp

@@ -3639,12 +3639,20 @@ void PetscBDDCSolver::BDDCSolverConstructor(const PetscBDDCSolverParams &opts)
       // make sure ess/nat_dof have been collectively set
       PetscBool lpr = PETSC_FALSE,pr;
       if (opts.ess_dof) { lpr = PETSC_TRUE; }
+#if PETSC_VERSION_LT(3,24,0)
       mpiierr = MPI_Allreduce(&lpr,&pr,1,MPIU_BOOL,MPI_LOR,comm);
+#else
+      mpiierr = MPI_Allreduce(&lpr,&pr,1,MPI_C_BOOL,MPI_LOR,comm);
+#endif
       CCHKERRQ(comm,mpiierr);
       MFEM_VERIFY(lpr == pr,"ess_dof should be collectively set");
       lpr = PETSC_FALSE;
       if (opts.nat_dof) { lpr = PETSC_TRUE; }
+#if PETSC_VERSION_LT(3,24,0)
       mpiierr = MPI_Allreduce(&lpr,&pr,1,MPIU_BOOL,MPI_LOR,comm);
+#else
+      mpiierr = MPI_Allreduce(&lpr,&pr,1,MPI_C_BOOL,MPI_LOR,comm);
+#endif
       CCHKERRQ(comm,mpiierr);
       MFEM_VERIFY(lpr == pr,"nat_dof should be collectively set");
       // make sure fields have been collectively set
@@ -4058,8 +4066,13 @@ void PetscNonlinearSolver::SetOperator(const Operator &op)
       ls = (PetscBool)(height == op.Height() && width  == op.Width() &&
                        (void*)&op == fctx &&
                        (void*)&op == jctx);
+#if PETSC_VERSION_LT(3,24,0)
       mpiierr = MPI_Allreduce(&ls,&gs,1,MPIU_BOOL,MPI_LAND,
                               PetscObjectComm((PetscObject)snes));
+#else
+      mpiierr = MPI_Allreduce(&ls,&gs,1,MPI_C_BOOL,MPI_LAND,
+                              PetscObjectComm((PetscObject)snes));
+#endif
       CCHKERRQ(PetscObjectComm((PetscObject)snes),mpiierr);
       if (!gs)
       {

linalg/solvers.cpp

@@ -26,9 +26,8 @@ namespace mfem
 
 using namespace std;
 
-template <class T>
-IterativeSolverMP<T>::IterativeSolverMP()
-   : SolverMP<T>(0, true)
+IterativeSolver::IterativeSolver()
+   : Solver(0, true)
 {
    oper = NULL;
    prec = NULL;
@@ -42,9 +41,8 @@ IterativeSolverMP<T>::IterativeSolverMP()
 
 #ifdef MFEM_USE_MPI
 
-template <class T>
-IterativeSolverMP<T>::IterativeSolverMP(MPI_Comm comm_)
-   : SolverMP<T>(0, true)
+IterativeSolver::IterativeSolver(MPI_Comm comm_)
+   : Solver(0, true)
 {
    oper = NULL;
    prec = NULL;
@@ -57,8 +55,7 @@ IterativeSolverMP<T>::IterativeSolverMP(MPI_Comm comm_)
 
 #endif // MFEM_USE_MPI
 
-template <class T>
-T IterativeSolverMP<T>::Dot(const VectorMP<T> &x, const VectorMP<T> &y) const
+real_t IterativeSolver::Dot(const Vector &x, const Vector &y) const
 {
    if (dot_oper) { return dot_oper->Eval(x,y); } // Use custom inner product (if provided)
 
@@ -76,8 +73,7 @@ T IterativeSolverMP<T>::Dot(const VectorMP<T> &x, const VectorMP<T> &y) const
 #endif
 }
 
-template <class T>
-void IterativeSolverMP<T>::SetPrintLevel(int print_lvl)
+void IterativeSolver::SetPrintLevel(int print_lvl)
 {
    print_options = FromLegacyPrintLevel(print_lvl);
    int print_level_ = print_lvl;
@@ -98,8 +94,7 @@ void IterativeSolverMP<T>::SetPrintLevel(int print_lvl)
    print_level = print_level_;
 }
 
-template <class T>
-void IterativeSolverMP<T>::SetPrintLevel(PrintLevel options)
+void IterativeSolver::SetPrintLevel(PrintLevel options)
 {
    print_options = options;
 
@@ -121,9 +116,7 @@ void IterativeSolverMP<T>::SetPrintLevel(PrintLevel options)
    print_level = derived_print_level;
 }
 
-template <class T>
-typename IterativeSolverMP<T>::PrintLevel
-IterativeSolverMP<T>::FromLegacyPrintLevel(
+IterativeSolver::PrintLevel IterativeSolver::FromLegacyPrintLevel(
    int print_level_)
 {
 #ifdef MFEM_USE_MPI
@@ -158,8 +151,7 @@ IterativeSolverMP<T>::FromLegacyPrintLevel(
    }
 }
 
-template <class T>
-int IterativeSolverMP<T>::GuessLegacyPrintLevel(PrintLevel print_options_)
+int IterativeSolver::GuessLegacyPrintLevel(PrintLevel print_options_)
 {
    if (print_options_.iterations)
    {
@@ -183,28 +175,25 @@ int IterativeSolverMP<T>::GuessLegacyPrintLevel(PrintLevel print_options_)
    }
 }
 
-template <class T>
-void IterativeSolverMP<T>::SetPreconditioner(SolverMP<T> &pr)
+void IterativeSolver::SetPreconditioner(Solver &pr)
 {
    prec = &pr;
    prec->iterative_mode = false;
 }
 
-template <class T>
-void IterativeSolverMP<T>::SetOperator(const OperatorMP<T> &op)
+void IterativeSolver::SetOperator(const Operator &op)
 {
    oper = &op;
-   this->height = op.Height();
-   this->width = op.Width();
+   height = op.Height();
+   width = op.Width();
    if (prec)
    {
       prec->SetOperator(*oper);
    }
 }
 
-template <class T>
-bool IterativeSolverMP<T>::Monitor(int it, T norm, const VectorMP<T>& r,
-                                   const VectorMP<T>& x, bool final) const
+bool IterativeSolver::Monitor(int it, real_t norm, const Vector& r,
+                              const Vector& x, bool final) const
 {
    if (controller != nullptr)
    {
@@ -862,31 +851,30 @@ void SLI(const Operator &A, Solver &B, const Vector &b, Vector &x,
    sli.Mult(b, x);
 }
 
-template <class T>
-void CGSolverMP<T>::UpdateVectors()
-{
-   MemoryType mt = GetMemoryType(this->oper->GetMemoryClass());
 
-   r.SetSize(this->width, mt);
+void CGSolver::UpdateVectors()
+{
+   MemoryType mt = GetMemoryType(oper->GetMemoryClass());
+
+   r.SetSize(width, mt);
    r.UseDevice(true);
 
-   d.SetSize(this->width, mt);
+   d.SetSize(width, mt);
    d.UseDevice(true);
 
-   z.SetSize(this->width, mt);
+   z.SetSize(width, mt);
    z.UseDevice(true);
 }
 
-template <class T>
-void CGSolverMP<T>::Mult(const VectorMP<T> &b, VectorMP<T> &x) const
+void CGSolver::Mult(const Vector &b, Vector &x) const
 {
    int i;
-   T r0, den, nom, nom0, betanom, alpha, beta;
+   real_t r0, den, nom, nom0, betanom, alpha, beta;
 
    x.UseDevice(true);
-   if (this->iterative_mode)
+   if (iterative_mode)
    {
-      this->oper->Mult(x, r);
+      oper->Mult(x, r);
       subtract(b, r, r); // r = b - A x
    }
    else
@@ -895,56 +883,56 @@ void CGSolverMP<T>::Mult(const VectorMP<T> &b, VectorMP<T> &x) const
       x = 0.0;
    }
 
-   if (this->prec)
+   if (prec)
    {
-      this->prec->Mult(r, z); // z = B r
+      prec->Mult(r, z); // z = B r
       d = z;
    }
    else
    {
       d = r;
    }
-   nom0 = nom = this->Dot(d, r);
-   if (nom0 >= 0.0) { this->initial_norm = sqrt(nom0); }
+   nom0 = nom = Dot(d, r);
+   if (nom0 >= 0.0) { initial_norm = sqrt(nom0); }
    MFEM_VERIFY(IsFinite(nom), "nom = " << nom);
-   if (this->print_options.iterations || this->print_options.first_and_last)
+   if (print_options.iterations || print_options.first_and_last)
    {
       mfem::out << "   Iteration : " << setw(3) << 0 << "  (B r, r) = "
-                << nom << (this->print_options.first_and_last ? " ...\n" : "\n");
+                << nom << (print_options.first_and_last ? " ...\n" : "\n");
    }
 
    if (nom < 0.0)
    {
-      if (this->print_options.warnings)
+      if (print_options.warnings)
       {
          mfem::out << "PCG: The preconditioner is not positive definite. (Br, r) = "
                    << nom << '\n';
       }
       converged = false;
       final_iter = 0;
-      this->initial_norm = nom;
+      initial_norm = nom;
       final_norm = nom;
 
-      this->Monitor(0, nom, r, x, true);
+      Monitor(0, nom, r, x, true);
       return;
    }
-   r0 = std::max(nom*this->rel_tol*this->rel_tol, this->abs_tol*this->abs_tol);
-   if (this->Monitor(0, nom, r, x) || nom <= r0)
+   r0 = std::max(nom*rel_tol*rel_tol, abs_tol*abs_tol);
+   if (Monitor(0, nom, r, x) || nom <= r0)
    {
       converged = true;
       final_iter = 0;
       final_norm = sqrt(nom);
 
-      this->Monitor(0, nom, r, x, true);
+      Monitor(0, nom, r, x, true);
       return;
    }
 
    oper->Mult(d, z);  // z = A d
-   den = this->Dot(z, d);
+   den = Dot(z, d);
    MFEM_VERIFY(IsFinite(den), "den = " << den);
    if (den <= 0.0)
    {
-      if (this->Dot(d, d) > 0.0 && print_options.warnings)
+      if (Dot(d, d) > 0.0 && print_options.warnings)
       {
          mfem::out << "PCG: The operator is not positive definite. (Ad, d) = "
                    << den << '\n';
@@ -955,7 +943,7 @@ void CGSolverMP<T>::Mult(const VectorMP<T> &b, VectorMP<T> &x) const
          final_iter = 0;
          final_norm = sqrt(nom);
 
-         this->Monitor(0, nom, r, x, true);
+         Monitor(0, nom, r, x, true);
          return;
       }
    }
@@ -972,11 +960,11 @@ void CGSolverMP<T>::Mult(const VectorMP<T> &b, VectorMP<T> &x) const
       if (prec)
       {
          prec->Mult(r, z);      //  z = B r
-         betanom = this->Dot(r, z);
+         betanom = Dot(r, z);
       }
       else
       {
-         betanom = this->Dot(r, r);
+         betanom = Dot(r, r);
       }
       MFEM_VERIFY(IsFinite(betanom), "betanom = " << betanom);
       if (betanom < 0.0)
@@ -997,7 +985,7 @@ void CGSolverMP<T>::Mult(const VectorMP<T> &b, VectorMP<T> &x) const
                    << betanom << std::endl;
       }
 
-      if (this->Monitor(i, betanom, r, x) || betanom <= r0)
+      if (Monitor(i, betanom, r, x) || betanom <= r0)
       {
          converged = true;
          final_iter = i;
@@ -1019,11 +1007,11 @@ void CGSolverMP<T>::Mult(const VectorMP<T> &b, VectorMP<T> &x) const
          add(r, beta, d, d);
       }
       oper->Mult(d, z);       //  z = A d
-      den = this->Dot(d, z);
+      den = Dot(d, z);
       MFEM_VERIFY(IsFinite(den), "den = " << den);
       if (den <= 0.0)
       {
-         if (this->Dot(d, d) > 0.0 && print_options.warnings)
+         if (Dot(d, d) > 0.0 && print_options.warnings)
          {
             mfem::out << "PCG: The operator is not positive definite. (Ad, d) = "
                       << den << '\n';
@@ -1058,7 +1046,7 @@ void CGSolverMP<T>::Mult(const VectorMP<T> &b, VectorMP<T> &x) const
 
    final_norm = sqrt(betanom);
 
-   this->Monitor(final_iter, final_norm, r, x, true);
+   Monitor(final_iter, final_norm, r, x, true);
 }
 
 void CG(const Operator &A, const Vector &b, Vector &x,
@@ -1076,35 +1064,22 @@ void CG(const Operator &A, const Vector &b, Vector &x,
    cg.Mult(b, x);
 }
 
-template <class T>
-void PCG(const OperatorMP<T> &A, SolverMP<T> &B, const VectorMP<T> &b,
-         VectorMP<T> &x,
+void PCG(const Operator &A, Solver &B, const Vector &b, Vector &x,
          int print_iter, int max_num_iter,
-         double RTOLERANCE, double ATOLERANCE)
+         real_t RTOLERANCE, real_t ATOLERANCE)
 {
    MFEM_PERF_FUNCTION;
 
-   CGSolverMP<T> pcg;
+   CGSolver pcg;
    pcg.SetPrintLevel(print_iter);
    pcg.SetMaxIter(max_num_iter);
-   pcg.SetRelTol(sqrt((T)RTOLERANCE));
-   pcg.SetAbsTol(sqrt((T)ATOLERANCE));
+   pcg.SetRelTol(sqrt(RTOLERANCE));
+   pcg.SetAbsTol(sqrt(ATOLERANCE));
    pcg.SetOperator(A);
    pcg.SetPreconditioner(B);
    pcg.Mult(b, x);
 }
 
-template
-void PCG<float>(const OperatorMP<float> &A, SolverMP<float> &B,
-                const VectorMP<float> &b, VectorMP<float> &x,
-                int print_iter, int max_num_iter,
-                double RTOLERANCE, double ATOLERANCE);
-
-template
-void PCG<double>(const OperatorMP<double> &A, SolverMP<double> &B,
-                 const VectorMP<double> &b, VectorMP<double> &x,
-                 int print_iter, int max_num_iter,
-                 double RTOLERANCE, double ATOLERANCE);
 
 inline void GeneratePlaneRotation(real_t &dx, real_t &dy,
                                   real_t &cs, real_t &sn)
@@ -1965,7 +1940,7 @@ void MINRESSolver::Mult(const Vector &b, Vector &x) const
       }
       else if (it == 2)
       {
-         add((real_t) 1./rho1, *z, -rho2/rho1, w1, w0);   // (w0 == 0)
+         add(1./rho1, *z, -rho2/rho1, w1, w0);   // (w0 == 0)
       }
       else
       {
@@ -3082,7 +3057,7 @@ void BlockILU::SetOperator(const Operator &op)
    Factorize();
 }
 
-void BlockILU::CreateBlockPattern(const SparseMatrixMP<real_t> &A)
+void BlockILU::CreateBlockPattern(const SparseMatrix &A)
 {
    MFEM_VERIFY(k_fill == 0, "Only block ILU(0) is currently supported.");
    if (A.Height() % block_size != 0)
@@ -4649,10 +4624,4 @@ void NNLSSolver::Solve(const Vector& rhs_lb, const Vector& rhs_ub,
 }
 #endif // MFEM_USE_LAPACK
 
-template class CGSolverMP<float>;
-template class CGSolverMP<double>;
-
-template class IterativeSolverMP<float>;
-template class IterativeSolverMP<double>;
-
 }

linalg/solvers.hpp

@@ -14,7 +14,6 @@
 
 #include "../config/config.hpp"
 #include "densemat.hpp"
-#include "sparsemat.hpp"
 #include "handle.hpp"
 #include <memory>
 
@@ -33,26 +32,21 @@ namespace mfem
 
 class BilinearForm;
 
-
-template <class T>
-class IterativeSolverMP;
-
 /// Abstract base class for an iterative solver controller
-template <class T>
-class IterativeSolverControllerMP
+class IterativeSolverController
 {
 protected:
    /// The last IterativeSolver to which this controller was attached.
-   const class IterativeSolverMP<T> *iter_solver;
+   const class IterativeSolver *iter_solver;
 
    /// In MonitorResidual or MonitorSolution, this member variable can be set
    /// to true to indicate early convergence.
    bool converged = false;
 
 public:
-   IterativeSolverControllerMP() : iter_solver(nullptr) {}
+   IterativeSolverController() : iter_solver(nullptr) {}
 
-   virtual ~IterativeSolverControllerMP() {}
+   virtual ~IterativeSolverController() {}
 
    /// Has the solver converged?
    ///
@@ -67,13 +61,13 @@ public:
    virtual void Reset() { converged = false; }
 
    /// Monitor the solution vector r
-   virtual void MonitorResidual(int it, T norm, const VectorMP<T> &r,
+   virtual void MonitorResidual(int it, real_t norm, const Vector &r,
                                 bool final)
    {
    }
 
    /// Monitor the solution vector x
-   virtual void MonitorSolution(int it, T norm, const VectorMP<T> &x,
+   virtual void MonitorSolution(int it, real_t norm, const Vector &x,
                                 bool final)
    {
    }
@@ -85,16 +79,15 @@ public:
 
    /** @brief This method is invoked by IterativeSolver::SetController(),
        informing the controller which IterativeSolver is using it. */
-   void SetIterativeSolver(const IterativeSolverMP<T> &solver)
+   void SetIterativeSolver(const IterativeSolver &solver)
    { iter_solver = &solver; }
 };
 
 /// Keeping the alias for backward compatibility
-using IterativeSolverMonitor = IterativeSolverControllerMP<real_t>;
+using IterativeSolverMonitor = IterativeSolverController;
 
 /// Abstract base class for iterative solver
-template <class T>
-class IterativeSolverMP : public SolverMP<T>
+class IterativeSolver : public Solver
 {
 public:
    /** @brief Settings for the output behavior of the IterativeSolver.
@@ -149,10 +142,10 @@ private:
 #endif
 
 protected:
-   const OperatorMP<T> *oper;
-   SolverMP<T> *prec;
-   IterativeSolverControllerMP<T> *controller = nullptr;
-   InnerProductOperatorMP<T> *dot_oper = nullptr;
+   const Operator *oper;
+   Solver *prec;
+   IterativeSolverController *controller = nullptr;
+   InnerProductOperator *dot_oper = nullptr;
 
    /// @name Reporting (protected attributes and member functions)
    ///@{
@@ -184,10 +177,10 @@ protected:
    int max_iter;
 
    /// Relative tolerance.
-   T rel_tol;
+   real_t rel_tol;
 
    /// Absolute tolerance.
-   T abs_tol;
+   real_t abs_tol;
 
    ///@}
 
@@ -197,7 +190,7 @@ protected:
 
    mutable int final_iter = -1;
    mutable bool converged = false;
-   mutable T initial_norm = -1.0, final_norm = -1.0;
+   mutable real_t initial_norm = -1.0, final_norm = -1.0;
 
    ///@}
 
@@ -207,23 +200,23 @@ protected:
        @details Overriding this method in a derived class enables a
        custom inner product.
       */
-   virtual T Dot(const VectorMP<T> &x, const VectorMP<T> &y) const;
+   virtual real_t Dot(const Vector &x, const Vector &y) const;
 
    /// Return the inner product norm of @a x, using the inner product defined by Dot()
-   T Norm(const VectorMP<T> &x) const { return sqrt(Dot(x, x)); }
+   real_t Norm(const Vector &x) const { return sqrt(Dot(x, x)); }
 
    /// Indicated if the controller requires an update of the solution
    bool ControllerRequiresUpdate() const { return controller && controller->RequiresUpdatedSolution(); }
 
    /// Monitor both the residual @a r and the solution @a x
-   bool Monitor(int it, T norm, const VectorMP<T>& r, const VectorMP<T>& x,
+   bool Monitor(int it, real_t norm, const Vector& r, const Vector& x,
                 bool final=false) const;
 
 public:
-   IterativeSolverMP();
+   IterativeSolver();
 
 #ifdef MFEM_USE_MPI
-   IterativeSolverMP(MPI_Comm comm_);
+   IterativeSolver(MPI_Comm comm_);
 #endif
 
    /** @name Convergence
@@ -242,8 +235,8 @@ public:
        X depends on the specific iterative solver.
       */
    ///@{
-   void SetRelTol(T rtol) { rel_tol = rtol; }
-   void SetAbsTol(T atol) { abs_tol = atol; }
+   void SetRelTol(real_t rtol) { rel_tol = rtol; }
+   void SetAbsTol(real_t atol) { abs_tol = atol; }
    void SetMaxIter(int max_it) { max_iter = max_it; }
    ///@}
 
@@ -301,20 +294,20 @@ public:
    /// This function returns the norm of the residual (or preconditioned
    /// residual, depending on the solver), computed before the start of the
    /// iteration.
-   T GetInitialNorm() const { return initial_norm; }
+   real_t GetInitialNorm() const { return initial_norm; }
    /// @brief Returns the final residual norm after termination of the solver
    /// during the last call to Mult().
    ///
    /// This function returns the norm of the residual (or preconditioned
    /// residual, depending on the solver), corresponding to the returned
    /// solution.
-   T GetFinalNorm() const { return final_norm; }
+   real_t GetFinalNorm() const { return final_norm; }
    /// @brief Returns the final residual norm after termination of the solver
    /// during the last call to Mult(), divided by the initial residual norm.
    /// Returns -1 if one of these norms is left undefined by the solver.
    ///
    /// @sa GetFinalNorm(), GetInitialNorm()
-   T GetFinalRelNorm() const
+   real_t GetFinalRelNorm() const
    {
       if (final_norm < 0.0 || initial_norm < 0.0) { return -1.0; }
       return final_norm / initial_norm;
@@ -323,20 +316,20 @@ public:
    ///@}
 
    /// This should be called before SetOperator
-   virtual void SetPreconditioner(SolverMP<T> &pr);
+   virtual void SetPreconditioner(Solver &pr);
 
    /// Also calls SetOperator for the preconditioner if there is one
-   void SetOperator(const OperatorMP<T> &op) override;
+   void SetOperator(const Operator &op) override;
 
    /// Set the iterative solver controller
-   void SetController(IterativeSolverControllerMP<T> &c)
+   void SetController(IterativeSolverController &c)
    { controller = &c; c.SetIterativeSolver(*this); }
 
    /// An alias of SetController() for backward compatibility
-   void SetMonitor(IterativeSolverControllerMP<T> &m) { SetController(m); }
+   void SetMonitor(IterativeSolverMonitor &m) { SetController(m); }
 
    /// Set a user-defined inner product operator (not owned)
-   void SetInnerProduct(InnerProductOperatorMP<T> *ipo) { dot_oper = ipo; }
+   void SetInnerProduct(InnerProductOperator *ipo) { dot_oper = ipo; }
 
 #ifdef MFEM_USE_MPI
    /** @brief Return the associated MPI communicator, or MPI_COMM_NULL if no
@@ -346,7 +339,6 @@ public:
 #endif
 };
 
-using IterativeSolver = IterativeSolverMP<real_t>;
 
 /** @brief Inner product operator constrained to a list of indices/dofs.
     The method Eval() computes the inner product of two vectors
@@ -631,50 +623,37 @@ void SLI(const Operator &A, Solver &B, const Vector &b, Vector &x,
 
 
 /// Conjugate gradient method
-template <class T>
-class CGSolverMP : public IterativeSolverMP<T>
+class CGSolver : public IterativeSolver
 {
 protected:
-   mutable VectorMP<T> r, d, z;
+   mutable Vector r, d, z;
 
    void UpdateVectors();
 
-   using IterativeSolverMP<T>::converged;
-   using IterativeSolverMP<T>::final_iter;
-   using IterativeSolverMP<T>::final_norm;
-   using IterativeSolverMP<T>::oper;
-   using IterativeSolverMP<T>::print_options;
-   using IterativeSolverMP<T>::max_iter;
-   using IterativeSolverMP<T>::prec;
-
 public:
-   CGSolverMP() { }
+   CGSolver() { }
 
 #ifdef MFEM_USE_MPI
-   CGSolverMP(MPI_Comm comm_) : IterativeSolverMP<T>(comm_) { }
+   CGSolver(MPI_Comm comm_) : IterativeSolver(comm_) { }
 #endif
 
-   void SetOperator(const OperatorMP<T> &op) override
-   { IterativeSolverMP<T>::SetOperator(op); UpdateVectors(); }
+   void SetOperator(const Operator &op) override
+   { IterativeSolver::SetOperator(op); UpdateVectors(); }
 
    /** @brief Iterative solution of the linear system using the Conjugate
        Gradient method. */
-   void Mult(const VectorMP<T> &b, VectorMP<T> &x) const override;
+   void Mult(const Vector &b, Vector &x) const override;
 };
 
-using CGSolver = CGSolverMP<real_t>;
-
 /// Conjugate gradient method. (tolerances are squared)
 void CG(const Operator &A, const Vector &b, Vector &x,
         int print_iter = 0, int max_num_iter = 1000,
         real_t RTOLERANCE = 1e-12, real_t ATOLERANCE = 1e-24);
 
 /// Preconditioned conjugate gradient method. (tolerances are squared)
-template <class T>
-void PCG(const OperatorMP<T> &A, SolverMP<T> &B, const VectorMP<T> &b,
-         VectorMP<T> &x,
+void PCG(const Operator &A, Solver &B, const Vector &b, Vector &x,
          int print_iter = 0, int max_num_iter = 1000,
-         double RTOLERANCE = 1e-12, double ATOLERANCE = 1e-24);
+         real_t RTOLERANCE = 1e-12, real_t ATOLERANCE = 1e-24);
 
 
 /// GMRES method
@@ -1173,7 +1152,7 @@ public:
 
 private:
    /// Set up the block CSR structure corresponding to a sparse matrix @a A
-   void CreateBlockPattern(const class SparseMatrixMP<real_t> &A);
+   void CreateBlockPattern(const class SparseMatrix &A);
 
    /// Perform the block ILU factorization
    void Factorize();

linalg/sparsemat.hpp

@@ -34,26 +34,22 @@
 namespace mfem
 {
 
-template <class T>
 class
 #if defined(__alignas_is_defined)
-   alignas(T)
+   alignas(real_t)
 #endif
    RowNode
 {
 public:
-   T Value;
+   real_t Value;
    RowNode *Prev;
    int Column;
 };
 
 /// Data type sparse matrix
-template <class T>
-class SparseMatrixMP : public AbstractSparseMatrixMP<T>
+class SparseMatrix : public AbstractSparseMatrix
 {
 protected:
-   using OperatorBase::height;
-   using OperatorBase::width;
    /// @name Arrays used by the CSR storage format.
    /** */
    ///@{
@@ -69,22 +65,22 @@ protected:
    Memory<int> J;
    /** @brief %Array with size #I[#height], containing the actual entries of the
        sparse matrix, as indexed by the #I array. */
-   Memory<T> A;
+   Memory<real_t> A;
    ///@}
 
    /** @brief %Array of linked lists, one for every row. This array represents
        the linked list (LIL) storage format. */
-   RowNode<T> **Rows;
+   RowNode **Rows;
 
    mutable int current_row;
    mutable int* ColPtrJ;
-   mutable RowNode<T> ** ColPtrNode;
+   mutable RowNode ** ColPtrNode;
 
    /// Transpose of A. Owned. Used to perform MultTranspose() on devices.
-   mutable SparseMatrixMP<T> *At;
+   mutable SparseMatrix *At;
 
 #ifdef MFEM_USE_MEMALLOC
-   typedef MemAlloc <RowNode<T>, 1024> RowNodeAlloc;
+   typedef MemAlloc <RowNode, 1024> RowNodeAlloc;
    RowNodeAlloc * NodesMem;
 #endif
 
@@ -140,7 +136,7 @@ protected:
 
 public:
    /// Create an empty SparseMatrix.
-   SparseMatrixMP()
+   SparseMatrix()
    {
       SetEmpty();
 
@@ -152,25 +148,25 @@ public:
    /** New entries are added as needed by methods like AddSubMatrix(),
        SetSubMatrix(), etc. Calling Finalize() will convert the SparseMatrix to
        the more compact compressed sparse row (CSR) format. */
-   explicit SparseMatrixMP(int nrows, int ncols = -1);
+   explicit SparseMatrix(int nrows, int ncols = -1);
 
    /** @brief Create a sparse matrix in CSR format. Ownership of @a i, @a j, and
        @a data is transferred to the SparseMatrix. */
-   SparseMatrixMP(int *i, int *j, T *data, int m, int n);
+   SparseMatrix(int *i, int *j, real_t *data, int m, int n);
 
    /** @brief Create a sparse matrix in CSR format. Ownership of @a i, @a j, and
        @a data is optionally transferred to the SparseMatrix. */
    /** If the parameter @a data is NULL, then the internal #A array is allocated
        by this constructor (initializing it with zeros and taking ownership,
        regardless of the parameter @a owna). */
-   SparseMatrixMP(int *i, int *j, T *data, int m, int n, bool ownij,
-                  bool owna, bool issorted);
+   SparseMatrix(int *i, int *j, real_t *data, int m, int n, bool ownij,
+                bool owna, bool issorted);
 
    /** @brief Create a sparse matrix in CSR format where each row has space
        allocated for exactly @a rowsize entries. */
    /** SetRow() can then be called or the #I, #J, #A arrays can be used
        directly. */
-   SparseMatrixMP(int nrows, int ncols, int rowsize);
+   SparseMatrix(int nrows, int ncols, int rowsize);
 
    /// Copy constructor (deep copy).
    /** If @a mat is finalized and @a copy_graph is false, the #I and #J arrays
@@ -180,16 +176,11 @@ public:
        SparseMatrix's #I, #J, and #A arrays will be the same as @a mat,
        otherwise the type will be @a mt for those arrays that are deep
        copied. */
-   SparseMatrixMP(const SparseMatrixMP<T> &mat, bool copy_graph = true,
-                  MemoryType mt = MemoryType::PRESERVE);
+   SparseMatrix(const SparseMatrix &mat, bool copy_graph = true,
+                MemoryType mt = MemoryType::PRESERVE);
 
    /// Create a SparseMatrix with diagonal @a v, i.e. A = Diag(v)
-   SparseMatrixMP(const VectorMP<T> & v);
-
-   using DiagonalPolicy = OperatorBase::DiagonalPolicy;
-   using OperatorBase::DIAG_ZERO;
-   using OperatorBase::DIAG_ONE;
-   using OperatorBase::DIAG_KEEP;
+   SparseMatrix(const Vector & v);
 
    /// @brief Sets the height and width of the matrix.
    /** @warning This does not modify in any way the underlying CSR or LIL
@@ -211,18 +202,16 @@ public:
    void UseCuSparse(bool useCuSparse_ = true) { UseGPUSparse(useCuSparse_); }
 
    /// Assignment operator: deep copy
-   SparseMatrixMP<T>& operator=(const SparseMatrixMP<T> &rhs);
+   SparseMatrix& operator=(const SparseMatrix &rhs);
 
    /** @brief Clear the contents of the SparseMatrix and make it a reference to
        @a master */
    /** After this call, the matrix will point to the same data as @a master but
        it will not own its data. The @a master must be finalized. */
-   void MakeRef(const SparseMatrixMP<T> &master);
-
-   //using int OperatorBase::Height();
+   void MakeRef(const SparseMatrix &master);
 
    /// For backward compatibility, define Size() to be synonym of Height().
-   int Size() const { return this->Height(); }
+   int Size() const { return Height(); }
 
    /// Clear the contents of the SparseMatrix.
    void Clear() { Destroy(); SetEmpty(); }
@@ -248,25 +237,25 @@ public:
    inline const int *GetJ() const { return J; }
 
    /// Return the element data, i.e. the array #A.
-   inline T *GetData() { return A; }
+   inline real_t *GetData() { return A; }
    /// Return the element data, i.e. the array #A, const version.
-   inline const T *GetData() const { return A; }
+   inline const real_t *GetData() const { return A; }
 
    // Memory access methods for the #I array.
    Memory<int> &GetMemoryI() { return I; }
    const Memory<int> &GetMemoryI() const { return I; }
    const int *ReadI(bool on_dev = true) const
-   { return mfem::Read(I, this->Height()+1, on_dev); }
+   { return mfem::Read(I, Height()+1, on_dev); }
    int *WriteI(bool on_dev = true)
-   { return mfem::Write(I, this->Height()+1, on_dev); }
+   { return mfem::Write(I, Height()+1, on_dev); }
    int *ReadWriteI(bool on_dev = true)
-   { return mfem::ReadWrite(I, this->Height()+1, on_dev); }
+   { return mfem::ReadWrite(I, Height()+1, on_dev); }
    const int *HostReadI() const
-   { return mfem::Read(I, this->Height()+1, false); }
+   { return mfem::Read(I, Height()+1, false); }
    int *HostWriteI()
-   { return mfem::Write(I, this->Height()+1, false); }
+   { return mfem::Write(I, Height()+1, false); }
    int *HostReadWriteI()
-   { return mfem::ReadWrite(I, this->Height()+1, false); }
+   { return mfem::ReadWrite(I, Height()+1, false); }
 
    // Memory access methods for the #J array.
    Memory<int> &GetMemoryJ() { return J; }
@@ -285,19 +274,19 @@ public:
    { return mfem::ReadWrite(J, J.Capacity(), false); }
 
    // Memory access methods for the #A array.
-   Memory<T> &GetMemoryData() { return A; }
-   const Memory<T> &GetMemoryData() const { return A; }
-   const T *ReadData(bool on_dev = true) const
+   Memory<real_t> &GetMemoryData() { return A; }
+   const Memory<real_t> &GetMemoryData() const { return A; }
+   const real_t *ReadData(bool on_dev = true) const
    { return mfem::Read(A, A.Capacity(), on_dev); }
-   T *WriteData(bool on_dev = true)
+   real_t *WriteData(bool on_dev = true)
    { return mfem::Write(A, A.Capacity(), on_dev); }
-   T *ReadWriteData(bool on_dev = true)
+   real_t *ReadWriteData(bool on_dev = true)
    { return mfem::ReadWrite(A, A.Capacity(), on_dev); }
-   const T *HostReadData() const
+   const real_t *HostReadData() const
    { return mfem::Read(A, A.Capacity(), false); }
-   T *HostWriteData()
+   real_t *HostWriteData()
    { return mfem::Write(A, A.Capacity(), false); }
-   T *HostReadWriteData()
+   real_t *HostReadWriteData()
    { return mfem::ReadWrite(A, A.Capacity(), false); }
 
    /// Returns the number of elements in row @a i.
@@ -312,9 +301,9 @@ public:
    const int *GetRowColumns(const int row) const;
 
    /// Return a pointer to the entries in a row.
-   T *GetRowEntries(const int row);
+   real_t *GetRowEntries(const int row);
    /// Return a pointer to the entries in a row, const version.
-   const T *GetRowEntries(const int row) const;
+   const real_t *GetRowEntries(const int row) const;
 
    /// Change the width of a SparseMatrix.
    /*!
@@ -338,16 +327,16 @@ public:
    void MoveDiagonalFirst();
 
    /// Returns reference to a_{ij}.
-   T &Elem(int i, int j) override;
+   real_t &Elem(int i, int j) override;
 
    /// Returns constant reference to a_{ij}.
-   const T &Elem(int i, int j) const override;
+   const real_t &Elem(int i, int j) const override;
 
    /// Returns reference to A[i][j].
-   T &operator()(int i, int j);
+   real_t &operator()(int i, int j);
 
    /// Returns reference to A[i][j].
-   const T &operator()(int i, int j) const;
+   const real_t &operator()(int i, int j) const;
 
    /// Returns the Diagonal of A
    void GetDiag(Vector & d) const;
@@ -365,24 +354,24 @@ public:
    }
 
    /// Matrix vector multiplication.
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   void Mult(const Vector &x, Vector &y) const override;
 
    /// y += A * x (default)  or  y += a * A * x
-   void AddMult(const VectorMP<T> &x, VectorMP<T> &y,
-                const T a = 1.0) const override;
+   void AddMult(const Vector &x, Vector &y,
+                const real_t a = 1.0) const override;
 
    /// Multiply a vector with the transposed matrix. y = At * x
    /** If the matrix is modified, call ResetTranspose() and optionally
        EnsureMultTranspose() to make sure this method uses the correct updated
        transpose. */
-   void MultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   void MultTranspose(const Vector &x, Vector &y) const override;
 
    /// y += At * x (default)  or  y += a * At * x
    /** If the matrix is modified, call ResetTranspose() and optionally
        EnsureMultTranspose() to make sure this method uses the correct updated
        transpose. */
-   void AddMultTranspose(const VectorMP<T> &x, VectorMP<T> &y,
-                         const T a = 1.0) const override;
+   void AddMultTranspose(const Vector &x, Vector &y,
+                         const real_t a = 1.0) const override;
 
    /** @brief Build and store internally the transpose of this matrix which will
        be used in the methods AddMultTranspose(), MultTranspose(), and
@@ -426,7 +415,7 @@ public:
 
    void PartMult(const Array<int> &rows, const Vector &x, Vector &y) const;
    void PartAddMult(const Array<int> &rows, const Vector &x, Vector &y,
-                    const T a=1.0) const;
+                    const real_t a=1.0) const;
 
    /// y = A * x, treating all entries as booleans (zero=false, nonzero=true).
    /** The actual values stored in the data array, #A, are not used - this means
@@ -441,27 +430,27 @@ public:
    void BooleanMultTranspose(const Array<int> &x, Array<int> &y) const;
 
    /// y = |A| * x, using entry-wise absolute values of matrix A
-   void AbsMult(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   void AbsMult(const Vector &x, Vector &y) const override;
 
    /// y = |At| * x, using entry-wise absolute values of the transpose of matrix A
    /** If the matrix is modified, call ResetTranspose() and optionally
        EnsureMultTranspose() to make sure this method uses the correct updated
        transpose. */
-   void AbsMultTranspose(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   void AbsMultTranspose(const Vector &x, Vector &y) const override;
 
    /// Compute y^t A x
-   T InnerProduct(const VectorMP<T> &x, const VectorMP<T> &y) const;
+   real_t InnerProduct(const Vector &x, const Vector &y) const;
 
    /// For all i compute $ x_i = \sum_j A_{ij} $
-   void GetRowSums(VectorMP<T> &x) const;
+   void GetRowSums(Vector &x) const;
    /// For i = irow compute $ x_i = \sum_j | A_{i, j} | $
-   T GetRowNorml1(int irow) const;
+   real_t GetRowNorml1(int irow) const;
 
    /// This virtual method is not supported: it always returns NULL.
-   MatrixInverseMP<T> *Inverse() const override;
+   MatrixInverse *Inverse() const override;
 
    /// Eliminates a column from the transpose matrix.
-   void EliminateRow(int row, const T sol, Vector &rhs);
+   void EliminateRow(int row, const real_t sol, Vector &rhs);
 
    /// Eliminates a row from the matrix.
    /*!
@@ -489,7 +478,7 @@ public:
 
    /** @brief Similar to EliminateCols + save the eliminated entries into
        @a Ae so that (*this) + Ae is equal to the original matrix. */
-   void EliminateCols(const Array<int> &col_marker, SparseMatrixMP &Ae);
+   void EliminateCols(const Array<int> &col_marker, SparseMatrix &Ae);
 
    /// Eliminate row @a rc and column @a rc and modify the @a rhs using @a sol.
    /** Eliminates the column @a rc to the @a rhs, deletes the row @a rc and
@@ -497,7 +486,7 @@ public:
        is assembled if and only if the element (rc,i) is assembled.
        By default, elements (rc,rc) are set to 1.0, although this behavior
        can be adjusted by changing the @a dpolicy parameter. */
-   void EliminateRowCol(int rc, const T sol, Vector &rhs,
+   void EliminateRowCol(int rc, const real_t sol, Vector &rhs,
                         DiagonalPolicy dpolicy = DIAG_ONE);
 
    /** @brief Similar to
@@ -509,7 +498,7 @@ public:
                                    DiagonalPolicy dpolicy = DIAG_ONE);
 
    /// Perform elimination and set the diagonal entry to the given value
-   void EliminateRowColDiag(int rc, T value);
+   void EliminateRowColDiag(int rc, real_t value);
 
    /// Eliminate row @a rc and column @a rc.
    void EliminateRowCol(int rc, DiagonalPolicy dpolicy = DIAG_ONE);
@@ -517,7 +506,7 @@ public:
    /** @brief Similar to EliminateRowCol(int, DiagonalPolicy) + save the
        eliminated entries into @a Ae so that (*this) + Ae is equal to the
        original matrix */
-   void EliminateRowCol(int rc, SparseMatrixMP &Ae,
+   void EliminateRowCol(int rc, SparseMatrix &Ae,
                         DiagonalPolicy dpolicy = DIAG_ONE);
 
    /** @brief Eliminate essential (Dirichlet) boundary conditions.
@@ -531,31 +520,31 @@ public:
    /// If a row contains only one diag entry of zero, set it to 1.
    void SetDiagIdentity();
    /// If a row contains only zeros, set its diagonal to 1.
-   void EliminateZeroRows(const T threshold = 1e-12) override;
+   void EliminateZeroRows(const real_t threshold = 1e-12) override;
 
    /// Gauss-Seidel forward and backward iterations over a vector x.
-   void Gauss_Seidel_forw(const VectorMP<T> &x, VectorMP<T> &y) const;
-   void Gauss_Seidel_back(const VectorMP<T> &x, VectorMP<T> &y) const;
+   void Gauss_Seidel_forw(const Vector &x, Vector &y) const;
+   void Gauss_Seidel_back(const Vector &x, Vector &y) const;
 
    /// Determine appropriate scaling for Jacobi iteration
-   T GetJacobiScaling() const;
+   real_t GetJacobiScaling() const;
    /** One scaled Jacobi iteration for the system A x = b.
        x1 = x0 + sc D^{-1} (b - A x0)  where D is the diag of A.
        Absolute values of D are used when use_abs_diag = true. */
    void Jacobi(const Vector &b, const Vector &x0, Vector &x1,
-               T sc, bool use_abs_diag = false) const;
+               real_t sc, bool use_abs_diag = false) const;
 
    /// x = sc b / A_ii. When use_abs_diag = true, |A_ii| is used.
    void DiagScale(const Vector &b, Vector &x,
-                  T sc = 1.0, bool use_abs_diag = false) const;
+                  real_t sc = 1.0, bool use_abs_diag = false) const;
 
    /** x1 = x0 + sc D^{-1} (b - A x0) where $ D_{ii} = \sum_j |A_{ij}| $. */
    void Jacobi2(const Vector &b, const Vector &x0, Vector &x1,
-                T sc = 1.0) const;
+                real_t sc = 1.0) const;
 
    /** x1 = x0 + sc D^{-1} (b - A x0) where $ D_{ii} = \sum_j A_{ij} $. */
    void Jacobi3(const Vector &b, const Vector &x0, Vector &x1,
-                T sc = 1.0) const;
+                real_t sc = 1.0) const;
 
    /** @brief Finalize the matrix initialization, switching the storage format
        from LIL to CSR. */
@@ -575,12 +564,12 @@ public:
    /** @brief Remove entries smaller in absolute value than a given tolerance
        @a tol. If @a fix_empty_rows is true, a zero value is inserted in the
        diagonal entry (for square matrices only) */
-   void Threshold(T tol, bool fix_empty_rows = false);
+   void Threshold(real_t tol, bool fix_empty_rows = false);
 
    /** Split the matrix into M x N blocks of sparse matrices in CSR format.
        The 'blocks' array is M x N (i.e. M and N are determined by its
        dimensions) and its entries are overwritten by the new blocks. */
-   void GetBlocks(Array2D<SparseMatrixMP *> &blocks) const;
+   void GetBlocks(Array2D<SparseMatrix *> &blocks) const;
 
    void GetSubMatrix(const Array<int> &rows, const Array<int> &cols,
                      DenseMatrix &subm) const;
@@ -599,24 +588,24 @@ public:
        SparseMatrix, it will be added to the sparsity pattern initialized with
        zero. If the matrix is finalized and the entry is not found, an error
        will be generated. */
-   inline T &SearchRow(const int col);
+   inline real_t &SearchRow(const int col);
    /// Add a value to an entry in the "current row". See SetColPtr().
-   inline void _Add_(const int col, const T a)
+   inline void _Add_(const int col, const real_t a)
    { SearchRow(col) += a; }
    /// Set an entry in the "current row". See SetColPtr().
-   inline void _Set_(const int col, const T a)
+   inline void _Set_(const int col, const real_t a)
    { SearchRow(col) = a; }
    /// Read the value of an entry in the "current row". See SetColPtr().
-   inline T _Get_(const int col) const;
+   inline real_t _Get_(const int col) const;
 
-   inline T &SearchRow(const int row, const int col);
-   inline void _Add_(const int row, const int col, const T a)
+   inline real_t &SearchRow(const int row, const int col);
+   inline void _Add_(const int row, const int col, const real_t a)
    { SearchRow(row, col) += a; }
-   inline void _Set_(const int row, const int col, const T a)
+   inline void _Set_(const int row, const int col, const real_t a)
    { SearchRow(row, col) = a; }
 
-   void Set(const int i, const int j, const T val);
-   void Add(const int i, const int j, const T val);
+   void Set(const int i, const int j, const real_t val);
+   void Add(const int i, const int j, const real_t val);
 
    void SetSubMatrix(const Array<int> &rows, const Array<int> &cols,
                      const DenseMatrix &subm, int skip_zeros = 1);
@@ -645,12 +634,12 @@ public:
          when the matrix is finalized.
        @warning This method breaks the const-ness when the matrix is finalized
        because it gives write access to the #J and #A arrays. */
-   int GetRow(const int row, Array<int> &cols, VectorMP<T> &srow) const override;
+   int GetRow(const int row, Array<int> &cols, Vector &srow) const override;
 
    void SetRow(const int row, const Array<int> &cols, const Vector &srow);
    void AddRow(const int row, const Array<int> &cols, const Vector &srow);
 
-   void ScaleRow(const int row, const T scale);
+   void ScaleRow(const int row, const real_t scale);
    /// this = diag(sl) * this;
    void ScaleRows(const Vector & sl);
    /// this = this * diag(sr);
@@ -658,15 +647,15 @@ public:
 
    /** @brief Add the sparse matrix 'B' to '*this'. This operation will cause an
        error if '*this' is finalized and 'B' has larger sparsity pattern. */
-   SparseMatrixMP &operator+=(const SparseMatrixMP &B);
+   SparseMatrix &operator+=(const SparseMatrix &B);
 
    /** @brief Add the sparse matrix 'B' scaled by the scalar 'a' into '*this'.
        Only entries in the sparsity pattern of '*this' are added. */
-   void Add(const T a, const SparseMatrixMP &B);
+   void Add(const real_t a, const SparseMatrix &B);
 
-   SparseMatrixMP &operator=(T a);
+   SparseMatrix &operator=(real_t a);
 
-   SparseMatrixMP &operator*=(T a);
+   SparseMatrix &operator*=(real_t a);
 
    /// Prints matrix to stream out.
    /** @note The host in synchronized when the finalized matrix is on the device. */
@@ -705,7 +694,7 @@ public:
    void PrintInfo(std::ostream &out) const;
 
    /// Returns max_{i,j} |(i,j)-(j,i)| for a finalized matrix
-   T IsSymmetric() const;
+   real_t IsSymmetric() const;
 
    /// (*this) = 1/2 ((*this) + (*this)^t)
    void Symmetrize();
@@ -713,10 +702,10 @@ public:
    /// Returns the number of the nonzero elements in the matrix
    int NumNonZeroElems() const override;
 
-   T MaxNorm() const;
+   real_t MaxNorm() const;
 
    /// Count the number of entries with |a_ij| <= tol.
-   int CountSmallElems(T tol) const;
+   int CountSmallElems(real_t tol) const;
 
    /// Count the number of entries that are NOT finite, i.e. Inf or Nan.
    int CheckFinite() const;
@@ -737,18 +726,14 @@ public:
    /// Lose the ownership of the graph (I, J) and data (A) arrays.
    void LoseData() { SetGraphOwner(false); SetDataOwner(false); }
 
-   void Swap(SparseMatrixMP &other);
+   void Swap(SparseMatrix &other);
 
    /// Destroys sparse matrix.
-   virtual ~SparseMatrixMP();
+   virtual ~SparseMatrix();
 
-   using Type = OperatorBase::Type;
-
-   Type GetType() const { return OperatorBase::MFEM_SPARSEMAT; }
+   Type GetType() const { return MFEM_SPARSEMAT; }
 };
 
-using SparseMatrix = SparseMatrixMP<real_t>;
-
 inline std::ostream& operator<<(std::ostream& os, SparseMatrix const& mat)
 {
    mat.Print(os);
@@ -756,17 +741,14 @@ inline std::ostream& operator<<(std::ostream& os, SparseMatrix const& mat)
 }
 
 /// Applies f() to each element of the matrix (after it is finalized).
-template <class T>
-void SparseMatrixFunction(SparseMatrixMP<T> &S, T (*f)(T));
+void SparseMatrixFunction(SparseMatrix &S, real_t (*f)(real_t));
 
 
 /// Transpose of a sparse matrix. A must be finalized.
-template <class T>
-SparseMatrixMP<T> *Transpose(const SparseMatrixMP<T> &A);
+SparseMatrix *Transpose(const SparseMatrix &A);
 /// Transpose of a sparse matrix. A does not need to be a CSR matrix.
-template <class T>
-SparseMatrixMP<T> *TransposeAbstractSparseMatrix(const AbstractSparseMatrix &A,
-                                                 int useActualWidth);
+SparseMatrix *TransposeAbstractSparseMatrix (const AbstractSparseMatrix &A,
+                                             int useActualWidth);
 
 /// Matrix product A.B.
 /** If @a OAB is not NULL, we assume it has the structure of A.B and store the
@@ -774,100 +756,78 @@ SparseMatrixMP<T> *TransposeAbstractSparseMatrix(const AbstractSparseMatrix &A,
     the result and return a pointer to it.
 
     All matrices must be finalized. */
-template <class T>
-SparseMatrixMP<T> *Mult(const SparseMatrixMP<T> &A, const SparseMatrixMP<T> &B,
-                        SparseMatrixMP<T> *OAB = NULL);
+SparseMatrix *Mult(const SparseMatrix &A, const SparseMatrix &B,
+                   SparseMatrix *OAB = NULL);
 
 /// C = A^T B
-template <class T>
-SparseMatrixMP<T> *TransposeMult(const SparseMatrixMP<T> &A,
-                                 const SparseMatrixMP<T> &B);
+SparseMatrix *TransposeMult(const SparseMatrix &A, const SparseMatrix &B);
 
 /// Matrix product of sparse matrices. A and B do not need to be CSR matrices
-template <class T>
-SparseMatrixMP<T> *MultAbstractSparseMatrix(const AbstractSparseMatrix &A,
-                                            const AbstractSparseMatrix &B);
+SparseMatrix *MultAbstractSparseMatrix (const AbstractSparseMatrix &A,
+                                        const AbstractSparseMatrix &B);
 
 /// Matrix product A.B
-template <class T>
-DenseMatrix *Mult(const SparseMatrixMP<T> &A, DenseMatrix &B);
+DenseMatrix *Mult(const SparseMatrix &A, DenseMatrix &B);
 
 /// RAP matrix product (with R=P^T)
-template <class T>
-DenseMatrix *RAP(const SparseMatrixMP<T> &A, DenseMatrix &P);
+DenseMatrix *RAP(const SparseMatrix &A, DenseMatrix &P);
 
 /// RAP matrix product (with R=P^T)
-template <class T>
-DenseMatrix *RAP(DenseMatrix &A, const SparseMatrixMP<T> &P);
+DenseMatrix *RAP(DenseMatrix &A, const SparseMatrix &P);
 
 /** RAP matrix product (with P=R^T). ORAP is like OAB above.
     All matrices must be finalized. */
-template <class T>
-SparseMatrixMP<T> *RAP(const SparseMatrixMP<T> &A, const SparseMatrixMP<T> &R,
-                       SparseMatrixMP<T> *ORAP = NULL);
+SparseMatrix *RAP(const SparseMatrix &A, const SparseMatrix &R,
+                  SparseMatrix *ORAP = NULL);
 
 /// General RAP with given R^T, A and P
-template <class T>
-SparseMatrixMP<T> *RAP(const SparseMatrixMP<T> &Rt, const SparseMatrixMP<T> &A,
-                       const SparseMatrixMP<T> &P);
+SparseMatrix *RAP(const SparseMatrix &Rt, const SparseMatrix &A,
+                  const SparseMatrix &P);
 
 /// Matrix multiplication A^t D A. All matrices must be finalized.
-template <class T>
-SparseMatrixMP<T> *Mult_AtDA(const SparseMatrixMP<T> &A, const Vector &D,
-                             SparseMatrixMP<T> *OAtDA = NULL);
+SparseMatrix *Mult_AtDA(const SparseMatrix &A, const Vector &D,
+                        SparseMatrix *OAtDA = NULL);
 
 
 /// Matrix addition result = A + B.
-template <class T>
-SparseMatrixMP<T> * Add(const SparseMatrixMP<T> & A,
-                        const SparseMatrixMP<T> & B);
+SparseMatrix * Add(const SparseMatrix & A, const SparseMatrix & B);
 /// Matrix addition result = a*A + b*B
-template <class T, class U>
-SparseMatrixMP<T> * Add(U a, const SparseMatrixMP<T> & A, U b,
-                        const SparseMatrixMP<T> & B);
+SparseMatrix * Add(real_t a, const SparseMatrix & A, real_t b,
+                   const SparseMatrix & B);
 /// Matrix addition result = sum_i A_i
-template <class T>
-SparseMatrixMP<T> * Add(Array<SparseMatrixMP<T> *> & Ai);
+SparseMatrix * Add(Array<SparseMatrix *> & Ai);
 
 /// B += alpha * A
-template <class T, class U>
-void Add(const SparseMatrixMP<T> &A, U alpha, DenseMatrix &B);
+void Add(const SparseMatrix &A, real_t alpha, DenseMatrix &B);
 
 /// Produces a block matrix with blocks A_{ij}*B
 DenseMatrix *OuterProduct(const DenseMatrix &A, const DenseMatrix &B);
 
 /// Produces a block matrix with blocks A_{ij}*B
-template <class T>
-SparseMatrixMP<T> *OuterProduct(const DenseMatrix &A,
-                                const SparseMatrixMP<T> &B);
+SparseMatrix *OuterProduct(const DenseMatrix &A, const SparseMatrix &B);
 
 /// Produces a block matrix with blocks A_{ij}*B
-template <class T>
-SparseMatrixMP<T> *OuterProduct(const SparseMatrixMP<T> &A,
-                                const DenseMatrix &B);
+SparseMatrix *OuterProduct(const SparseMatrix &A, const DenseMatrix &B);
 
 /// Produces a block matrix with blocks A_{ij}*B
-template <class T>
-SparseMatrixMP<T> *OuterProduct(const SparseMatrixMP<T> &A,
-                                const SparseMatrixMP<T> &B);
+SparseMatrix *OuterProduct(const SparseMatrix &A, const SparseMatrix &B);
 
 
 // Inline methods
 
-template <class T>
-inline void SparseMatrixMP<T>::SetColPtr(const int row) const
+inline void SparseMatrix::SetColPtr(const int row) const
 {
    if (Rows)
    {
       if (ColPtrNode == NULL)
       {
-         ColPtrNode = new RowNode<T> *[width];
+         ColPtrNode = new RowNode *[width];
          for (int i = 0; i < width; i++)
          {
             ColPtrNode[i] = NULL;
          }
       }
-      for (RowNode<T> *node_p = Rows[row]; node_p != NULL; node_p = node_p->Prev)
+      for (RowNode *node_p = Rows[row]; node_p != NULL; node_p = node_p->Prev)
       {
          ColPtrNode[node_p->Column] = node_p;
       }
@@ -890,12 +850,11 @@ inline void SparseMatrixMP<T>::SetColPtr(const int row) const
    current_row = row;
 }
 
-template <class T>
-inline void SparseMatrixMP<T>::ClearColPtr() const
+inline void SparseMatrix::ClearColPtr() const
 {
    if (Rows)
    {
-      for (RowNode<T> *node_p = Rows[current_row]; node_p != NULL;
+      for (RowNode *node_p = Rows[current_row]; node_p != NULL;
            node_p = node_p->Prev)
       {
          ColPtrNode[node_p->Column] = NULL;
@@ -910,18 +869,17 @@ inline void SparseMatrixMP<T>::ClearColPtr() const
    }
 }
 
-template <class T>
-inline T &SparseMatrixMP<T>::SearchRow(const int col)
+inline real_t &SparseMatrix::SearchRow(const int col)
 {
    if (Rows)
    {
-      RowNode<T> *node_p = ColPtrNode[col];
+      RowNode *node_p = ColPtrNode[col];
       if (node_p == NULL)
       {
 #ifdef MFEM_USE_MEMALLOC
          node_p = NodesMem->Alloc();
 #else
-         node_p = new RowNode<T>;
+         node_p = new RowNode;
 #endif
          node_p->Prev = Rows[current_row];
          node_p->Column = col;
@@ -938,12 +896,11 @@ inline T &SparseMatrixMP<T>::SearchRow(const int col)
    }
 }
 
-template <class T>
-inline T SparseMatrixMP<T>::_Get_(const int col) const
+inline real_t SparseMatrix::_Get_(const int col) const
 {
    if (Rows)
    {
-      RowNode<T> *node_p = ColPtrNode[col];
+      RowNode *node_p = ColPtrNode[col];
       return (node_p == NULL) ? 0.0 : node_p->Value;
    }
    else
@@ -953,12 +910,11 @@ inline T SparseMatrixMP<T>::_Get_(const int col) const
    }
 }
 
-template <class T>
-inline T &SparseMatrixMP<T>::SearchRow(const int row, const int col)
+inline real_t &SparseMatrix::SearchRow(const int row, const int col)
 {
    if (Rows)
    {
-      RowNode<T> *node_p;
+      RowNode *node_p;
 
       for (node_p = Rows[row]; 1; node_p = node_p->Prev)
       {
@@ -967,7 +923,7 @@ inline T &SparseMatrixMP<T>::SearchRow(const int row, const int col)
 #ifdef MFEM_USE_MEMALLOC
             node_p = NodesMem->Alloc();
 #else
-            node_p = new RowNode<T>;
+            node_p = new RowNode;
 #endif
             node_p->Prev = Rows[row];
             node_p->Column = col;
@@ -998,7 +954,7 @@ inline T &SparseMatrixMP<T>::SearchRow(const int row, const int col)
 }
 
 /// Specialization of the template function Swap<> for class SparseMatrix
-template<class T> inline void Swap(SparseMatrixMP<T> &a, SparseMatrixMP<T> &b)
+template<> inline void Swap<SparseMatrix>(SparseMatrix &a, SparseMatrix &b)
 {
    a.Swap(b);
 }

linalg/sparsesmoothers.cpp

@@ -20,23 +20,37 @@
 namespace mfem
 {
 
-template <class T>
-void SparseSmootherMP<T>::SetOperator(const OperatorMP<T> &a)
+void SparseSmoother::SetOperator(const Operator &a)
 {
-   oper = dynamic_cast<const SparseMatrixMP<T>*>(&a);
-   if (oper == NULL)
-   {
-      mfem_error("SparseSmoother::SetOperator : not a SparseMatrix!");
-   }
-   this->height = oper->Height();
-   this->width = oper->Width();
+   oper = dynamic_cast<const SparseMatrix*>(&a);
+   MFEM_VERIFY(oper != nullptr, "Operator must be a SparseMatrix");
+   height = oper->Height();
+   width = oper->Width();
+
+   At.reset();
+   oper_T = nullptr;
 }
 
-/// Matrix vector multiplication with GS Smoother.
-template <class T>
-void GSSmootherMP<T>::Mult(const VectorMP<T> &x, VectorMP<T> &y) const
+void SparseSmoother::EnsureTranspose() const
 {
-   if (!this->iterative_mode)
+   if (oper_T) { return; }
+
+   const real_t tol = 1e-14;
+   if (oper->IsSymmetric() > tol * oper->MaxNorm())
+   {
+      At.reset(Transpose(*oper));
+      oper_T = At.get();
+   }
+   else
+   {
+      At.reset();
+      oper_T = oper;
+   }
+}
+
+void GSSmoother::Mult(const Vector &x, Vector &y) const
+{
+   if (!iterative_mode)
    {
       y = 0.0;
    }
@@ -44,30 +58,42 @@ void GSSmootherMP<T>::Mult(const VectorMP<T> &x, VectorMP<T> &y) const
    {
       if (type != 2)
       {
-         this->oper->Gauss_Seidel_forw(x, y);
+         oper->Gauss_Seidel_forw(x, y);
       }
       if (type != 1)
       {
-         this->oper->Gauss_Seidel_back(x, y);
+         oper->Gauss_Seidel_back(x, y);
       }
    }
 }
 
-/// Create the Jacobi smoother.
-DSmoother::DSmoother(const SparseMatrix &a, int t, real_t s, int it)
-   : SparseSmoother(a)
+void GSSmoother::MultTranspose(const Vector &x, Vector &y) const
 {
-   type = t;
-   scale = s;
-   iterations = it;
+   EnsureTranspose();
+
+   if (!iterative_mode)
+   {
+      y = 0.0;
+   }
+
+   for (int i = 0; i < iterations; i++)
+   {
+      if (type != 1)
+      {
+         oper_T->Gauss_Seidel_forw(x, y);
+      }
+      if (type != 2)
+      {
+         oper_T->Gauss_Seidel_back(x, y);
+      }
+   }
 }
 
-/// Matrix vector multiplication with Jacobi smoother.
-void DSmoother::Mult(const Vector &x, Vector &y) const
+void DSmoother::Mult_(const SparseMatrix &A, const Vector &x, Vector &y) const
 {
    if (!iterative_mode && type == 0 && iterations == 1)
    {
-      oper->DiagScale(x, y, scale, use_abs_diag);
+      A.DiagScale(x, y, scale, use_abs_diag);
       return;
    }
 
@@ -92,25 +118,41 @@ void DSmoother::Mult(const Vector &x, Vector &y) const
    {
       if (type == 0)
       {
-         oper->Jacobi(x, *p, *r, scale, use_abs_diag);
+         A.Jacobi(x, *p, *r, scale, use_abs_diag);
       }
       else if (type == 1)
       {
-         oper->Jacobi2(x, *p, *r, scale);
+         A.Jacobi2(x, *p, *r, scale);
       }
       else if (type == 2)
       {
-         oper->Jacobi3(x, *p, *r, scale);
+         A.Jacobi3(x, *p, *r, scale);
       }
       else
       {
-         mfem_error("DSmoother::Mult wrong type");
+         MFEM_ABORT("Invalid type.");
       }
       Swap<Vector*>(r, p);
    }
 }
 
-template class GSSmootherMP<float>;
-template class GSSmootherMP<double>;
+void DSmoother::Mult(const Vector &x, Vector &y) const
+{
+   Mult_(*oper, x, y);
+}
+
+void DSmoother::MultTranspose(const Vector &x, Vector &y) const
+{
+   if (iterations == 1 && !iterative_mode)
+   {
+      Mult_(*oper, x, y);
+      return;
+   }
+
+   EnsureTranspose();
+   MFEM_VERIFY(type == 0 || !At, "l1 or lumped Jacobi transpose not implemented"
+               " for non-symmetric matrices");
+   Mult_(*oper_T, x, y);
+}
 
 }

linalg/sparsesmoothers.hpp

@@ -15,74 +15,159 @@
 #include "../config/config.hpp"
 #include "sparsemat.hpp"
 
+#include <memory>
+
 namespace mfem
 {
 
-template <class T>
-class SparseSmootherMP : public MatrixInverseMP<T>
+/// Abstract base class for smoothers created from a SparseMatrix.
+class SparseSmoother : public MatrixInverse
 {
 protected:
-   const SparseMatrixMP<T> *oper;
+   const SparseMatrix *oper = nullptr; ///< The underlying matrix.
+
+   /// Pointer to the transpose of the underlying matrix. If the matrix is
+   /// symmetric, this will be the same as @a oper. If the matrix is not
+   /// symmetric, the transpose will be formed and stored in @a At. The
+   /// transpose will only be formed if MultTranspose() is called.
+   mutable const SparseMatrix *oper_T = nullptr;
+
+   mutable std::unique_ptr<SparseMatrix> At; ///< Transpose of A, if needed.
+
+   void EnsureTranspose() const; ///< Ensure that the transpose is set.
 
 public:
-   SparseSmootherMP() { oper = NULL; }
+   SparseSmoother() = default;
 
-   SparseSmootherMP(const SparseMatrixMP<T> &a)
-      : MatrixInverseMP<T>(a) { oper = &a; }
+   SparseSmoother(const SparseMatrix &a) { SetOperator(a); }
 
-   void SetOperator(const OperatorMP<T> &a) override;
+   /// Sets the underlying matrix. @a a must be a SparseMatrix.
+   void SetOperator(const Operator &a) override;
 };
 
-using SparseSmoother = SparseSmootherMP<real_t>;
-
-/// Data type for Gauss-Seidel smoother of sparse matrix
-template <class T>
-class GSSmootherMP : public SparseSmootherMP<T>
+/// Gauss-Seidel smoother of a sparse matrix.
+class GSSmoother : public SparseSmoother
 {
+public:
+   enum GSType
+   {
+      SYMMETRIC, ///< Forward Gauss-Seidel, then backward.
+      FORWARD, ///< Forward Gauss-Seidel ($L^{-1}$).
+      BACKWARD ///< Backward Gauss-Seidel ($U^{-1}$).
+   };
 protected:
-   int type; // 0, 1, 2 - symmetric, forward, backward
-   int iterations;
+   GSType type; ///< Type of Gauss-Seidel, see GSSmoother::GSType.
+   int iterations; ///< Number of stationary iterations.
 
 public:
-   /// Create GSSmoother.
-   GSSmootherMP(int t = 0, int it = 1) { type = t; iterations = it; }
+   /// @brief Create a Gauss-Seidel smoother. SetOperator() will need to be
+   /// called with a SparseMatrix before first use.
+   ///
+   /// @param[in]  t        Type of GS smoother (see GSSmoother::GSType)
+   /// @param[in]  it       Number of stationary iterations to perform
+   GSSmoother(GSType t = SYMMETRIC, int it = 1) { type = t; iterations = it; }
 
-   /// Create GSSmoother.
-   GSSmootherMP(const SparseMatrixMP<T> &a, int t = 0, int it = 1)
-      : SparseSmootherMP<T>(a) { type = t; iterations = it; }
+   /// @brief Create a Jacobi smoother using the SparseMatrix @a a.
+   ///
+   /// @param[in]  a        The underlying SparseMatrix
+   /// @param[in]  t        Type of GS smoother (see GSSmoother::GSType)
+   /// @param[in]  it       Number of stationary iterations to perform
+   GSSmoother(const SparseMatrix &a, GSType t = SYMMETRIC, int it = 1)
+      : GSSmoother(t, it) { SetOperator(a); }
 
-   /// Matrix vector multiplication with GS Smoother.
-   void Mult(const VectorMP<T> &x, VectorMP<T> &y) const override;
+   /// Same as GSSmoother(GSType,int), for backwards compatibility.
+   GSSmoother(int t, int it = 1) : GSSmoother(GSType(t), it) { }
+
+   /// @brief Same as GSSmoother(const SparseMatrix&,GSType,int), for
+   /// backwards compatibility.
+   GSSmoother(const SparseMatrix &a, int t, int it = 1)
+      : GSSmoother(a, GSType(t), it) { }
+
+   /// @brief Application of the Gauss-Seidel smoother.
+   ///
+   /// Applies a stationary Gauss-Seidel iteration. If Solver::iterative_mode is
+   /// true, then @a y is used as the initial guess, and Gauss-Seidel is applied
+   /// to the residual $x - Ay$.
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /// Application of the transpose of the Gauss-Seidel smoother.
+   void MultTranspose(const Vector &x, Vector &y) const override;
 };
 
-using GSSmoother = GSSmootherMP<real_t>;
-
-
-/// Data type for scaled Jacobi-type smoother of sparse matrix
+/// Jacobi-type diagonal smoother of a sparse matrix.
 class DSmoother : public SparseSmoother
 {
+public:
+   enum JacobiType
+   {
+      JACOBI,       ///< Scale by the diagonal of the matrix.
+      L1_JACOBI,    ///< Scale by the l1-norm of the rows.
+      LUMPED_JACOBI ///< Scale by the sum of the rows.
+   };
 protected:
-   int type; // 0, 1, 2 - scaled Jacobi, scaled l1-Jacobi, scaled lumped-Jacobi
-   real_t scale;
-   int iterations;
-   /// Uses abs values of the diagonal entries. Relevant only when type = 0.
+   JacobiType type; ///< Type of diagonal scaling, see DSmoother::JacobiType.
+   real_t scale; ///< Scaling (damping) factor.
+   int iterations; ///< Number of stationary iterations to perform.
+
+   /// @brief Uses abs values of the diagonal entries. Relevant only with type
+   /// JacobiType::JACOBI.
    bool use_abs_diag = false;
 
-   mutable Vector z;
+   mutable Vector z; ///< Temporary work vector.
+
+   /// Apply the Jacobi smoother (used internally by Mult() and MultTranspose())
+   void Mult_(const SparseMatrix &A, const Vector &x, Vector &y) const;
 
 public:
-   /// Create Jacobi smoother.
-   DSmoother(int t = 0, real_t s = 1., int it = 1)
+   /// @brief Create a Jacobi smoother. SetOperator() will need to be called
+   /// with a SparseMatrix before first use.
+   ///
+   /// @param[in]  t        Type of Jacobi smoother (see DSmoother::JacobiType)
+   /// @param[in]  s        Scaling factor
+   /// @param[in]  it       Number of stationary iterations to perform
+   DSmoother(JacobiType t = JACOBI, real_t s = 1., int it = 1)
    { type = t; scale = s; iterations = it; }
 
-   /// Create Jacobi smoother.
-   DSmoother(const SparseMatrix &a, int t = 0, real_t s = 1., int it = 1);
+   /// @brief Create a Jacobi smoother using the SparseMatrix @a a.
+   ///
+   /// @param[in]  a        The underlying SparseMatrix
+   /// @param[in]  t        Type of Jacobi smoother (see DSmoother::JacobiType)
+   /// @param[in]  s        Scaling factor
+   /// @param[in]  it       Number of stationary iterations to perform
+   DSmoother(const SparseMatrix &a, JacobiType t = JACOBI, real_t s = 1.,
+             int it = 1) : DSmoother(t, s, it) { SetOperator(a); }
 
-   /// Replace diag entries with their abs values. Relevant only when type = 0.
+   /// @brief Same as DSmoother(JacobiType,real_t,int), for backwards compatbility.
+   DSmoother(int t, real_t s = 1., int it = 1)
+      : DSmoother(JacobiType(t), s, it) { }
+
+   /// @brief Same as DSmoother(const SparseMatrix&,JacobiType,real_t,int), for
+   /// backwards compatbility.
+   DSmoother(const SparseMatrix &a, int t, real_t s = 1., int it = 1)
+      : DSmoother(a, JacobiType(t), s, it) { }
+
+   /// @brief Replace diagonal entries with their absolute values. Relevant only
+   /// with JacobiType::JACOBI.
    void SetPositiveDiagonal(bool pos_diag = true) { use_abs_diag = pos_diag; }
 
-   /// Matrix vector multiplication with Jacobi smoother.
+   /// @brief Apply the Jacobi smoother.
+   ///
+   /// Applies a stationary iteration with diagonal scaling. If
+   /// Solver::iterative_mode is true, then @a y is used as the initial guess
+   /// (and the diagonal scaling is applied to the residual $x - Ay$, giving
+   /// $D^{-1}(x - Ay)$).
+   ///
+   /// By default, Solver::iterative_mode is false and only one iteration is
+   /// performed, corresponding to $y = D^{-1}x$.
    void Mult(const Vector &x, Vector &y) const override;
+
+   /// @brief Apply the transpose of the Jacobi smoother.
+   ///
+   /// If the underlying matrix is symmetric, or if only one iteration is
+   /// performed with zero initial guess (Solver::iterative_mode is false), then
+   /// this is the same as Mult(). For non-symmetric matrices with iteration
+   /// count greater than one, only JacobiType::JACOBI is supported.
+   void MultTranspose(const Vector &x, Vector &y) const override;
 };
 
 }

linalg/vector.hpp

@@ -40,8 +40,7 @@ namespace mfem
 
 /** Count the number of entries in an array of doubles for which isfinite
     is false, i.e. the entry is a NaN or +/-Inf. */
-template <class T>
-inline int CheckFinite(const T *v, const int n);
+inline int CheckFinite(const real_t *v, const int n);
 
 /// Define a shortcut for std::numeric_limits<double>::infinity()
 #ifndef __CYGWIN__
@@ -78,84 +77,60 @@ inline real_t rand_real()
 #endif
 }
 
-template <class T>
-class VectorMP;
-
-template <class T>
-void add(const VectorMP<T> &v1, const VectorMP<T> &v2, VectorMP<T> &v);
-
-template <class T, class U>
-void add(const VectorMP<T> &v1, U alpha, const VectorMP<T> &v2, VectorMP<T> &v);
-
-template <class T, class U>
-void add(const U a, const VectorMP<T> &x, const VectorMP<T> &y, VectorMP<T> &z);
-
-template <class T, class U>
-void add(const U a, const VectorMP<T> &x,
-         const U b, const VectorMP<T> &y, VectorMP<T> &z);
-
-template <class T>
-void subtract(const VectorMP<T> &x, const VectorMP<T> &y, VectorMP<T> &z);
-
-template <class T, class U>
-void subtract(const U a, const VectorMP<T> &x, const VectorMP<T> &y,
-              VectorMP<T> &z);
-
 /// Vector data type.
-template <class T>
-class VectorMP
+class Vector
 {
 protected:
 
-   Memory<T> data;
+   Memory<real_t> data;
    int size;
 
 public:
 
    /** Default constructor for Vector. Sets size = 0, and calls Memory::Reset on
        data through Memory<double>'s default constructor. */
-   VectorMP(): size(0) { }
+   Vector(): size(0) { }
 
    /// Copy constructor. Allocates a new data array and copies the data.
-   VectorMP(const VectorMP<T> &);
+   Vector(const Vector &);
 
    /// Move constructor. "Steals" data from its argument.
-   VectorMP(VectorMP<T>&& v);
+   Vector(Vector&& v);
 
    /// @brief Creates vector of size s.
    /// @warning Entries are not initialized to zero!
-   explicit VectorMP(int s);
+   explicit Vector(int s);
 
    /// Creates a vector referencing an array of doubles, owned by someone else.
    /** The pointer @a data_ can be NULL. The data array can be replaced later
        with SetData(). */
-   VectorMP(T *data_, int size_)
+   Vector(real_t *data_, int size_)
    { data.Wrap(data_, size_, false); size = size_; }
 
    /** @brief Create a Vector referencing a sub-vector of the Vector @a base
        starting at the given offset, @a base_offset, and size @a size_. */
-   VectorMP(VectorMP<T> &base, int base_offset, int size_)
+   Vector(Vector &base, int base_offset, int size_)
       : data(base.data, base_offset, size_), size(size_) { }
 
    /// Create a Vector of size @a size_ using MemoryType @a mt.
-   VectorMP(int size_, MemoryType mt)
+   Vector(int size_, MemoryType mt)
       : data(size_, mt), size(size_) { }
 
    /** @brief Create a Vector of size @a size_ using host MemoryType @a h_mt and
        device MemoryType @a d_mt. */
-   VectorMP(int size_, MemoryType h_mt, MemoryType d_mt)
+   Vector(int size_, MemoryType h_mt, MemoryType d_mt)
       : data(size_, h_mt, d_mt), size(size_) { }
 
    /// Create a vector from a statically sized C-style array of convertible type
    template <typename CT, int N>
-   explicit VectorMP(const CT (&values)[N]) : VectorMP(N)
+   explicit Vector(const CT (&values)[N]) : Vector(N)
    { std::copy(values, values + N, begin()); }
 
    /// Create a vector using a braced initializer list
    template <typename CT, typename std::enable_if<
-                std::is_convertible<CT,T>::value,bool>::type = true>
-   explicit VectorMP(std::initializer_list<CT> values) :
-      VectorMP(static_cast<int> (values.size()))
+                std::is_convertible<CT,real_t>::value,bool>::type = true>
+   explicit Vector(std::initializer_list<CT> values) :
+      Vector(static_cast<int> (values.size()))
    { std::copy(values.begin(), values.end(), begin()); }
 
    /// Enable execution of Vector operations using the mfem::Device.
@@ -195,7 +170,7 @@ public:
    void SetSize(int s, MemoryType mt);
 
    /// Resize the vector to size @a s using the MemoryType of @a v.
-   void SetSize(int s, const VectorMP<T> &v) { SetSize(s, v.GetMemory().GetMemoryType()); }
+   void SetSize(int s, const Vector &v) { SetSize(s, v.GetMemory().GetMemoryType()); }
 
    /// Update \ref Capacity() to @a res (if less than current), keeping existing entries.
    void Reserve(int res);
@@ -206,20 +181,20 @@ public:
 
    /// Set the Vector data.
    /// @warning This method should be called only when OwnsData() is false.
-   void SetData(T *d) { data.Wrap(d, data.Capacity(), false); }
+   void SetData(real_t *d) { data.Wrap(d, data.Capacity(), false); }
 
    /// Set the Vector data and size.
    /** The Vector does not assume ownership of the new data. The new size is
        also used as the new Capacity().
        @warning This method should be called only when OwnsData() is false.
        @sa NewDataAndSize(). */
-   void SetDataAndSize(T *d, int s) { data.Wrap(d, s, false); size = s; }
+   void SetDataAndSize(real_t *d, int s) { data.Wrap(d, s, false); size = s; }
 
    /// Set the Vector data and size, deleting the old data, if owned.
    /** The Vector does not assume ownership of the new data. The new size is
        also used as the new Capacity().
        @sa SetDataAndSize(). */
-   void NewDataAndSize(T *d, int s)
+   void NewDataAndSize(real_t *d, int s)
    {
       data.Delete();
       SetDataAndSize(d, s);
@@ -234,14 +209,14 @@ public:
        the Vector object takes ownership of all pointers owned by @a mem.
 
        @sa NewDataAndSize(). */
-   inline void NewMemoryAndSize(const Memory<T> &mem, int s, bool own_mem);
+   inline void NewMemoryAndSize(const Memory<real_t> &mem, int s, bool own_mem);
 
    /// Reset the Vector to be a reference to a sub-vector of @a base.
-   inline void MakeRef(VectorMP<T> &base, int offset, int size);
+   inline void MakeRef(Vector &base, int offset, int size);
 
    /** @brief Reset the Vector to be a reference to a sub-vector of @a base
        without changing its current size. */
-   inline void MakeRef(VectorMP<T> &base, int offset);
+   inline void MakeRef(Vector &base, int offset);
 
    /// Set the Vector data (host pointer) ownership flag.
    void MakeDataOwner() const { data.SetHostPtrOwner(true); }
@@ -265,72 +240,72 @@ public:
    /// Return a pointer to the beginning of the Vector data.
    /** @warning This method should be used with caution as it gives write access
        to the data of const-qualified Vector%s. */
-   inline T *GetData() const
-   { return const_cast<T*>((const T*)data); }
+   inline real_t *GetData() const
+   { return const_cast<real_t*>((const real_t*)data); }
 
    /// Conversion to `double *`. Deprecated.
-   MFEM_DEPRECATED inline operator T *() { return data; }
+   MFEM_DEPRECATED inline operator real_t *() { return data; }
 
    /// Conversion to `const double *`. Deprecated.
-   MFEM_DEPRECATED inline operator const T *() const { return data; }
+   MFEM_DEPRECATED inline operator const real_t *() const { return data; }
 
    /// STL-like begin.
-   inline T *begin() { return data; }
+   inline real_t *begin() { return data; }
 
    /// STL-like end.
-   inline T *end() { return data + size; }
+   inline real_t *end() { return data + size; }
 
    /// STL-like begin (const version).
-   inline const T *begin() const { return data; }
+   inline const real_t *begin() const { return data; }
 
    /// STL-like end (const version).
-   inline const T *end() const { return data + size; }
+   inline const real_t *end() const { return data + size; }
 
    /// Return a reference to the Memory object used by the Vector.
-   Memory<T> &GetMemory() { return data; }
+   Memory<real_t> &GetMemory() { return data; }
 
    /** @brief Return a reference to the Memory object used by the Vector, const
        version. */
-   const Memory<T> &GetMemory() const { return data; }
+   const Memory<real_t> &GetMemory() const { return data; }
 
    /// Update the memory location of the vector to match @a v.
-   void SyncMemory(const VectorMP<T> &v) const { GetMemory().Sync(v.GetMemory()); }
+   void SyncMemory(const Vector &v) const { GetMemory().Sync(v.GetMemory()); }
 
    /// Update the alias memory location of the vector to match @a v.
-   void SyncAliasMemory(const VectorMP<T> &v) const
+   void SyncAliasMemory(const Vector &v) const
    { GetMemory().SyncAlias(v.GetMemory(),Size()); }
 
    /// Read the Vector data (host pointer) ownership flag.
    inline bool OwnsData() const { return data.OwnsHostPtr(); }
 
    /// Changes the ownership of the data; after the call the Vector is empty
-   inline void StealData(T **p)
+   inline void StealData(real_t **p)
    { *p = data; data.Reset(); size = 0; }
 
    /// Changes the ownership of the data; after the call the Vector is empty
-   inline T *StealData() { T *p; StealData(&p); return p; }
+   inline real_t *StealData() { real_t *p; StealData(&p); return p; }
 
    /// Access Vector entries. Index i = 0 .. size-1.
-   T &Elem(int i);
+   real_t &Elem(int i);
 
    /// Read only access to Vector entries. Index i = 0 .. size-1.
-   const T &Elem(int i) const;
+   const real_t &Elem(int i) const;
 
    /// Access Vector entries using () for 0-based indexing.
    /** @note If MFEM_DEBUG is enabled, bounds checking is performed. */
-   inline T &operator()(int i);
+   inline real_t &operator()(int i);
 
    /// Read only access to Vector entries using () for 0-based indexing.
    /** @note If MFEM_DEBUG is enabled, bounds checking is performed. */
-   inline const T &operator()(int i) const;
+   inline const real_t &operator()(int i) const;
 
    /// Access Vector entries using [] for 0-based indexing.
    /** @note If MFEM_DEBUG is enabled, bounds checking is performed. */
-   inline T &operator[](int i) { return (*this)(i); }
+   inline real_t &operator[](int i) { return (*this)(i); }
 
    /// Read only access to Vector entries using [] for 0-based indexing.
    /** @note If MFEM_DEBUG is enabled, bounds checking is performed. */
-   inline const T &operator[](int i) const { return (*this)(i); }
+   inline const real_t &operator[](int i) const { return (*this)(i); }
 
    /// Dot product with a `double *` array.
    /// This function always executes on the CPU. A HostRead() will be called if
@@ -338,54 +313,54 @@ public:
    /// To optionally execute on the device:
    /// Vector tmp(v, Size());
    /// res = (*this) * tmp;
-   T operator*(const T *v) const;
+   real_t operator*(const real_t *v) const;
 
    /// Return the inner-product.
-   T operator*(const VectorMP<T> &v) const;
+   real_t operator*(const Vector &v) const;
 
    /// Copy Size() entries from @a v.
-   VectorMP &operator=(const T *v);
+   Vector &operator=(const real_t *v);
 
    /// Copy assignment.
    /** @note Defining this method overwrites the implicitly defined copy
        assignment operator. */
-   VectorMP &operator=(const VectorMP<T> &v);
+   Vector &operator=(const Vector &v);
 
    /// Move assignment
-   VectorMP &operator=(VectorMP<T>&& v);
+   Vector &operator=(Vector&& v);
 
    /// Redefine '=' for vector = constant.
-   VectorMP &operator=(T value);
+   Vector &operator=(real_t value);
 
-   VectorMP &operator*=(T c);
+   Vector &operator*=(real_t c);
 
    /// Component-wise scaling: (*this)(i) *= v(i)
-   VectorMP &operator*=(const VectorMP<T> &v);
+   Vector &operator*=(const Vector &v);
 
-   VectorMP &operator/=(T c);
+   Vector &operator/=(real_t c);
 
    /// Component-wise division: (*this)(i) /= v(i)
-   VectorMP &operator/=(const VectorMP<T> &v);
+   Vector &operator/=(const Vector &v);
 
-   VectorMP &operator-=(T c);
+   Vector &operator-=(real_t c);
 
-   VectorMP &operator-=(const VectorMP<T> &v);
+   Vector &operator-=(const Vector &v);
 
-   VectorMP &operator+=(T c);
+   Vector &operator+=(real_t c);
 
-   VectorMP &operator+=(const VectorMP<T> &v);
+   Vector &operator+=(const Vector &v);
 
    /// (*this) += a * Va
-   VectorMP &Add(const T a, const VectorMP<T> &Va);
+   Vector &Add(const real_t a, const Vector &Va);
 
    /// (*this) = a * x
-   VectorMP &Set(const T a, const VectorMP<T> &x);
+   Vector &Set(const real_t a, const Vector &x);
 
    /// (*this)[i + offset] = v[i]
-   void SetVector(const VectorMP<T> &v, int offset);
+   void SetVector(const Vector &v, int offset);
 
    /// (*this)[i + offset] += v[i]
-   void AddSubVector(const VectorMP<T> &v, int offset);
+   void AddSubVector(const Vector &v, int offset);
 
    /// (*this) = -(*this)
    void Neg();
@@ -397,57 +372,53 @@ public:
    void Abs();
 
    /// (*this)(i) = pow((*this)(i), p)
-   void Pow(const T p);
+   void Pow(const real_t p);
 
    /// Swap the contents of two Vectors
    /** Implemented without using move assignment, avoiding Destroy() calls. */
-   inline void Swap(VectorMP<T> &other);
+   inline void Swap(Vector &other);
 
    /// Set v = v1 + v2.
-   friend void add<T>(const VectorMP<T> &v1, const VectorMP<T> &v2,
-                      VectorMP<T> &v);
+   friend void add(const Vector &v1, const Vector &v2, Vector &v);
 
    /// Set v = v1 + alpha * v2.
-   friend void add<T>(const VectorMP<T> &v1, T alpha, const VectorMP<T> &v2,
-                      VectorMP<T> &v);
+   friend void add(const Vector &v1, real_t alpha, const Vector &v2, Vector &v);
 
    /// z = a * (x + y)
-   friend void add<T>(const T a, const VectorMP<T> &x, const VectorMP<T> &y,
-                      VectorMP<T> &z);
+   friend void add(const real_t a, const Vector &x, const Vector &y, Vector &z);
 
    /// z = a * x + b * y
-   friend void add<T>(const T a, const VectorMP<T> &x,
-                      const T b, const VectorMP<T> &y, VectorMP<T> &z);
+   friend void add(const real_t a, const Vector &x,
+                   const real_t b, const Vector &y, Vector &z);
 
    /// Set v = v1 - v2.
-   friend void subtract<T>(const VectorMP<T> &v1, const VectorMP<T> &v2,
-                           VectorMP<T> &v);
+   friend void subtract(const Vector &v1, const Vector &v2, Vector &v);
 
    /// z = a * (x - y)
-   friend void subtract<T>(const T a, const VectorMP<T> &x,
-                           const VectorMP<T> &y, VectorMP<T> &z);
+   friend void subtract(const real_t a, const Vector &x,
+                        const Vector &y, Vector &z);
 
    /// Computes cross product of this vector with another 3D vector.
    /// vout = this x vin.
-   void cross3D(const VectorMP<T> &vin, VectorMP<T> &vout) const;
+   void cross3D(const Vector &vin, Vector &vout) const;
 
    /// v = median(v,lo,hi) entrywise.  Implementation assumes lo <= hi.
-   void median(const VectorMP<T> &lo, const VectorMP<T> &hi);
+   void median(const Vector &lo, const Vector &hi);
 
    /// Extract entries listed in @a dofs to the output Vector @a elemvect.
    /** Negative dof values cause the -dof-1 position in @a elemvect to receive
        the -val in from this Vector. */
-   void GetSubVector(const Array<int> &dofs, VectorMP<T> &elemvect) const;
+   void GetSubVector(const Array<int> &dofs, Vector &elemvect) const;
 
    /// Extract entries listed in @a dofs to the output array @a elem_data.
    /** Negative dof values cause the -dof-1 position in @a elem_data to receive
        the -val in from this Vector. */
-   void GetSubVector(const Array<int> &dofs, T *elem_data) const;
+   void GetSubVector(const Array<int> &dofs, real_t *elem_data) const;
 
    /// Set the entries listed in @a dofs to the given @a value.
    /** Negative dof values cause the -dof-1 position in this Vector to receive
        the -value. */
-   void SetSubVector(const Array<int> &dofs, const T value);
+   void SetSubVector(const Array<int> &dofs, const real_t value);
 
    /// Set the entries listed in @a dofs to the given @a value (always on host).
    /** Negative dof values cause the -dof-1 position in this Vector to receive
@@ -456,36 +427,36 @@ public:
        As opposed to SetSubVector(const Array<int>&, const real_t), this
        function will execute only on host, even if the vector or the @a dofs
        array have the device flag set. */
-   void SetSubVectorHost(const Array<int> &dofs, const T value);
+   void SetSubVectorHost(const Array<int> &dofs, const real_t value);
 
    /** @brief Set the entries listed in @a dofs to the values given in the @a
        elemvect Vector. Negative dof values cause the -dof-1 position in this
        Vector to receive the -val from @a elemvect. */
-   void SetSubVector(const Array<int> &dofs, const VectorMP<T> &elemvect);
+   void SetSubVector(const Array<int> &dofs, const Vector &elemvect);
 
    /** @brief Set the entries listed in @a dofs to the values given the @a ,
        elem_data array. Negative dof values cause the -dof-1 position in this
        Vector to receive the -val from @a elem_data. */
-   void SetSubVector(const Array<int> &dofs, T *elem_data);
+   void SetSubVector(const Array<int> &dofs, real_t *elem_data);
 
    /** @brief Add elements of the @a elemvect Vector to the entries listed in @a
        dofs. Negative dof values cause the -dof-1 position in this Vector to add
        the -val from @a elemvect. */
-   void AddElementVector(const Array<int> & dofs, const VectorMP<T> & elemvect);
+   void AddElementVector(const Array<int> & dofs, const Vector & elemvect);
 
    /** @brief Add elements of the @a elem_data array to the entries listed in @a
        dofs. Negative dof values cause the -dof-1 position in this Vector to add
        the -val from @a elem_data. */
-   void AddElementVector(const Array<int> & dofs, T *elem_data);
+   void AddElementVector(const Array<int> & dofs, real_t *elem_data);
 
    /** @brief Add @a times the elements of the @a elemvect Vector to the entries
        listed in @a dofs. Negative dof values cause the -dof-1 position in this
        Vector to add the -a*val from @a elemvect. */
-   void AddElementVector(const Array<int> & dofs, const T a,
-                         const VectorMP<T> & elemvect);
+   void AddElementVector(const Array<int> & dofs, const real_t a,
+                         const Vector & elemvect);
 
    /// Set all vector entries NOT in the @a dofs Array to the given @a val.
-   void SetSubVectorComplement(const Array<int> &dofs, const T val);
+   void SetSubVectorComplement(const Array<int> &dofs, const real_t val);
 
    /// Prints vector to stream out.
    void Print(std::ostream &out = mfem::out, int width = 8) const;
@@ -516,63 +487,61 @@ public:
    /// Set random values in the vector.
    void Randomize(int seed = 0);
    /// Returns the l2 norm of the vector.
-   T Norml2() const;
+   real_t Norml2() const;
    /// Returns the l_infinity norm of the vector.
-   T Normlinf() const;
+   real_t Normlinf() const;
    /// Returns the l_1 norm of the vector.
-   T Norml1() const;
+   real_t Norml1() const;
    /// Returns the l_p norm of the vector.
-   T Normlp(T p) const;
+   real_t Normlp(real_t p) const;
    /// Returns the maximal element of the vector.
-   T Max() const;
+   real_t Max() const;
    /// Returns the minimal element of the vector.
-   T Min() const;
+   real_t Min() const;
    /// Return the sum of the vector entries
-   T Sum() const;
+   real_t Sum() const;
    /// Compute the square of the Euclidean distance to another vector.
-   inline T DistanceSquaredTo(const T *p) const;
+   inline real_t DistanceSquaredTo(const real_t *p) const;
    /// Compute the square of the Euclidean distance to another vector.
-   inline T DistanceSquaredTo(const VectorMP<T> &p) const;
+   inline real_t DistanceSquaredTo(const Vector &p) const;
    /// Compute the Euclidean distance to another vector.
-   inline T DistanceTo(const T *p) const;
+   inline real_t DistanceTo(const real_t *p) const;
    /// Compute the Euclidean distance to another vector.
-   inline T DistanceTo(const VectorMP<T> &p) const;
+   inline real_t DistanceTo(const Vector &p) const;
 
    /** @brief Count the number of entries in the Vector for which isfinite
        is false, i.e. the entry is a NaN or +/-Inf. */
    int CheckFinite() const { return mfem::CheckFinite(HostRead(), size); }
 
    /// Destroys vector.
-   virtual ~VectorMP<T>();
+   virtual ~Vector();
 
    /// Shortcut for mfem::Read(vec.GetMemory(), vec.Size(), on_dev).
-   virtual const T *Read(bool on_dev = true) const
+   virtual const real_t *Read(bool on_dev = true) const
    { return mfem::Read(data, size, on_dev); }
 
    /// Shortcut for mfem::Read(vec.GetMemory(), vec.Size(), false).
-   virtual const T *HostRead() const
+   virtual const real_t *HostRead() const
    { return mfem::Read(data, size, false); }
 
    /// Shortcut for mfem::Write(vec.GetMemory(), vec.Size(), on_dev).
-   virtual T *Write(bool on_dev = true)
+   virtual real_t *Write(bool on_dev = true)
    { return mfem::Write(data, size, on_dev); }
 
    /// Shortcut for mfem::Write(vec.GetMemory(), vec.Size(), false).
-   virtual T *HostWrite()
+   virtual real_t *HostWrite()
    { return mfem::Write(data, size, false); }
 
    /// Shortcut for mfem::ReadWrite(vec.GetMemory(), vec.Size(), on_dev).
-   virtual T *ReadWrite(bool on_dev = true)
+   virtual real_t *ReadWrite(bool on_dev = true)
    { return mfem::ReadWrite(data, size, on_dev); }
 
    /// Shortcut for mfem::ReadWrite(vec.GetMemory(), vec.Size(), false).
-   virtual T *HostReadWrite()
+   virtual real_t *HostReadWrite()
    { return mfem::ReadWrite(data, size, false); }
 
 };
 
-using Vector = VectorMP<real_t>;
-
 // Inline methods
 
 template <typename T>
@@ -581,8 +550,7 @@ inline T ZeroSubnormal(T val)
    return (std::fpclassify(val) == FP_SUBNORMAL) ? 0.0 : val;
 }
 
-template <class T>
-inline bool IsFinite(const T &val)
+inline bool IsFinite(const real_t &val)
 {
    // isfinite didn't appear in a standard until C99, and later C++11. It wasn't
    // standard in C89 or C++98. PGI as of 14.7 still defines it as a macro.
@@ -593,8 +561,7 @@ inline bool IsFinite(const T &val)
 #endif
 }
 
-template <class T>
-inline int CheckFinite(const T *v, const int n)
+inline int CheckFinite(const real_t *v, const int n)
 {
    int bad = 0;
    for (int i = 0; i < n; i++)
@@ -604,8 +571,7 @@ inline int CheckFinite(const T *v, const int n)
    return bad;
 }
 
-template <class T>
-inline VectorMP<T>::VectorMP(int s)
+inline Vector::Vector(int s)
 {
    MFEM_ASSERT(s>=0,"Unexpected negative size.");
    size = s;
@@ -615,8 +581,7 @@ inline VectorMP<T>::VectorMP(int s)
    }
 }
 
-template <class T>
-inline void VectorMP<T>::SetSize(int s)
+inline void Vector::SetSize(int s)
 {
    if (s == size)
    {
@@ -636,8 +601,7 @@ inline void VectorMP<T>::SetSize(int s)
    data.UseDevice(use_dev);
 }
 
-template <class T>
-inline void VectorMP<T>::SetSize(int s, MemoryType mt)
+inline void Vector::SetSize(int s, MemoryType mt)
 {
    if (mt == data.GetMemoryType())
    {
@@ -666,12 +630,11 @@ inline void VectorMP<T>::SetSize(int s, MemoryType mt)
    data.UseDevice(use_dev);
 }
 
-template <class T>
-inline void VectorMP<T>::Reserve(int res)
+inline void Vector::Reserve(int res)
 {
    if (res > Capacity())
    {
-      Memory<T> p(res, data.GetMemoryType());
+      Memory<real_t> p(res, data.GetMemoryType());
       p.CopyFrom(data, size);
       p.UseDevice(data.UseDevice());
       data.Delete();
@@ -679,9 +642,8 @@ inline void VectorMP<T>::Reserve(int res)
    }
 }
 
-template <class T>
-inline void VectorMP<T>::NewMemoryAndSize(const Memory<T> &mem, int s,
-                                          bool own_mem)
+inline void Vector::NewMemoryAndSize(const Memory<real_t> &mem, int s,
+                                     bool own_mem)
 {
    data.Delete();
    size = s;
@@ -695,23 +657,20 @@ inline void VectorMP<T>::NewMemoryAndSize(const Memory<T> &mem, int s,
    }
 }
 
-template <class T>
-inline void VectorMP<T>::MakeRef(VectorMP<T> &base, int offset, int s)
+inline void Vector::MakeRef(Vector &base, int offset, int s)
 {
    data.Delete();
    size = s;
    data.MakeAlias(base.GetMemory(), offset, s);
 }
 
-template <class T>
-inline void VectorMP<T>::MakeRef(VectorMP<T> &base, int offset)
+inline void Vector::MakeRef(Vector &base, int offset)
 {
    data.Delete();
    data.MakeAlias(base.GetMemory(), offset, size);
 }
 
-template <class T>
-inline void VectorMP<T>::Destroy()
+inline void Vector::Destroy()
 {
    const bool use_dev = data.UseDevice();
    data.Delete();  // calls data.Reset(h_mt) as well
@@ -719,8 +678,7 @@ inline void VectorMP<T>::Destroy()
    data.UseDevice(use_dev);
 }
 
-template <class T>
-inline T &VectorMP<T>::operator()(int i)
+inline real_t &Vector::operator()(int i)
 {
    MFEM_ASSERT(data && i >= 0 && i < size,
                "index [" << i << "] is out of range [0," << size << ")");
@@ -728,8 +686,7 @@ inline T &VectorMP<T>::operator()(int i)
    return data[i];
 }
 
-template <class T>
-inline const T &VectorMP<T>::operator()(int i) const
+inline const real_t &Vector::operator()(int i) const
 {
    MFEM_ASSERT(data && i >= 0 && i < size,
                "index [" << i << "] is out of range [0," << size << ")");
@@ -737,8 +694,7 @@ inline const T &VectorMP<T>::operator()(int i) const
    return data[i];
 }
 
-template <class T>
-inline void VectorMP<T>::Swap(VectorMP<T> &other)
+inline void Vector::Swap(Vector &other)
 {
    mfem::Swap(data, other.data);
    mfem::Swap(size, other.size);
@@ -746,22 +702,19 @@ inline void VectorMP<T>::Swap(VectorMP<T> &other)
 
 /** @brief Swap of Vector objects for use with standard library algorithms.
     Also, used by mfem::Swap(). */
-template <class T>
-inline void swap(VectorMP<T> &a, VectorMP<T> &b)
+inline void swap(Vector &a, Vector &b)
 {
    a.Swap(b);
 }
 
-template <class T>
-inline VectorMP<T>::~VectorMP()
+inline Vector::~Vector()
 {
    data.Delete();
 }
 
-template <class T>
-inline T DistanceSquared(const T *x, const T *y, const int n)
+inline real_t DistanceSquared(const real_t *x, const real_t *y, const int n)
 {
-   T d = 0.0;
+   real_t d = 0.0;
 
    for (int i = 0; i < n; i++)
    {
@@ -771,50 +724,43 @@ inline T DistanceSquared(const T *x, const T *y, const int n)
    return d;
 }
 
-template <class T>
-inline T Distance(const T *x, const T *y, const int n)
+inline real_t Distance(const real_t *x, const real_t *y, const int n)
 {
    return std::sqrt(DistanceSquared(x, y, n));
 }
 
-template <class T>
-inline T Distance(const VectorMP<T> &x, const VectorMP<T> &y)
+inline real_t Distance(const Vector &x, const Vector &y)
 {
    return x.DistanceTo(y);
 }
 
-template <class T>
-inline T VectorMP<T>::DistanceSquaredTo(const T *p) const
+inline real_t Vector::DistanceSquaredTo(const real_t *p) const
 {
-   return DistanceSquared<T>(data, p, size);
+   return DistanceSquared(data, p, size);
 }
 
-template <class T>
-inline T VectorMP<T>::DistanceSquaredTo(const VectorMP<T> &p) const
+inline real_t Vector::DistanceSquaredTo(const Vector &p) const
 {
    MFEM_ASSERT(p.Size() == Size(), "Incompatible vector sizes.");
-   return DistanceSquared<T>(data, p.data, size);
+   return DistanceSquared(data, p.data, size);
 }
 
-template <class T>
-inline T VectorMP<T>::DistanceTo(const T *p) const
+inline real_t Vector::DistanceTo(const real_t *p) const
 {
-   return Distance<T>(data, p, size);
+   return Distance(data, p, size);
 }
 
-template <class T>
-inline T VectorMP<T>::DistanceTo(const VectorMP<T> &p) const
+inline real_t Vector::DistanceTo(const Vector &p) const
 {
    MFEM_ASSERT(p.Size() == Size(), "Incompatible vector sizes.");
-   return Distance<T>(data, p.data, size);
+   return Distance(data, p.data, size);
 }
 
 /// Returns the inner product of x and y
 /** In parallel this computes the inner product of the local vectors,
     producing different results on each MPI rank.
 */
-template <class T>
-inline T InnerProduct(const VectorMP<T> &x, const VectorMP<T> &y)
+inline real_t InnerProduct(const Vector &x, const Vector &y)
 {
    return x * y;
 }
@@ -824,25 +770,11 @@ inline T InnerProduct(const VectorMP<T> &x, const VectorMP<T> &y)
 /** In parallel this computes the inner product of the global vectors,
     producing identical results on each MPI rank.
 */
-template <class T>
-inline T InnerProduct(MPI_Comm comm, const VectorMP<T> &x, const VectorMP<T> &y)
+inline real_t InnerProduct(MPI_Comm comm, const Vector &x, const Vector &y)
 {
-   T loc_prod = x * y;
-   T glb_prod;
-
-   if (std::is_same<T, double>::value)
-   {
-      MPI_Allreduce(&loc_prod, &glb_prod, 1, MPI_DOUBLE, MPI_SUM, comm);
-   }
-   else if (std::is_same<T, float>::value)
-   {
-      MPI_Allreduce(&loc_prod, &glb_prod, 1, MPI_FLOAT, MPI_SUM, comm);
-   }
-   else
-   {
-      MFEM_ABORT("Floating point type not supported");
-   }
-
+   real_t loc_prod = x * y;
+   real_t glb_prod;
+   MPI_Allreduce(&loc_prod, &glb_prod, 1, MFEM_MPI_REAL_T, MPI_SUM, comm);
    return glb_prod;
 }
 #endif

makefile

@@ -794,7 +794,6 @@ status info:
 	$(info MFEM_MPI_NP            = $(MFEM_MPI_NP))
 	@true
 
-ASTYLE_BIN = astyle
 ASTYLE = $(ASTYLE_BIN) --options=$(SRC)config/mfem.astylerc
 ASTYLE_VER = "Artistic Style Version 3.1"
 FORMAT_FILES = $(foreach dir,$(DIRS) $(EM_DIRS) config,$(dir)/*.?pp)

mesh/attribute_sets.cpp

@@ -113,13 +113,13 @@ AttributeSets::GetAttributeSetMarker(const std::string & set_name) const
 
 Array<int> AttributeSets::AttrToMarker(int max_attr, const Array<int> &attrs)
 {
-   MFEM_ASSERT(attrs.Max() <= max_attr, "Invalid attribute number present.");
+   MFEM_VERIFY(attrs.Min() >= 1, "Found attribute less than one")
+   MFEM_ASSERT(attrs.Max() <= max_attr, "Found attribute greater than max_attr")
 
    Array<int> marker(max_attr);
    marker = 0;
    for (auto const &attr : attrs)
    {
-      MFEM_VERIFY(attr > 0, "Attribute number less than one!");
       marker[attr-1] = 1;
    }
    return marker;

mesh/mesh.cpp

@@ -36,6 +36,7 @@
 #include <numeric>
 #include <unordered_map>
 #include <unordered_set>
+#include <list>
 
 // Include the METIS header, if using version 5. If using METIS 4, the needed
 // declarations are inlined below, i.e. no header is needed.
@@ -4570,8 +4571,9 @@ Mesh::Mesh(const Mesh &mesh, bool copy_nodes)
       FiniteElementSpace *fes_copy =
          new FiniteElementSpace(*fes, this, fec_copy);
       Nodes = new GridFunction(fes_copy);
-      Nodes->MakeOwner(fec_copy);
-      *Nodes = *mesh.Nodes;
+      Nodes->MakeOwner();
+      // only copy underlying Vector data
+      *Nodes = static_cast<Vector &>(*mesh.Nodes);
       own_nodes = 1;
    }
    else
@@ -4772,12 +4774,12 @@ Mesh::Mesh(real_t *vertices_, int num_vertices,
    FinalizeTopology();
 }
 
-Mesh::Mesh( const NURBSExtension& ext )
+Mesh::Mesh(const NURBSExtension& ext)
    : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes)
 {
    SetEmpty();
    /// make an internal copy of the NURBSExtension
-   NURBSext = new NURBSExtension( ext );
+   NURBSext = new NURBSExtension(ext);
 
    Dim              = NURBSext->Dimension();
    NumOfVertices    = NURBSext->GetNV();
@@ -4791,11 +4793,12 @@ Mesh::Mesh( const NURBSExtension& ext )
    if (NURBSext->HavePatches())
    {
       NURBSFECollection  *fec = new NURBSFECollection(NURBSext->GetOrder());
-      FiniteElementSpace *fes = new FiniteElementSpace(this, fec, Dim,
+      const int vdim = NURBSext->GetPatchSpaceDimension();
+      FiniteElementSpace *fes = new FiniteElementSpace(this, fec, vdim,
                                                        Ordering::byVDIM);
       Nodes = new GridFunction(fes);
-      Nodes->MakeOwner(fec);
-      NURBSext->SetCoordsFromPatches(*Nodes);
+      Nodes->MakeOwner();
+      NURBSext->SetCoordsFromPatches(*Nodes, vdim);
       own_nodes = 1;
       spaceDim = Nodes->VectorDim();
       for (int i = 0; i < spaceDim; i++)
@@ -6409,7 +6412,7 @@ void Mesh::UpdateNURBS()
    NURBSext->SetKnotsFromPatches();
 
    Dim = NURBSext->Dimension();
-   spaceDim = Dim;
+   spaceDim = Nodes->FESpace()->GetVDim();
 
    if (NumOfElements != NURBSext->GetNE())
    {
@@ -6434,7 +6437,8 @@ void Mesh::UpdateNURBS()
    Nodes->FESpace()->Update();
    Nodes->Update();
    NodesUpdated();
-   NURBSext->SetCoordsFromPatches(*Nodes);
+   const int vdim = Nodes->FESpace()->GetVDim();
+   NURBSext->SetCoordsFromPatches(*Nodes, vdim);
 
    if (NumOfVertices != NURBSext->GetNV())
    {
@@ -6537,6 +6541,8 @@ void Mesh::LoadPatchTopo(std::istream &input, Array<int> &edge_to_ukv)
       Array<int> ukv_to_rpkv;
       GetEdgeToUniqueKnotvector(edge_to_ukv, ukv_to_rpkv);
    }
+
+   CorrectPatchTopoOrientations(edge_to_ukv);
 }
 
 void Mesh::GetEdgeToUniqueKnotvector(Array<int> &edge_to_ukv,
@@ -6547,9 +6553,9 @@ void Mesh::GetEdgeToUniqueKnotvector(Array<int> &edge_to_ukv,
    const int NPKV = NP * dim;    // number of patch knotvectors
    constexpr int notset = -9999999;
    // Sign convention
-   auto sign = [](int i) { return -1 - i; };
-   auto unsign = [](int i) { return (i < 0) ? -1 - i : i; };
-   // Edge index -> dimension convention
+   auto flipSign = [](int i) { return -1 - i; };
+   auto unSign = [](int i) { return (i < 0) ? -1 - i : i; };
+   // Local edge index -> dimension convention
    auto edge_to_dim = [](int i) { return (i < 8) ? ((i & 1) ? 1 : 0) : 2; };
 
    Array<int> v(2); // vertices of an edge
@@ -6564,7 +6570,7 @@ void Mesh::GetEdgeToUniqueKnotvector(Array<int> &edge_to_ukv,
       {
          GetElementVertices(i, v);
          // Sign is based on the edge's vertex indices
-         edge_to_ukv[i] = (v[1] > v[0]) ? i : sign(i);
+         edge_to_ukv[i] = (v[1] > v[0]) ? i : flipSign(i);
          ukv_to_rpkv[i] = i;
       }
       return;
@@ -6614,14 +6620,14 @@ void Mesh::GetEdgeToUniqueKnotvector(Array<int> &edge_to_ukv,
          // We've set this edge already - link this index to it
          if (edge_to_pkv[edge] != notset)
          {
-            const int pkv_other = unsign(edge_to_pkv[edge]);
+            const int pkv_other = unSign(edge_to_pkv[edge]);
             unite(pkv, pkv_other);
          }
          else
          {
             GetEdgeVertices(edge, v);
             // Sign is based on the edge's vertex indices
-            edge_to_pkv[edge] = (v[1] > v[0]) ? pkv : sign(pkv);
+            edge_to_pkv[edge] = (v[1] > v[0]) ? pkv : flipSign(pkv);
          }
       }
    }
@@ -6648,11 +6654,255 @@ void Mesh::GetEdgeToUniqueKnotvector(Array<int> &edge_to_ukv,
    edge_to_ukv.SetSize(NumOfEdges);
    for (int i = 0; i < NumOfEdges; i++)
    {
-      const int pkv = unsign(edge_to_pkv[i]);
+      const int pkv = unSign(edge_to_pkv[i]);
       const int rpkv = pkv_to_rpkv[pkv];
       const int ukv = rpkv_to_ukv[rpkv];
-      edge_to_ukv[i] = (edge_to_pkv[i] < 0) ? sign(ukv) : ukv;
+      edge_to_ukv[i] = (edge_to_pkv[i] < 0) ? flipSign(ukv) : ukv;
    }
+
+   CorrectPatchTopoOrientations(edge_to_ukv);
+}
+
+void Mesh::CorrectPatchTopoOrientations(Array<int> &edge_to_ukv) const
+{
+   const int dim = Dimension(); // Topological (not physical) dimension
+   if (dim == 1) { return; }
+
+   // Sign convention
+   auto flipSign = [](int i) { return -1 - i; };
+
+   const Table *face2elem = GetFaceToElementTable();
+   Array<int> pfaces, orient;
+   Array<int> fe, feo;
+
+   // Finds elements sharing a face containing knotvector kv.
+   auto faceNeighbors = [&](int p, int kv, std::unordered_set<int> &nghb)
+   {
+      if (dim == 2) { GetElementEdges(p, pfaces, orient); }
+      else { GetElementFaces(p, pfaces, orient); }
+
+      for (auto face : pfaces)
+      {
+         // Check whether this face contains kv.
+         GetFaceEdges(face, fe, feo);
+         bool hasKV = false;
+         for (auto e : fe)
+         {
+            const int skv = edge_to_ukv[e];
+            if (skv == kv || flipSign(skv) == kv) { hasKV = true; }
+         }
+         if (hasKV)
+         {
+            Array<int> row;
+            face2elem->GetRow(face, row);
+            for (auto elem : row) { nghb.insert(elem); }
+         }
+      }
+   };
+
+   std::vector<std::vector<int>> dir_edges;
+   if (dim == 2)
+   {
+      dir_edges =
+      {
+         {0,2},
+         {1,3}
+      };
+   }
+   else
+   {
+      dir_edges =
+      {
+         {0,2,4,6},
+         {1,3,5,7},
+         {8,9,10,11}
+      };
+   }
+
+   Array<int> ukvs((dim==2) ? 4 : 12);
+   Array<int> pe, oe;
+   bool initKV = false;
+
+   auto setPatchDirections = [&](int p, int kv, Array<bool> &edgeSet,
+                                 std::unordered_set<int> &visited)
+   {
+      // Edges and orientations for this patch
+      GetElementEdges(p, pe, oe);
+
+      // Get the signed unique knot vector indices
+      for (int i = 0; i < pe.Size(); i++)
+      {
+         ukvs[i] = edge_to_ukv[pe[i]];
+         ukvs[i] = (oe[i] < 0) ? flipSign(ukvs[i]) : ukvs[i];
+      }
+
+      // Find the direction with this kv.
+      int thisDir = -1;
+      for (int d=0; d<dim; ++d) // Loop over directions.
+      {
+         const int skv = edge_to_ukv[pe[dir_edges[d][0]]];
+         if (skv == kv || flipSign(skv) == kv)
+         {
+            thisDir = d;
+         }
+      }
+      MFEM_VERIFY(thisDir >= 0, "");
+
+      // For this direction, find any edge already set. If no edge is set, we
+      // arbitrarily take the first.
+      int ref_edge0 = dir_edges[thisDir][0];
+      for (auto ref_edge : dir_edges[thisDir])
+      {
+         const int edge = pe[ref_edge];
+         if (edgeSet[edge])
+         {
+            ref_edge0 = ref_edge;
+         }
+      }
+
+      if (initKV && !edgeSet[pe[ref_edge0]])
+      {
+         visited.erase(p);
+         return false; // There is no set edge in this direction on this patch.
+      }
+
+      initKV = true;
+
+      // Use ref_edge0 to set other edges in this direction.
+      edgeSet[pe[ref_edge0]] = true;
+      for (auto i : dir_edges[thisDir])
+      {
+         if (i == ref_edge0)
+         {
+            continue;
+         }
+
+         const int edge = pe[i];
+         if ((dim == 2 && ukvs[i] != flipSign(ukvs[ref_edge0])) ||
+             (dim == 3 && ukvs[i] == flipSign(ukvs[ref_edge0])))
+         {
+            // Flip the sign of this edge
+            MFEM_VERIFY(!edgeSet[edge], "");
+            edge_to_ukv[edge] = flipSign(edge_to_ukv[edge]);
+         }
+
+         edgeSet[edge] = true;
+      }
+
+      return true;
+   };
+
+   Array<bool> edgeSet(NumOfEdges); // Whether edge has orientation set
+   edgeSet = false;
+
+   std::unordered_set<int> unset; // Patches with an unset edge
+   for (int i=0; i<NumOfElements; ++i) { unset.insert(i); }
+
+   const int max_iter = 3 * NumOfElements;
+   for (int iter=0; iter<max_iter; ++iter)
+   {
+      // Iteratively choose an unset patch (meaning not all edges have
+      // orientation set), choose a knotvector index for which the corresponding
+      // edges on this patch are not set, and sweep over all patches containing
+      // this knotvector. The patch sweep is ordered, by maintaining an ordered
+      // list `nextPatches` set by finding face-neighbor patches of visited
+      // patches, where the common face contains the knotvector. When each patch
+      // is visited, the edge orientations are set consistently. This iteration
+      // terminates when all edges have been set on all patches.
+
+      std::list<int> nextPatches; // Next patches to visit, ordered
+      std::unordered_set<int> nextSet; // nextPatches as a set
+      std::unordered_set<int> visited; // Visit each patch only once
+
+      if (unset.size() == 0)
+      {
+         break;
+      }
+
+      const int p0 = *unset.begin();
+      nextPatches.push_back(p0); // Start from arbitrary unset patch
+      nextSet.insert(p0);
+
+      // Choose an arbitrary unset direction for the first patch.
+      GetElementEdges(p0, pe, oe);
+      int unsetDim = -1;
+      for (int d=0; d<dim; ++d) // Loop over dimensions.
+      {
+         if (!edgeSet[pe[dir_edges[d][0]]])
+         {
+            unsetDim = d;
+         }
+      }
+
+      if (unsetDim == -1)
+      {
+         unset.erase(p0);
+         continue;
+      }
+
+      const int kv_signed = edge_to_ukv[pe[dir_edges[unsetDim][0]]];
+      const int kv = kv_signed < 0 ? flipSign(kv_signed) : kv_signed;
+      MFEM_VERIFY(!edgeSet[pe[dir_edges[unsetDim][0]]], "");
+
+      initKV = false;
+
+      while (nextPatches.size() > 0)
+      {
+         const int p = nextPatches.front();
+         nextPatches.pop_front();
+         nextSet.erase(p);
+         visited.insert(p);
+
+         const bool somethingSet = setPatchDirections(p, kv, edgeSet, visited);
+         if (!somethingSet)
+         {
+            continue;
+         }
+
+         // Find neighbors of patch p sharing a conforming face, via face2elem.
+         std::unordered_set<int> neighbors;
+         faceNeighbors(p, kv, neighbors);
+
+         bool allSet = true;
+         GetElementEdges(p, pe, oe);
+         for (auto edge : pe)
+         {
+            if (!edgeSet[edge])
+            {
+               allSet = false;
+            }
+         }
+         if (allSet)
+         {
+            unset.erase(p);
+         }
+
+         // Add neighbors not done to nextPatches.
+         for (auto n : neighbors)
+         {
+            if (n != p && visited.count(n) == 0 && unset.count(n) > 0)
+            {
+               if (nextSet.count(n) == 0)
+               {
+                  nextPatches.push_back(n);
+                  nextSet.insert(n);
+               }
+            }
+         }
+      }
+   }
+
+   bool allSet = true;
+   for (auto eset : edgeSet)
+   {
+      if (!eset)
+      {
+         allSet = false;
+      }
+   }
+   MFEM_VERIFY(allSet && unset.size() == 0, "Some edge is not set");
+
+   delete face2elem;
 }
 
 void Mesh::LoadNonconformingPatchTopo(std::istream &input,
@@ -6820,7 +7070,7 @@ void Mesh::SetCurvature(int order, bool discont, int space_dim, int ordering)
 
    const int old_space_dim = spaceDim;
    SetNodalFESpace(nfes);
-   Nodes->MakeOwner(nfec);
+   Nodes->MakeOwner();
 
    if (spaceDim != old_space_dim)
    {
@@ -7587,6 +7837,17 @@ bool Mesh::IsMixedMesh() const
 
 void Mesh::GetElementEdges(int i, Array<int> &edges, Array<int> &cor) const
 {
+   if (Dim == 1)
+   {
+      // In 1D, elements are segments and can be treated as edges.
+      edges.SetSize(1);
+      cor.SetSize(1);
+      edges[0] = i;
+      const int *v = elements[i]->GetVertices();
+      cor[0] = (v[0] < v[1]) ? (1) : (-1);
+      return;
+   }
+
    if (el_to_edge)
    {
       el_to_edge->GetRow(i, edges);
@@ -9563,6 +9824,8 @@ void Mesh::GetVertices(Vector &vert_coord) const
 
 void Mesh::SetVertices(const Vector &vert_coord)
 {
+   MFEM_VERIFY(vert_coord.Size() == spaceDim * NumOfVertices, "");
+   vertices.SetSize(NumOfVertices);
    for (int i = 0, nv = vertices.Size(); i < nv; i++)
       for (int j = 0; j < spaceDim; j++)
       {
@@ -12140,6 +12403,38 @@ void Mesh::PrintTopoEdges(std::ostream &os, const Array<int> &e_to_k,
 {
    Array<int> vert;
 
+   // In 1D patch-topology NURBS meshes, knotvector orientation is stored in the
+   // file's `edges` section, but the topological 1D mesh has NumOfEdges == 0
+   // (its "faces" are vertices). When a valid edge->knotvector map is provided,
+   // print a pseudo-edge list derived from the 1D elements so external tools
+   // (e.g. VisIt) can consume the mapping.
+   if (Dim == 1 && NumOfEdges == 0 && e_to_k.Size() == NumOfElements)
+   {
+      const int ne = NumOfElements;
+      os << "\nedges\n" << ne << '\n';
+      for (int i = 0; i < ne; i++)
+      {
+         const int *v = elements[i]->GetVertices();
+         int v0 = v[0], v1 = v[1];
+
+         int ki = e_to_k[i];
+         const bool flip = (ki < 0); // desired output vertex order: descending
+         if (flip) { ki = -1 - ki; } // print the unsigned knotvector index
+
+         // Encode the sign of e_to_k in the vertex ordering, consistent with
+         // Mesh::LoadPatchTopo(): v0 > v1 => negative sign.
+         if ((v0 > v1) != flip) { std::swap(v0, v1); }
+
+         os << ki << ' ' << v0 << ' ' << v1 << '\n';
+      }
+
+      if (!vmap)
+      {
+         os << "\nvertices\n" << NumOfVertices << '\n';
+      }
+      return;
+   }
+
    os << "\nedges\n" << NumOfEdges << '\n';
    for (int i = 0; i < NumOfEdges; i++)
    {
@@ -15205,7 +15500,7 @@ Mesh *Extrude1D(Mesh *mesh, const int ny, const real_t sy, const bool closed)
       fes2d = new FiniteElementSpace(mesh2d, fec2d, 2);
       mesh2d->SetNodalFESpace(fes2d);
       GridFunction *nodes2d = mesh2d->GetNodes();
-      nodes2d->MakeOwner(fec2d);
+      nodes2d->MakeOwner();
 
       NodeExtrudeCoefficient ecoeff(2, ny, sy);
       Vector lnodes;
@@ -15431,7 +15726,7 @@ Mesh *Extrude2D(Mesh *mesh, const int nz, const real_t sz)
       fes3d = new FiniteElementSpace(mesh3d, fec3d, 3);
       mesh3d->SetNodalFESpace(fes3d);
       GridFunction *nodes3d = mesh3d->GetNodes();
-      nodes3d->MakeOwner(fec3d);
+      nodes3d->MakeOwner();
 
       NodeExtrudeCoefficient ecoeff(3, nz, sz);
       Vector lnodes;
@@ -15452,6 +15747,128 @@ Mesh *Extrude2D(Mesh *mesh, const int nz, const real_t sz)
    return mesh3d;
 }
 
+Mesh PartitionMPI(int dim, int mpi_cnt, int elem_per_mpi, bool print,
+                  int &par_ref, Array<int> &partitioning)
+{
+   MFEM_VERIFY(dim > 1, "Not implemented for 1D meshes.");
+
+   auto factor = [&](int N)
+   {
+      for (int i = static_cast<int>(sqrt(N)); i > 0; i--)
+      { if (N % i == 0) { return i; } }
+      return 1;
+   };
+
+   par_ref = 0;
+   const int ref_factor = (dim == 2) ? 4 : 8;
+
+   // Elements per task before performing parallel refinements.
+   // This will be used to form the serial mesh.
+   int el0 = elem_per_mpi;
+   while (el0 % ref_factor == 0)
+   {
+      el0 /= ref_factor;
+      par_ref++;
+   }
+
+   // In the serial mesh we have:
+   // The number of MPI blocks is mpi_cnt = mp_x.mpy_y.mpy_z.
+   // The size of each MPI block is el0 = el0_x.el0_y.el0_z.
+   int mpi_x, mpi_y, mpi_z;
+   int el0_x, el0_y, el0_z;
+   if (dim == 2)
+   {
+      mpi_x = factor(mpi_cnt);
+      mpi_y = mpi_cnt / mpi_x;
+
+      // Switch order for better balance.
+      el0_y = factor(el0);
+      el0_x = el0 / el0_y;
+   }
+   else
+   {
+      mpi_x = factor(mpi_cnt);
+      mpi_y = factor(mpi_cnt / mpi_x);
+      mpi_z = mpi_cnt / mpi_x / mpi_y;
+
+      // Switch order for better balance.
+      el0_z = factor(el0);
+      el0_y = factor(el0 / el0_z);
+      el0_x = el0 / el0_y / el0_z;
+   }
+
+   if (print && dim == 2)
+   {
+      int elem_par_x = mpi_x * el0_x * pow(2, par_ref),
+          elem_par_y = mpi_y * el0_y * pow(2, par_ref);
+
+      mfem::out << "--- Mesh generation: \n";
+      mfem::out << "Par mesh:    " << elem_par_x << " x " << elem_par_y
+                << " (" << elem_par_x * elem_par_y << " elements)\n"
+                << "Elem / task: "
+                << el0_x * pow(2, par_ref) << " x "
+                << el0_y * pow(2, par_ref)
+                << " (" << el0_x * pow(2, 2*par_ref) * el0_y << " elements)\n"
+                << "MPI blocks:  " << mpi_x << " x " << mpi_y
+                << " (" << mpi_x * mpi_y << " mpi tasks)\n" << "-\n"
+                << "Serial mesh: "
+                << mpi_x * el0_x << " x " << mpi_y * el0_y
+                << " (" << mpi_x * el0_x * mpi_y * el0_y << " elements)\n"
+                << "Elem / task: " << el0_x << " x " << el0_y << std::endl
+                << "Par refine:  " << par_ref << std::endl;
+      mfem::out << "--- \n";
+   }
+
+   if (print && dim == 3)
+   {
+      int elem_par_x = mpi_x * el0_x * pow(2, par_ref),
+          elem_par_y = mpi_y * el0_y * pow(2, par_ref),
+          elem_par_z = mpi_z * el0_z * pow(2, par_ref);
+
+      mfem::out << "--- Mesh generation: \n";
+      mfem::out << "Par mesh:    "
+                << elem_par_x << " x " << elem_par_y << " x " << elem_par_z
+                << " (" << elem_par_x*elem_par_y*elem_par_z << " elements)\n"
+                << "Elem / task: "
+                << el0_x * pow(2, par_ref) << " x "
+                << el0_y * pow(2, par_ref) << " x "
+                << el0_z * pow(2, par_ref)
+                << " (" << el0_x*pow(2, 3*par_ref)*el0_y*el0_z << " elements)\n"
+                << "MPI blocks:  " << mpi_x << " x " << mpi_y << " x " << mpi_z
+                << " (" << mpi_x * mpi_y * mpi_z << " mpi tasks)\n" << "-\n"
+                << "Serial mesh: "
+                << mpi_x*el0_x << " x " << mpi_y*el0_y << " x " << mpi_z*el0_z
+                << " (" << mpi_x*el0_x*mpi_y*el0_y*mpi_z*el0_z << " elements)\n"
+                << "Elem / task: "
+                << el0_x << " x " << el0_y << " x " << el0_z  << std::endl
+                << "Par refine:  " << par_ref << std::endl;
+      mfem::out << "--- \n";
+   }
+
+   Mesh mesh;
+   int nxyz[3];
+   if (dim == 2)
+   {
+      mesh = Mesh::MakeCartesian2D(mpi_x * el0_x,
+                                   mpi_y * el0_y, Element::QUADRILATERAL, true);
+      nxyz[0] = mpi_x; nxyz[1] = mpi_y;
+   }
+   else
+   {
+      mesh = Mesh::MakeCartesian3D(mpi_x * el0_x,
+                                   mpi_y * el0_y,
+                                   mpi_z * el0_z, Element::HEXAHEDRON, true);
+      nxyz[0] = mpi_x; nxyz[1] = mpi_y; nxyz[2] = mpi_z;
+   }
+
+   const int NE = mesh.GetNE();
+   partitioning.SetSize(NE);
+   std::unique_ptr<int[]> p_raw(mesh.CartesianPartitioning(nxyz));
+   std::copy(p_raw.get(), p_raw.get() + NE, partitioning.GetData());
+
+   return mesh;
+}
+
 bool Mesh::Conforming() const
 {
    if (NURBSext)

mesh/mesh.hpp

@@ -527,6 +527,9 @@ protected:
    void PrintTopoEdges(std::ostream &out, const Array<int> &e_to_k,
                        bool vmap = false) const;
 
+   /// Set signs to ensure knotvectors are pointed in the same direction.
+   void CorrectPatchTopoOrientations(Array<int> &edge_to_ukv) const;
+
    /// Used in GetFaceElementTransformations (...)
    void GetLocalPtToSegTransformation(IsoparametricTransformation &,
                                       int i) const;
@@ -984,8 +987,8 @@ public:
 
    ///@}
 
-   /// Construct a Mesh from a NURBSExtension
-   explicit Mesh( const NURBSExtension& ext );
+   /// Construct a Mesh from a NURBSExtension, which is deep-copied.
+   explicit Mesh(const NURBSExtension& ext);
 
    /** @anchor mfem_Mesh_construction
        @name Methods for piecewise Mesh construction.
@@ -2538,13 +2541,16 @@ public:
        changing the mesh file itself. Examples in miniapps/nurbs/meshes. */
    void RefineNURBSFromFile(std::string ref_file);
 
-   /// For NURBS meshes, insert the new knots in @a kv, for each direction.
+   /// For NURBS meshes, insert the new knots in @a kv, for each KnotVector.
+   /// The size of @a kv should be the number of KnotVectors in NURBSExtension.
    void KnotInsert(Array<KnotVector*> &kv);
 
-   /// For NURBS meshes, insert the knots in @a kv, for each direction.
+   /// For NURBS meshes, insert the knots in @a kv, for each KnotVector.
+   /// The size of @a kv should be the number of KnotVectors in NURBSExtension.
    void KnotInsert(Array<Vector*> &kv);
 
-   /// For NURBS meshes, remove the knots in @a kv, for each direction.
+   /// For NURBS meshes, remove the knots in @a kv, for each KnotVector.
+   /// The size of @a kv should be the number of KnotVectors in NURBSExtension.
    void KnotRemove(Array<Vector*> &kv);
 
    /* For each knot vector:
@@ -3201,6 +3207,28 @@ Mesh *Extrude1D(Mesh *mesh, const int ny, const real_t sy,
 /// Extrude a 2D mesh
 Mesh *Extrude2D(Mesh *mesh, const int nz, const real_t sz);
 
+/** @brief Constructs the smallest possible [0,1]^dim serial mesh that can be
+    used later to obtain a ParMesh with @a elem_per_mpi elements, with the same
+    topology, for each of the @a mpi_cnt MPI tasks. For quads and hexes.
+
+    The serial mesh has the smallest possible number of elements. The parallel
+    mesh will be obtained by parallel refinements. Each MPI task will have
+    elements with the same topology (same number, same connectivity).
+
+    @param[in]  dim          dimension (2 or 3).
+    @param[in]  mpi_cnt      number of MPI tasks.
+    @param[in]  elem_per_mpi number of elements per MPI task.
+    @param[in]  print        shows meshing info in the terminal.
+    @param[out] par_ref      number of parallel refinement needed afterwards.
+    @param[out] partitioning partitioning to create the desired ParMesh.
+
+    Usual use case:
+    Mesh mesh = PartitionMPI(dim, mpi_cnt, elem_per_mpi, print, par_ref, par);
+    ParMesh pmesh(MPI_COMM_WORLD, mesh, par.GetData());
+    for (int lev = 0; lev < par_ref; lev++) { pmesh.UniformRefinement(); }   */
+Mesh PartitionMPI(int dim, int mpi_cnt, int elem_per_mpi, bool print,
+                  int &par_ref, Array<int> &partitioning);
+
 // shift cyclically 3 integers left-to-right
 inline void ShiftRight(int &a, int &b, int &c)
 {

mesh/mesh_readers.cpp

@@ -557,7 +557,7 @@ void Mesh::CreateVTKMesh(const Vector &points, const Array<int> &cell_data,
          fec = new QuadraticFECollection;
          fes = new FiniteElementSpace(this, fec, spaceDim);
          Nodes = new GridFunction(fes);
-         Nodes->MakeOwner(fec); // Nodes will destroy 'fec' and 'fes'
+         Nodes->MakeOwner(); // Nodes will destroy 'fec' and 'fes'
          own_nodes = 1;
 
          // Map vtk points to edge/face/element dofs
@@ -607,7 +607,7 @@ void Mesh::CreateVTKMesh(const Vector &points, const Array<int> &cell_data,
          fec = new H1_FECollection(order,Dim,BasisType::ClosedUniform);
          fes = new FiniteElementSpace(this, fec, spaceDim);
          Nodes = new GridFunction(fes);
-         Nodes->MakeOwner(fec); // Nodes will destroy 'fec' and 'fes'
+         Nodes->MakeOwner(); // Nodes will destroy 'fec' and 'fes'
          own_nodes = 1;
          Array<int> dofs;
 
@@ -1328,11 +1328,12 @@ void Mesh::ReadNURBSMesh(std::istream &input, int &curved, int &read_gf,
    if (NURBSext->HavePatches())
    {
       NURBSFECollection  *fec = new NURBSFECollection(NURBSext->GetOrder());
-      FiniteElementSpace *fes = new FiniteElementSpace(this, fec, Dim,
+      const int vdim = NURBSext->GetPatchSpaceDimension();
+      FiniteElementSpace *fes = new FiniteElementSpace(this, fec, vdim,
                                                        Ordering::byVDIM);
       Nodes = new GridFunction(fes);
-      Nodes->MakeOwner(fec);
-      NURBSext->SetCoordsFromPatches(*Nodes);
+      Nodes->MakeOwner();
+      NURBSext->SetCoordsFromPatches(*Nodes, vdim);
       own_nodes = 1;
       read_gf = 0;
       spaceDim = Nodes->VectorDim();
@@ -2524,7 +2525,7 @@ void Mesh::ReadGmshMesh(std::istream &input, int &curved, int &read_gf)
             nfes = new FiniteElementSpace(this, nfec, spaceDim,
                                           Ordering::byVDIM);
             Nodes_gf.SetSpace(nfes);
-            Nodes_gf.MakeOwner(nfec);
+            Nodes_gf.MakeOwner();
 
             int o = 0;
             int el_order = 1;
@@ -4097,7 +4098,7 @@ static void FinalizeCubitSecondOrderMesh(Mesh &mesh,
    FiniteElementSpace *fes = new FiniteElementSpace(&mesh, fec, Dim,
                                                     Ordering::byVDIM);
    GridFunction *Nodes = new GridFunction(fes);
-   Nodes->MakeOwner(fec); // Nodes will destroy 'fec' and 'fes'
+   Nodes->MakeOwner(); // Nodes will destroy 'fec' and 'fes'
    mesh.SetNodalGridFunction(Nodes, true);
 
    for (int block_id : unique_block_ids)

mesh/ncnurbs.cpp

@@ -2827,6 +2827,8 @@ void NCNURBSExtension::PropagateFactorsForKV(int rf_default)
       }
    }
 
+   delete face2elem;
+
    // For any unset entries of kvf, set to default refinement factor rf_default.
    for (size_t i=0; i<kvf.size(); ++i)
    {

mesh/nurbs.hpp

@@ -51,6 +51,21 @@ protected:
    /// Number of elements, defined by distinct knots.
    int NumOfElements;
 
+   // Stores the demko points
+   mutable Vector demko;
+
+   /// Compute all the Demko points
+   void ComputeDemko() const;
+
+#ifdef MFEM_USE_LAPACK
+   // Data for reusing banded matrix factorization in FindInterpolant().
+   mutable DenseMatrix fact_AB; /// Banded matrix factorization
+   mutable Array<int> fact_ipiv; /// Row pivot indices
+#else
+   mutable DenseMatrix A_coll_inv; /// Collocation matrix inverse
+#endif
+
+
 public:
    /// Create an empty KnotVector.
    KnotVector() = default;
@@ -63,6 +78,14 @@ public:
        order @a order and number of control points @a NCP. */
    KnotVector(int order, int NCP);
 
+   /** @brief Create a KnotVector with order @a order and knots @a knot.
+       If @a k has the correct number of repeated knots at the begin and end,
+       then this constructor will copy the knots as provided.
+       Otherwise, the knot vector will be extended by repeating the end knots
+       (order + 1) times. Internal knots will retain the multiplicity as given
+       in the input. */
+   KnotVector(int order, const Vector &k);
+
    /** @brief Create a KnotVector by passing in a degree, a Vector of interval
        lengths of length n, and a list of continuity of length n + 1.
 
@@ -103,13 +126,69 @@ public:
        with @a isElement for non-empty knot spans (elements). */
    int GetNKS() const { return NumOfControlPoints - Order; }
 
-   /** @brief Return the parameter for element reference coordinate @a xi
-       in [0,1], for the element beginning at knot @a ni. */
-   real_t getKnotLocation(real_t xi, int ni) const
-   { return (xi*knot(ni+1) + (1. - xi)*knot(ni)); }
+   /// Return whether knot location @a u is in a given span @a ni.
+   bool inSpan(real_t u, int ni) const
+   {
+      if ((u < knot(ni)) || (u > knot(ni+1))) { return false; }
+      return true;
+   }
 
    /// Return the index of the knot span containing parameter @a u.
-   int findKnotSpan(real_t u) const;
+   int GetSpan(real_t u) const;
+
+   /** @brief Return the reference coordinate in [0,1] for parameter @a u
+       in the element beginning at knot @a ni. */
+   real_t GetRefPoint(real_t u, int ni) const
+   { return (u-knot(ni))/(knot(ni+1)-knot(ni)); };
+
+   /** @brief Return the knot location for element reference coordinate @a xi
+       in [0,1], for the element beginning at knot @a ni. */
+   real_t GetKnotLocation(real_t xi, int ni) const
+   { return (xi*knot(ni+1) + (1. - xi)*knot(ni)); }
+
+   /** @brief Return the parameter for element reference coordinate @a xi
+       in [0,1], for the element beginning at knot @a ni. */
+   MFEM_DEPRECATED real_t getKnotLocation(real_t xi, int ni) const
+   { return (xi*knot(ni+1) + (1. - xi)*knot(ni)); } // Use GetKnotLocation instead
+
+   /// Return the index of the knot span containing parameter @a u.
+   MFEM_DEPRECATED int findKnotSpan(real_t u) const;  // Use GetSpan instead
+
+   /** Gives the @a i average knot location. Average is taken over @a Order
+       number of knots.*/
+   real_t GetGreville(int i) const;
+
+   void GetGreville(Vector &xi) const;
+
+   /** Gives the knot location where the @a i shape function is maximum.
+       Reverts to the Greville point if knot is repeated @a Order +1 times.
+       For background see:
+
+       Olivier Botella and Karim Shariff.
+       "B-spline methods in fluid dynamics."
+       International Journal of Computational Fluid Dynamics 17.2 (2003): 133-149.
+
+       Points are found using Newton iteration, with the Greville point as the
+       starting value. */
+   real_t GetBotella(int i) const;
+
+   void GetBotella(Vector &xi) const;
+
+   /** Gives the knot location of the @a i extremum of the Chebyshev spline.
+       For background see:
+
+       Stephen Demko
+       "On the existence of interpolating projections onto spline spaces."
+       Journal of approximation theory 43.2 (1985): 151-156.
+
+       Points are found using Remez iteration:
+        - Find interpolant, given by a, through given points, given by Demko
+        - Find extrema of this polynomial and update Demko points
+        - Repeat until converged
+        - Use the Greville point as starting point */
+   real_t GetDemko(int i) const;
+
+   void GetDemko(Vector &xi) const;
 
    // The following functions evaluate shape functions, which are B-spline basis
    // functions.
@@ -136,19 +215,32 @@ public:
    /** @brief Gives the locations of the maxima of the KnotVector in reference
        space. The function gives the knot span @a ks, the coordinate in the
        knot span @a xi, and the coordinate of the maximum in parameter space
-       @a u. */
-   void FindMaxima(Array<int> &ks, Vector &xi, Vector &u) const;
+       @a u.
+       The main purpose of this function is its use in FindInterpolant.
+       Use GetBotella instead for each shape function separately, perhaps in
+       conjuction with GetSpan and GetRefPoint.*/
+   MFEM_DEPRECATED void FindMaxima(Array<int> &ks, Vector &xi, Vector &u) const;
 
    /** @brief Global curve interpolation through the points @a x (overwritten).
        @a x is an array with the length of the spatial dimension containing
        vectors with spatial coordinates. The control points of the interpolated
        curve are returned in @a x in the same form.
+       Use GetInterpolant instead. For the knot location one can use either
+       GetBotella, GetDemko or GetGreville. FindInterpolant uses the Botella
+       points, however, the Demko points might be more appropriate. */
+   MFEM_DEPRECATED void FindInterpolant(Array<Vector*> &x, bool reuse_inverse);
 
-       The inverse of the collocation matrix, used in the interpolation, is
-       stored for repeated calls and used if @a reuse_inverse is true. Reuse is
-       valid only if this KnotVector has not changed since the initial call with
-       @a reuse_inverse false. */
-   void FindInterpolant(Array<Vector*> &x, bool reuse_inverse = false);
+   /** @brief Global curve interpolation through the points @a x (overwritten)
+       at the knot location @a u. The control points of the
+       interpolated curve are returned in @a x in the same form.
+       For the knot location one can use  for instance GetBotella, GetDemko or
+       GetGreville. The Demko points might be most appropriate.*/
+   void GetInterpolant(Array<Vector*> &x, const Vector &u,
+                       bool reuse_inverse = false) const;
+
+   /// Different interface to same routine
+   void GetInterpolant(const Vector &x, const Vector &u,
+                       Vector &a, bool reuse_inverse = false) const;
 
    /** Set @a diff, comprised of knots in @a kv not contained in this KnotVector.
        @a kv must be of the same order as this KnotVector. The current
@@ -191,6 +283,18 @@ public:
        number of samples of the shape functions per element.*/
    void PrintFunctions(std::ostream &os, int samples=11) const;
 
+   /** Prints the function with basis function coefficient @a a, and its first
+       and second derivatives associated with the KnotVector per element.
+       Use GetElements() to count the elements before using this function.
+       @a samples is the number of samples of the shape functions per element.*/
+   void PrintFunction(std::ostream &os, const Vector &a, int samples=11) const;
+
+   /** Prints the @a i-th function and its first and second
+       derivatives associated with the KnotVector per element. Use GetElements()
+       to count the elements before using this function. @a samples is the
+       number of samples of the shape functions per element.*/
+   void PrintFunction(std::ostream &os, int i, int samples=11) const;
+
    /// Destroys KnotVector
    ~KnotVector() { }
 
@@ -209,14 +313,6 @@ public:
    /** @brief Flag to indicate whether the KnotVector has been coarsened, which
        means it is ready for non-nested refinement. */
    bool coarse;
-
-#ifdef MFEM_USE_LAPACK
-   // Data for reusing banded matrix factorization in FindInterpolant().
-   DenseMatrix fact_AB; /// Banded matrix factorization
-   Array<int> fact_ipiv; /// Row pivot indices
-#else
-   DenseMatrix A_coll_inv; /// Collocation matrix inverse
-#endif
 };
 
 
@@ -596,28 +692,29 @@ protected:
        if the KnotVector index associated with edge @a edge is negative. */
    inline const KnotVector *KnotVec(int edge, int oedge, int *okv) const;
 
-   /// Throw an error if any patch has an inconsistent edge_to_ukv mapping.
-   void CheckPatches();
-
    /// Throw an error if any boundary patch has invalid KnotVector orientation.
-   void CheckBdrPatches();
+   MFEM_DEPRECATED void CheckBdrPatches();
+
+   /// Return the patch-topology edge indices that define the KnotVectors for
+   /// patch @a p in each parametric direction.
+   void GetPatchDirectionEdges(int p, Array<int> &edges);
 
    /** @brief Return the directions in @a kvdir of the KnotVectors in patch @a p
        based on the patch edge orientations. Each entry of @a kvdir is -1 if the
        KnotVector direction is flipped, +1 otherwise. */
    void CheckKVDirection(int p, Array <int> &kvdir);
 
-   /**  @brief Create the comprehensive set of KnotVectors. In 1D, this set is
-    identical to the unique set of KnotVectors. */
+   /** @brief Create the comprehensive set of KnotVectors, one per patch and
+       parametric direction, accounting for the edge orientations. */
    void CreateComprehensiveKV();
 
-   /**  Update the unique set of KnotVectors. In 1D, this set is identical to
-    the comprehensive set of KnotVectors. */
+   /** @brief Update the unique set of KnotVectors from the comprehensive set
+       of KnotVectors. */
    void UpdateUniqueKV();
 
    /** @brief Check if the comprehensive array of KnotVectors agrees with the
        unique set of KnotVectors, on each patch. Return false if there is a
-       difference, true otherwise. This function throws an error in 1D. */
+       difference, true otherwise. */
    bool ConsistentKVSets();
 
    /// Return KnotVectors in @a kv in each dimension for patch @a p.
@@ -794,6 +891,9 @@ public:
    void MergeGridFunctions(GridFunction *gf_array[], int num_pieces,
                            GridFunction &merged);
 
+   /// Returns false if any patch has an inconsistent edge_to_ukv mapping.
+   bool CheckPatches();
+
    /// Destroy a NURBSExtension.
    virtual ~NURBSExtension();
 
@@ -820,6 +920,13 @@ public:
    /// Return the dimension of the reference space (not physical space).
    int Dimension() const { return patchTopo->Dimension(); }
 
+   /** @brief Return the physical dimension of the NURBS geometry
+
+       The physical dimension is inferred from the first patch,
+       i.e. number of coordinates per control point minus one (for the weight).
+       This method requires patch data to be present, i.e. HavePatches() == true */
+   int GetPatchSpaceDimension() const;
+
    /// Return the number of patches.
    int GetNP()     const { return patchTopo->GetNE(); }
 
@@ -933,9 +1040,9 @@ public:
    void ConvertToPatches(const Vector &Nodes);
    /// Set KnotVectors from @a patches and construct mesh and space data.
    void SetKnotsFromPatches();
-   /** @brief Set FE coordinates in @a Nodes, using data from @a patches, and
-       erase @a patches. */
-   void SetCoordsFromPatches(Vector &Nodes);
+   /** @brief Set FE coordinates in @a Nodes, using data from @a patches,
+       with physical vector dimension @a vdim, and erase @a patches. */
+   void SetCoordsFromPatches(Vector &Nodes, int vdim);
 
    /** @brief Read a GridFunction @a sol from stream @a input, written
        patch-by-patch, e.g. with PrintSolution(). */

mesh/pmesh.cpp

@@ -86,7 +86,7 @@ ParMesh::ParMesh(const ParMesh &pmesh, bool copy_nodes)
       ParFiniteElementSpace *pfes_copy =
          new ParFiniteElementSpace(*fes, *this, fec_copy);
       Nodes = new ParGridFunction(pfes_copy);
-      Nodes->MakeOwner(fec_copy);
+      Nodes->MakeOwner();
       *Nodes = *pmesh.Nodes;
       own_nodes = 1;
    }
@@ -286,7 +286,7 @@ ParMesh::ParMesh(MPI_Comm comm, Mesh &mesh, const int *partitioning_,
             new ParFiniteElementSpace(this, nfec, glob_fes->GetVDim(),
                                       glob_fes->GetOrdering());
          Nodes = new ParGridFunction(pfes);
-         Nodes->MakeOwner(nfec); // Nodes will own nfec and pfes
+         Nodes->MakeOwner(); // Nodes will own nfec and pfes
       }
       own_nodes = 1;
 
@@ -2032,7 +2032,7 @@ void ParMesh::SetCurvature(int order, bool discont, int space_dim, int ordering)
    auto pnodes = new ParGridFunction(nfes);
    GetNodes(*pnodes);
    NewNodes(*pnodes, true);
-   Nodes->MakeOwner(nfec);
+   Nodes->MakeOwner();
 }
 
 void ParMesh::SetNodalFESpace(FiniteElementSpace *nfes)
@@ -2067,9 +2067,7 @@ void ParMesh::EnsureParNodes()
       *new_nodes = *Nodes;
       if (Nodes->OwnFEC())
       {
-         new_nodes->MakeOwner(Nodes->OwnFEC());
-         Nodes->MakeOwner(NULL); // takes away ownership of 'fec' and 'fes'
-         delete Nodes->FESpace();
+         new_nodes->MakeOwner();
       }
       delete Nodes;
       Nodes = new_nodes;
@@ -3041,7 +3039,7 @@ void ParMesh::GetSharedFaceTransformationsByLocalIndex(
    // for ghost faces we need a special version of GetFaceTransformation
    if (is_ghost)
    {
-      GetGhostFaceTransformation(FElTr, face_type, face_geom);
+      GetGhostFaceTransformation(FaceNo, FElTr);
       mask |= FaceElementTransformations::HAVE_FACE;
    }
 
@@ -3064,19 +3062,29 @@ void ParMesh::GetSharedFaceTransformationsByLocalIndex(
 }
 
 void ParMesh::GetGhostFaceTransformation(
-   FaceElementTransformations &FElTr, Element::Type face_type,
-   Geometry::Type face_geom) const
+   int FaceNo, FaceElementTransformations &FElTr) const
 {
+   MFEM_ASSERT(FaceNo >= GetNumFaces(), "Not a ghost face.");
+
+   // use the local face data
+   const int LocFaceNo = nc_faces_info[faces_info[FaceNo].NCFace].MasterFace;
+   FElTr.Attribute = (Dim == 1) ? 1 : faces[LocFaceNo]->GetAttribute();
+   FElTr.ElementNo = FaceNo;
+   FElTr.ElementType = ElementTransformation::FACE;
+   FElTr.mesh = this;
+
    // calculate composition of FElTr.Loc1 and FElTr.Elem1
    DenseMatrix &face_pm = FElTr.GetPointMat();
    FElTr.Reset();
    if (Nodes == NULL)
    {
+      const Element::Type face_type = GetFaceElementType(LocFaceNo);
       FElTr.Elem1->Transform(FElTr.Loc1.Transf.GetPointMat(), face_pm);
       FElTr.SetFE(GetTransformationFEforElementType(face_type));
    }
    else
    {
+      const Geometry::Type face_geom = GetFaceGeometry(LocFaceNo);
       const FiniteElement* face_el =
          Nodes->FESpace()->GetTraceElement(FElTr.Elem1No, face_geom);
       MFEM_VERIFY(dynamic_cast<const NodalFiniteElement*>(face_el),
@@ -5513,7 +5521,7 @@ Mesh ParMesh::GetSerialMesh(int save_rank) const
                                               spaceDim,
                                               GetNodalFESpace()->GetOrdering());
       serialmesh.SetNodalFESpace(fespace_serial);
-      serialmesh.GetNodes()->MakeOwner(fec_serial);
+      serialmesh.GetNodes()->MakeOwner();
       // The serial mesh owns its Nodes and they, in turn, own fec_serial and
       // fespace_serial.
    }

mesh/pmesh.hpp

@@ -150,15 +150,7 @@ protected:
                     int elem, int start, int end, const int fverts[][N]);
 
    void GetGhostFaceTransformation(
-      FaceElementTransformations &FElTr, Element::Type face_type,
-      Geometry::Type face_geom) const;
-   void GetGhostFaceTransformation(
-      FaceElementTransformations *FElTr, Element::Type face_type,
-      Geometry::Type face_geom) const
-   {
-      MFEM_ASSERT(FElTr, "Missing FaceElementTransformations object!");
-      GetGhostFaceTransformation(*FElTr, face_type, face_geom);
-   }
+      int FaceNo, FaceElementTransformations &FElTr) const;
 
    /// Update the groups after triangle refinement
    void RefineGroups(const DSTable &v_to_v, int *middle);

mesh/pncmesh.cpp

@@ -1195,8 +1195,17 @@ void ParNCMesh::GetFaceNeighbors(ParMesh &pmesh)
       }
    }
 
-   // If there are shared slaves, they will also need to be updated.
+   // If there are shared slaves, they will also need to be updated. First,
+   // check whether the update has already been done.
+   bool sharedUpdated = false;
    if (shared.slaves.Size())
+   {
+      int nfaces = NFaces, nghosts = NGhostFaces;
+      if (Dim <= 2) { nfaces = NEdges, nghosts = NGhostEdges; }
+      sharedUpdated = (pmesh.faces_info.Size() == nfaces + nghosts);
+   }
+
+   if (shared.slaves.Size() && !sharedUpdated)
    {
       int nfaces = NFaces, nghosts = NGhostFaces;
       if (Dim <= 2) { nfaces = NEdges, nghosts = NGhostEdges; }
@@ -1310,7 +1319,6 @@ void ParNCMesh::GetFaceNeighbors(ParMesh &pmesh)
       }
    }
 
-
    // In 3D some extra orientation data structures can be needed.
    if (Dim == 3)
    {

mesh/submesh/psubmesh.hpp

@@ -34,14 +34,16 @@ class ParNCSubMesh;
  * subset of the parent Mesh and reuses the parallel distribution.
  *
  * The attributes are taken from the parent. That means if a volume is extracted
- * from a volume, it has the same domain attribute as the parent. Its boundary
- * attributes are generated (there will be one boundary attribute 1 for all of
- * the boundaries).
+ * from a volume, it has the same domain attribute as the parent. Its new
+ * boundary attributes are, for any boundary common to the parent and the new
+ * submesh, the boundary attribute of the parent; and, for all new boundaries,
+ * a single, generated, common attribute equal to one plus the largest boundary
+ * attribute of the parent.
  *
  * If a surface is extracted from a volume, the boundary attribute from the
- * parent is assigned to be the new domain attribute. Its boundary attributes
- * are generated (there will be one boundary attribute 1 for all of the
- * boundaries).
+ * parent is assigned to be the new domain attribute. Its new boundary attribute
+ * is a single, generated, common attribute equal to one plus the largest
+ * boundary attribute of the parent.
  *
  * For more customized boundary attributes, the resulting ParSubMesh has to be
  * postprocessed.

mesh/submesh/submesh.hpp

@@ -28,14 +28,16 @@ class NCSubMesh;
  * subset of the parents Mesh and reuses the parallel distribution.
  *
  * The attributes are taken from the parent. That means if a volume is extracted
- * from a volume, it has the same domain attribute as the parent. Its boundary
- * attributes are generated (there will be one boundary attribute 1 for all of
- * the boundaries).
+ * from a volume, it has the same domain attribute as the parent. Its new
+ * boundary attributes are, for any boundary common to the parent and the new
+ * submesh, the boundary attribute of the parent; and, for all new boundaries,
+ * a single, generated, common attribute equal to one plus the largest boundary
+ * attribute of the parent.
  *
  * If a surface is extracted from a volume, the boundary attribute from the
- * parent is assigned to be the new domain attribute. Its boundary attributes
- * are generated (there will be one boundary attribute 1 for all of the
- * boundaries).
+ * parent is assigned to be the new domain attribute. Its new boundary attribute
+ * is a single, generated, common attribute equal to one plus the largest
+ * boundary attribute of the parent.
  *
  * For more customized boundary attributes, the resulting SubMesh has to be
  * postprocessed.

miniapps/common/mesh_extras.cpp

@@ -232,27 +232,6 @@ MergeMeshNodes(Mesh * mesh, int logging)
    }
 }
 
-void AttrToMarker(int max_attr, const Array<int> &attrs, Array<int> &marker)
-{
-   MFEM_ASSERT(attrs.Max() <= max_attr, "Invalid attribute number present.");
-
-   marker.SetSize(max_attr);
-   if (attrs.Size() == 1 && attrs[0] == -1)
-   {
-      marker = 1;
-   }
-   else
-   {
-      marker = 0;
-      for (int j=0; j<attrs.Size(); j++)
-      {
-         int attr = attrs[j];
-         MFEM_VERIFY(attr > 0, "Attribute number less than one!");
-         marker[attr-1] = 1;
-      }
-   }
-}
-
 void AffineTransformation::Eval(Vector &V, ElementTransformation &T,
                                 const IntegrationPoint &ip)
 {

miniapps/common/mesh_extras.hpp

@@ -33,9 +33,14 @@ void MergeMeshNodes(Mesh * mesh, int logging);
 /// Convert a set of attribute numbers to a marker array
 /** The marker array will be of size max_attr and it will contain only zeroes
     and ones. Ones indicate which attribute numbers are present in the attrs
-    array. In the special case when attrs has a single entry equal to -1 the
-    marker array will contain all ones. */
-void AttrToMarker(int max_attr, const Array<int> &attrs, Array<int> &marker);
+    array. In the special case when attrs has an entry equal to -1 the marker
+    array will contain all ones. */
+inline
+void AttrToMarker(int max_attr, const Array<int> &attrs, Array<int> &marker)
+{
+   if (attrs.Find(-1) != -1) { (marker = Array<int>(max_attr)) = 1; }
+   else { marker = AttributeSets::AttrToMarker(max_attr, attrs); }
+}
 
 /// Transform a mesh according to an arbitrary affine transformation
 ///    y = A x + b

miniapps/common/particles_extras.cpp

@@ -174,7 +174,6 @@ ParticleTrajectories::ParticleTrajectories(const ParticleSet &particles,
 
 void ParticleTrajectories::AddSegmentStart()
 {
-   if (!pset.GetNParticles()) { return; }
    // Create a new mesh for all particle segments for this timestep
    segment_meshes.emplace_front(1, pset.GetNParticles()*2,
                                 pset.GetNParticles(),
@@ -200,11 +199,10 @@ void ParticleTrajectories::AddSegmentStart()
 
 void ParticleTrajectories::SetSegmentEnd()
 {
-   if (segment_meshes.empty()) { return; } // no segments to end
-
    const Array<ParticleSet::IDType> &end_ids = pset.GetIDs();
 
-   // Add all endpoint vertices + segments for all particles
+   // Add all endpoint vertices + segments for all particles that were in
+   // SetSegmentStart
    int num_start = segment_ids.front().Size();
    for (int i = 0; i < num_start; i++)
    {
@@ -230,11 +228,6 @@ void ParticleTrajectories::SetSegmentEnd()
 void ParticleTrajectories::Visualize()
 {
    SetSegmentEnd();
-   if (segment_meshes.empty() && !mesh)
-   {
-      AddSegmentStart();
-      return;
-   }
 
    // Create a mesh of all the trajectory segments
    std::vector<Mesh*> all_meshes;
@@ -246,8 +239,23 @@ void ParticleTrajectories::Visualize()
    {
       all_meshes.push_back(mesh);
    }
+   if (mesh_bb)
+   {
+      all_meshes.push_back(mesh_bb);
+   }
 
    Mesh trajectories(all_meshes.data(), all_meshes.size());
+   bool vis = trajectories.GetNE() > 0;
+#ifdef MFEM_USE_MPI
+   MPI_Allreduce(MPI_IN_PLACE, &vis, 1, MFEM_MPI_CXX_BOOL,
+                 MPI_LOR, pset.GetComm());
+#endif // MFEM_USE_MPI
+   if (!vis) // if all rank have 0 elements, skip visualization
+   {
+      AddSegmentStart();
+      return;
+   }
+
 
 #ifdef MFEM_USE_MPI
    VisualizeMesh(sock, vishost, visport, trajectories, comm,
@@ -260,5 +268,97 @@ void ParticleTrajectories::Visualize()
    AddSegmentStart();
 }
 
+void ParticleTrajectories::SetVisualizationBoundingBox(const Vector &xmin,
+                                                       const Vector &xmax)
+{
+   MFEM_VERIFY(xmin.Size() == pset.GetDim() &&
+               xmax.Size() == pset.GetDim(),
+               "Bounding box dimension must match ParticleSet dimension.");
+
+   // Create a box mesh for visualization
+   if (mesh_bb)
+   {
+      delete mesh_bb;
+      mesh_bb = nullptr;
+   }
+
+   if (pset.GetDim() == 2)
+   {
+      int dim = 2;
+      int nvert = 4;
+      int nelem = 4;
+      mesh_bb = new Mesh(1, nvert, nelem, 0, dim);
+      Vector v0(dim), v1(dim), v2(dim), v3(dim);
+      v0 = xmin;
+      v1 = xmax;
+      v2[0] = xmax[0]; v2[1] = xmin[1];
+      v3[0] = xmin[0]; v3[1] = xmax[1];
+
+      mesh_bb->AddVertex(v0);
+      mesh_bb->AddVertex(v1);
+      mesh_bb->AddVertex(v2);
+      mesh_bb->AddVertex(v3);
+
+      int vi[2] = {0,1};
+      mesh_bb->AddSegment(vi);
+      vi[0] = 1; vi[1] = 2;
+      mesh_bb->AddSegment(vi);
+      vi[0] = 2; vi[1] = 3;
+      mesh_bb->AddSegment(vi);
+      vi[0] = 3; vi[1] = 0;
+      mesh_bb->AddSegment(vi);
+      mesh_bb->FinalizeMesh();
+   }
+   else // dim == 3
+   {
+      int dim = 3;
+      int nvert = 8;
+      int nelem = 12;
+      mesh_bb = new Mesh(1, nvert, nelem, 0, dim);
+      Vector v(dim);
+
+      // Vertices
+      v[0] = xmin[0]; v[1] = xmin[1]; v[2] = xmin[2];
+      mesh_bb->AddVertex(v); // 0: 000
+      v[0] = xmax[0]; v[1] = xmin[1]; v[2] = xmin[2];
+      mesh_bb->AddVertex(v); // 1: 100
+      v[0] = xmax[0]; v[1] = xmax[1]; v[2] = xmin[2];
+      mesh_bb->AddVertex(v); // 2: 110
+      v[0] = xmin[0]; v[1] = xmax[1]; v[2] = xmin[2];
+      mesh_bb->AddVertex(v); // 3: 010
+
+      v[0] = xmin[0]; v[1] = xmin[1]; v[2] = xmax[2];
+      mesh_bb->AddVertex(v); // 4: 001
+      v[0] = xmax[0]; v[1] = xmin[1]; v[2] = xmax[2];
+      mesh_bb->AddVertex(v); // 5: 101
+      v[0] = xmax[0]; v[1] = xmax[1]; v[2] = xmax[2];
+      mesh_bb->AddVertex(v); // 6: 111
+      v[0] = xmin[0]; v[1] = xmax[1]; v[2] = xmax[2];
+      mesh_bb->AddVertex(v); // 7: 011
+
+      // Segments
+      int vi[2];
+      // Bottom face
+      vi[0] = 0; vi[1] = 1; mesh_bb->AddSegment(vi);
+      vi[0] = 1; vi[1] = 2; mesh_bb->AddSegment(vi);
+      vi[0] = 2; vi[1] = 3; mesh_bb->AddSegment(vi);
+      vi[0] = 3; vi[1] = 0; mesh_bb->AddSegment(vi);
+
+      // Top face
+      vi[0] = 4; vi[1] = 5; mesh_bb->AddSegment(vi);
+      vi[0] = 5; vi[1] = 6; mesh_bb->AddSegment(vi);
+      vi[0] = 6; vi[1] = 7; mesh_bb->AddSegment(vi);
+      vi[0] = 7; vi[1] = 4; mesh_bb->AddSegment(vi);
+
+      // Vertical edges
+      vi[0] = 0; vi[1] = 4; mesh_bb->AddSegment(vi);
+      vi[0] = 1; vi[1] = 5; mesh_bb->AddSegment(vi);
+      vi[0] = 2; vi[1] = 6; mesh_bb->AddSegment(vi);
+      vi[0] = 3; vi[1] = 7; mesh_bb->AddSegment(vi);
+
+      mesh_bb->FinalizeMesh();
+   }
+}
+
 } // namespace common
 } // namespace mfem

miniapps/common/particles_extras.hpp

@@ -46,7 +46,8 @@ class ParticleTrajectories
 {
 protected:
    const ParticleSet &pset;
-   Mesh *mesh = nullptr;
+   Mesh *mesh = nullptr; // optional edge mesh to visualize along with particles
+   Mesh *mesh_bb = nullptr; // optional bounding box mesh for visualization
 
    socketstream sock;
    /// Track particle IDs that exist at the segment start.
@@ -90,10 +91,24 @@ public:
                         const char *keys_=nullptr);
 
    /// Add a mesh to be visualized along with the particle trajectories.
-   void AddMeshForVisualization(Mesh *mesh_) { mesh = mesh_; }
+   void AddMeshForVisualization(Mesh *mesh_)
+   {
+      MFEM_VERIFY(mesh_->Dimension() == 1,
+                  "Mesh dimension must be 1 to match the particle trajectory.");
+      mesh = mesh_;
+   }
 
    /// Visualize the particle trajectories (and mesh if provided).
    void Visualize();
+
+   /// Set the bounding box for visualization.
+   void SetVisualizationBoundingBox(const Vector &xmin, const Vector &xmax);
+
+   /// Destructor
+   ~ParticleTrajectories()
+   {
+      delete mesh_bb;
+   }
 };
 
 

miniapps/electromagnetics/CMakeLists.txt

@@ -34,11 +34,13 @@ if (MFEM_USE_MPI)
    EXTRA_HEADERS maxwell_solver.hpp ${MFEM_MINIAPPS_COMMON_HEADERS}
    LIBRARIES mfem-common)
 
- add_mfem_miniapp(lorentz
-   MAIN lorentz.cpp
-   EXTRA_HEADERS ${MFEM_MINIAPPS_COMMON_HEADERS}
-   LIBRARIES mfem-common)
-
+ if (MFEM_USE_GSLIB)
+  add_mfem_miniapp(lorentz
+    MAIN lorentz.cpp
+    EXTRA_HEADERS ${MFEM_MINIAPPS_COMMON_HEADERS}
+    LIBRARIES mfem-common)
+endif()
+  
   # Add the corresponding tests to the "test" target
   if (MFEM_ENABLE_TESTING)
     add_test(NAME tesla_np=4

miniapps/electromagnetics/makefile

@@ -21,7 +21,10 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 SEQ_MINIAPPS =
-PAR_MINIAPPS = volta tesla maxwell joule lorentz
+PAR_MINIAPPS = volta tesla maxwell joule
+ifeq ($(MFEM_USE_GSLIB), YES)
+   PAR_MINIAPPS += lorentz
+endif
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
 else
@@ -51,9 +54,11 @@ all: $(MINIAPPS)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $@.o $@_solver.o $(COMMON_LIB) \
 	  $(MFEM_LIBS)
 
+ifeq ($(MFEM_USE_MPI),YES)
 lorentz: %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK) | lib-common
 	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<)
 	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $@.o $(COMMON_LIB) $(MFEM_LIBS)
+endif
 
 # Rules for compiling miniapp dependencies
 $(addsuffix _solver.o,$(MINIAPPS)): \
@@ -112,10 +117,10 @@ joule-test-par: joule
 lorentz-test-par: lorentz-test-1 lorentz-test-2
 lorentz-test-1: lorentz volta-test-3
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
-	-er Volta-AMR-Parallel -ec 2 -x0 '0.5 0.5 0.9' -p0 '1 0 0')
+	-er Volta-AMR-Parallel -ec 2 -npt 100 -xmin '0.0 0.0 0.0' -xmax '1.0 1.0 1.0' -pmin '1 0 0' -pmax '1 0 0' -rdf 0 -vt 0 -nt 100')
 lorentz-test-2: lorentz tesla-test-2
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
-	-br Tesla-AMR-Parallel -bc 2 -x0 '0.1 0.5 0.1' -p0 '0 0.4 0.1' -tf 9)
+	-br Tesla-AMR-Parallel -bc 2 -br Tesla-AMR-Parallel -npt 10 -xmin '0.0 0.0 0.0' -xmax '1.0 1.0 1.0' -pmin '0 0.1 0.05' -pmax '0 0.4 0.1' -nt 1000 -rdf 0 -vt 0)
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

miniapps/fluids/navier/navier_bifurcation.cpp

@@ -127,7 +127,7 @@ int main(int argc, char *argv[])
    }
 
    // Load mesh + complete any serial refinements
-   Mesh mesh("../../data/channel-bifurcation-2d.mesh");
+   Mesh mesh("../../../data/channel-bifurcation-2d.mesh");
    for (int lev = 0; lev < ctx.rs_levels; lev++)
    {
       mesh.UniformRefinement();

miniapps/gslib/CMakeLists.txt

@@ -15,16 +15,14 @@ set(MESH_GF_FILES
     triple-pt-1.gf
     triple-pt-2.gf
 )
-
+# add target which keeps required mesh files in sync
 set(SRC_MESH_GF_FILES)
 foreach(MESH_GF_FILE ${MESH_GF_FILES})
   list(APPEND SRC_MESH_GF_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${MESH_GF_FILE})
 endforeach()
-add_custom_command(OUTPUT data_is_copied
+add_custom_target(copy_miniapps_gslib_data
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SRC_MESH_GF_FILES} .
-  COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
-  COMMENT "Copying gslib miniapps data files ...")
-add_custom_target(copy_miniapps_gslib_data DEPENDS data_is_copied)
+  COMMENT "Syncing gslib miniapps data files ...")
 
 if (MFEM_USE_GSLIB)
   add_mfem_miniapp(schwarz_ex1

miniapps/meshing/CMakeLists.txt

@@ -27,11 +27,9 @@ set(SRC_MESH_FILES)
 foreach(MESH_FILE ${MESH_FILES})
   list(APPEND SRC_MESH_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${MESH_FILE})
 endforeach()
-add_custom_command(OUTPUT data_is_copied
+add_custom_target(copy_miniapps_meshing_data
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SRC_MESH_FILES} .
-  COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
-  COMMENT "Copying meshing miniapps data files ...")
-add_custom_target(copy_miniapps_meshing_data DEPENDS data_is_copied)
+  COMMENT "Syncing meshing miniapps data files ...")
 
 add_mfem_miniapp(klein-bottle
   MAIN klein-bottle.cpp

miniapps/meshing/mesh-explorer.cpp

@@ -224,7 +224,7 @@ Mesh *skin_mesh(Mesh *mesh)
          FiniteElementSpace *fes_copy =
             new FiniteElementSpace(*fes, bmesh, fec_copy);
          GridFunction *bdr_nodes = new GridFunction(fes_copy);
-         bdr_nodes->MakeOwner(fec_copy);
+         bdr_nodes->MakeOwner();
 
          bmesh->NewNodes(*bdr_nodes, true);
 

miniapps/meshing/minimal-surface.cpp

@@ -389,7 +389,7 @@ public:
             add(*nodes, delta, *nodes);
          }
          // x = lambda*nodes + (1-lambda)*x
-         add(lambda, *nodes, (real_t)(1.0-lambda), x, x);
+         add(lambda, *nodes, (1.0-lambda), x, x);
          return Converged(rnorm);
       }
 

miniapps/meshing/pminimal-surface.cpp

@@ -396,7 +396,7 @@ public:
             add(*nodes, delta, *nodes);
          }
          // x = lambda*nodes + (1-lambda)*x
-         add(lambda, *nodes, (real_t)(1.0-lambda), x, x);
+         add(lambda, *nodes, (1.0-lambda), x, x);
          return Converged(rnorm);
       }