Add bool ownership flag to AddBdrConstraintIntegrator

Prototype of "maybe owning" smart pointer semantics.

.github/stale.yml

@@ -1,61 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 30
-
-# Number of days of inactivity before an Issue or Pull Request with the stale
-# label is closed.  Set to false to disable. If disabled, issues still need to
-# be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale.
-# Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set
-# to `[]` to disable
-exemptLabels:
-  - bug
-  - WIP
-  - ready-for-review
-  - in-review
-  - in-next
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking an issue as stale
-staleLabel: stale
-
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  :warning: This issue or PR has been automatically marked as stale because it has not
-  had any activity in the last month. *If no activity occurs in the next week, it will
-  be automatically closed.* Thank you for your contributions.
-
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-# only: issues
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed

.github/workflows/stale.yml

@@ -0,0 +1,31 @@
+# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
+# For more information, see: https://github.com/actions/stale
+name: Mark stale issues and pull requests
+
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: '0 0 * * *'
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+      actions: write
+
+    steps:
+    - uses: actions/stale@v9
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: ':warning: This issue has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
+        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
+        days-before-stale: 30
+        days-before-close: 7
+        stale-issue-label: 'stale'
+        stale-pr-label: 'stale'
+        operations-per-run: 500
+        exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
+        exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"

CHANGELOG

@@ -10,6 +10,8 @@
 
 Version 4.7.1 (development)
 ===========================
+- Refactored ALGOIM cut integration rules. The interface is unified with
+  the interface for moment based cut integration rules.
 
 Discretization improvements
 ---------------------------
@@ -18,6 +20,8 @@ Discretization improvements
 
 - Added support for boundary constraints to the hybridization class.
 
+- Added support for external boundary submeshes with nonconformal mesh adaptation.
+
 Meshing improvements
 --------------------
 - The ExodusII reader now handles pyramid and wedge element types. Mixed meshes
@@ -50,8 +54,15 @@ GPU computing
   or by explicitly calling `KernelReporter::Enable`. Users can then add
   specializations for these kernels to achieve higher performance.
 
+- Element assembly kernels have been added for low-order refined to
+  high-order transfer operators. New kernels can be offloaded as device
+  kernels. Example usage may be found in lor-transfer.cpp under miniapps/tools.
+
 Miscellaneous
 -------------
+- Added support for SUNDIALS v7. See the section "API changes" for some small
+  changes related to this new version.
+
 - Refactored the `ARKStepSolver` class (ARKODE interface) to use
   `TimeDependentOperator::Mult` only when the associated ODE operator is
   expressed in explicit form (i.e., `TimeDependentOperator::isExplicit()`),
@@ -68,6 +79,18 @@ API changes
 -----------
 - API change: in class GridFunction, 'fec' was renamed to 'fec_owned'.
 
+- API change: support for SUNDIALS v7:
+  * the SUNDIALS types `realtype` and `booleantype` are no longer defined by v7
+    and therefore MFEM now uses the new type names `sunrealtype` and
+    `sunbooleantype`, respectively, which MFEM defines when using SUNDIALS < v6
+    where these types were not defined.
+  * The SUNDIALS macro `SUNLS_SUCCESS` and some other `*_SUCCESS` macros were
+    removed and replaced by `SUN_SUCCESS` in v7, so to avoid tedious checks for
+    SUNDIALS versions, MFEM now defines and uses the constant `SUN_SUCCESS` when
+    using SUNDIALS < v7.
+  * The constants `SUN_PREC_*`, introduced by SUNDIALS v6 are now introduced by
+    MFEM when using SUNDIALS < v6 to avoid tedious version checks.
+
 
 Version 4.7, released on May 7, 2024
 ====================================
@@ -154,6 +177,15 @@ New and updated examples and miniapps
 - Added two new example codes: 38 and 39/39p described above. Substantially
   updated Example 18/18p.
 
+- Added ODE solvers selection routines. This creates a uniformity across examples,
+  miniapps and other executables in regard to ODE(time-integrator) selection.
+
+- Added new mechanism for retrieving and setting state vectors in ODE solvers.
+  This is relevant for AB/AM and gen-alpha solvers.
+
+- Added ODEsolver/ODEsolver2 unit tests to verify order of convergence and
+  read/write functionality.
+
 Miscellaneous
 -------------
 - Updated the Doxygen documentation style, which now requires Doxygen version

CMakeLists.txt

@@ -340,7 +340,10 @@ if (MFEM_USE_SUNDIALS)
   if (MFEM_USE_HIP)
     list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
   endif()
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
+  # The Core component was added in SUNDIALS v7, so we treat it as optional in
+  # order to support older versions.
+  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS}
+    OPTIONAL_COMPONENTS Core)
 endif()
 
 # SuperLU_DIST can only be enabled in parallel

INSTALL

@@ -502,10 +502,14 @@ MFEM_USE_CODIPACK = YES/NO
 MFEM_USE_ALGOIM = YES/NO
   Enable the usage of Algoim - a collection of high-order accurate numerical
   methods and C++ algorithms for working with implicitly-defined geometry and
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM
-  provides interface to Algoim v1. Thus, to check out the specific state use:
+  level set methods, see https://algoim.github.io. MFEM provides interface to
+  Algoim v1. To check out the specific Algoim state use:
+  https://github.com/algoim/algoim
   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
-  https://algoim.github.io
+  The Algoim library requires the Blitz++ library. To use the latest state of
+  Blitz++ that has been tested with MFEM, use:
+  https://github.com/blitzpp/blitz
+  git checkout f24a250a43dff88c31ad92916da828b7ea9a98b7
 
 MFEM_USE_ADFORWARD = YES/NO
   Enable forward mode for AD packages. This option is valid

config/cmake/modules/FindSUNDIALS.cmake

@@ -31,4 +31,5 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
   ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
   ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
   ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
-  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)
+  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol
+  ADD_COMPONENT Core "include" sundials/sundials_core.h "lib" sundials_core)

config/defaults.mk

@@ -289,6 +289,13 @@ endif
 ifeq ($(MFEM_USE_HIP),YES)
    SUNDIALS_LIB += -lsundials_nvechip
 endif
+SUNDIALS_CORE_PAT = $(subst\
+ @MFEM_DIR@,$(MFEM_DIR),$(SUNDIALS_DIR))/lib*/libsundials_core.*
+ifeq ($(MFEM_USE_SUNDIALS),YES)
+   ifneq ($(wildcard $(SUNDIALS_CORE_PAT)),)
+      SUNDIALS_LIB += -lsundials_core
+   endif
+endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
@@ -533,8 +540,10 @@ ifdef GOTCHA_DIR
 endif
 
 # BLITZ library configuration
-BLITZ_DIR = @MFEM_DIR@/../blitz
+# BLITZ_DIR must be the custom installation folder (-DCMAKE_INSTALL_PREFIX).
+BLITZ_DIR = @MFEM_DIR@/../blitz/install
 BLITZ_OPT = -I$(BLITZ_DIR)/include
+# On intel machines, use /lib64 instead of /lib.
 BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz
 
 # ALGOIM library configuration

examples/ex10.cpp

@@ -3,14 +3,14 @@
 // Compile with: make ex10
 //
 // Sample runs:
-//    ex10 -m ../data/beam-quad.mesh -s 3 -r 2 -o 2 -dt 3
-//    ex10 -m ../data/beam-tri.mesh -s 3 -r 2 -o 2 -dt 3
-//    ex10 -m ../data/beam-hex.mesh -s 2 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-tet.mesh -s 2 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-wedge.mesh -s 2 -r 1 -o 2 -dt 3
-//    ex10 -m ../data/beam-quad.mesh -s 14 -r 2 -o 2 -dt 0.03 -vs 20
-//    ex10 -m ../data/beam-hex.mesh -s 14 -r 1 -o 2 -dt 0.05 -vs 20
-//    ex10 -m ../data/beam-quad-amr.mesh -s 3 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-quad.mesh -s 23 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-tri.mesh -s 23 -r 2 -o 2 -dt 3
+//    ex10 -m ../data/beam-hex.mesh -s 22 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-tet.mesh -s 22 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-wedge.mesh -s 22 -r 1 -o 2 -dt 3
+//    ex10 -m ../data/beam-quad.mesh -s 4 -r 2 -o 2 -dt 0.03 -vs 20
+//    ex10 -m ../data/beam-hex.mesh -s 4 -r 1 -o 2 -dt 0.05 -vs 20
+//    ex10 -m ../data/beam-quad-amr.mesh -s 23 -r 2 -o 2 -dt 3
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -160,7 +160,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/beam-quad.mesh";
    int ref_levels = 2;
    int order = 2;
-   int ode_solver_type = 3;
+   int ode_solver_type = 23;
    real_t t_final = 300.0;
    real_t dt = 3.0;
    real_t visc = 1e-2;
@@ -177,11 +177,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
-                  "            11 - Forward Euler, 12 - RK2,\n\t"
-                  "            13 - RK3 SSP, 14 - RK4."
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  ODESolver::Types.c_str());
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -213,28 +209,7 @@ int main(int argc, char *argv[])
    // 3. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
-   ODESolver *ode_solver;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1: ode_solver = new BackwardEulerSolver; break;
-      case 2: ode_solver = new SDIRK23Solver(2); break;
-      case 3: ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
+   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -371,7 +346,6 @@ int main(int argc, char *argv[])
    }
 
    // 10. Free the used memory.
-   delete ode_solver;
    delete mesh;
 
    return 0;

examples/ex10p.cpp

@@ -3,14 +3,14 @@
 // Compile with: make ex10p
 //
 // Sample runs:
-//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 3 -rs 2 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-tri.mesh -s 3 -rs 2 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 2 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-tet.mesh -s 2 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-wedge.mesh -s 2 -rs 1 -dt 3
-//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 14 -rs 2 -dt 0.03 -vs 20
-//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 14 -rs 1 -dt 0.05 -vs 20
-//    mpirun -np 4 ex10p -m ../data/beam-quad-amr.mesh -s 3 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 23 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-tri.mesh -s 23 -rs 2 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 22 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-tet.mesh -s 22 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-wedge.mesh -s 22 -rs 1 -dt 3
+//    mpirun -np 4 ex10p -m ../data/beam-quad.mesh -s 4 -rs 2 -dt 0.03 -vs 20
+//    mpirun -np 4 ex10p -m ../data/beam-hex.mesh -s 4 -rs 1 -dt 0.05 -vs 20
+//    mpirun -np 4 ex10p -m ../data/beam-quad-amr.mesh -s 23 -rs 2 -dt 3
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -172,7 +172,7 @@ int main(int argc, char *argv[])
    int ser_ref_levels = 2;
    int par_ref_levels = 0;
    int order = 2;
-   int ode_solver_type = 3;
+   int ode_solver_type = 23;
    real_t t_final = 300.0;
    real_t dt = 3.0;
    real_t visc = 1e-2;
@@ -192,11 +192,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
-                  "            11 - Forward Euler, 12 - RK2,\n\t"
-                  "            13 - RK3 SSP, 14 - RK4."
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  ODESolver::Types.c_str());
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -238,31 +234,7 @@ int main(int argc, char *argv[])
    // 4. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
-   ODESolver *ode_solver;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         if (myid == 0)
-         {
-            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         }
-         delete mesh;
-         return 3;
-   }
+   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
 
    // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -433,7 +405,6 @@ int main(int argc, char *argv[])
    }
 
    // 12. Free the used memory.
-   delete ode_solver;
    delete pmesh;
 
    return 0;

examples/ex16.cpp

@@ -5,10 +5,10 @@
 // Sample runs:  ex16
 //               ex16 -m ../data/inline-tri.mesh
 //               ex16 -m ../data/disc-nurbs.mesh -tf 2
-//               ex16 -s 1 -a 0.0 -k 1.0
-//               ex16 -s 2 -a 1.0 -k 0.0
-//               ex16 -s 3 -a 0.5 -k 0.5 -o 4
-//               ex16 -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               ex16 -s 21 -a 0.0 -k 1.0
+//               ex16 -s 22 -a 1.0 -k 0.0
+//               ex16 -s 23 -a 0.5 -k 0.5 -o 4
+//               ex16 -s 4 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               ex16 -m ../data/fichera-q2.mesh
 //               ex16 -m ../data/fichera-mixed.mesh
 //               ex16 -m ../data/escher.mesh
@@ -95,11 +95,13 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int ref_levels = 2;
    int order = 2;
-   int ode_solver_type = 3;
+
+   int ode_solver_type = 23;  // SDIRK33Solver
    real_t t_final = 0.5;
    real_t dt = 1.0e-2;
    real_t alpha = 1.0e-2;
    real_t kappa = 0.5;
+
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
@@ -115,8 +117,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
-                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
+                  ODESolver::Types.c_str());
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -149,28 +150,7 @@ int main(int argc, char *argv[])
    // 3. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
-   ODESolver *ode_solver;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
+   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -287,7 +267,6 @@ int main(int argc, char *argv[])
    }
 
    // 10. Free the used memory.
-   delete ode_solver;
    delete mesh;
 
    return 0;

examples/ex16p.cpp

@@ -5,10 +5,10 @@
 // Sample runs:  mpirun -np 4 ex16p
 //               mpirun -np 4 ex16p -m ../data/inline-tri.mesh
 //               mpirun -np 4 ex16p -m ../data/disc-nurbs.mesh -tf 2
-//               mpirun -np 4 ex16p -s 1 -a 0.0 -k 1.0
-//               mpirun -np 4 ex16p -s 2 -a 1.0 -k 0.0
-//               mpirun -np 8 ex16p -s 3 -a 0.5 -k 0.5 -o 4
-//               mpirun -np 4 ex16p -s 14 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               mpirun -np 4 ex16p -s 21 -a 0.0 -k 1.0
+//               mpirun -np 4 ex16p -s 22 -a 1.0 -k 0.0
+//               mpirun -np 8 ex16p -s 23 -a 0.5 -k 0.5 -o 4
+//               mpirun -np 4 ex16p -s 4 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               mpirun -np 16 ex16p -m ../data/fichera-q2.mesh
 //               mpirun -np 16 ex16p -m ../data/fichera-mixed.mesh
 //               mpirun -np 16 ex16p -m ../data/escher-p2.mesh
@@ -104,11 +104,13 @@ int main(int argc, char *argv[])
    int ser_ref_levels = 2;
    int par_ref_levels = 1;
    int order = 2;
-   int ode_solver_type = 3;
+
+   int ode_solver_type = 23;  // SDIRK33Solver
    real_t t_final = 0.5;
    real_t dt = 1.0e-2;
    real_t alpha = 1.0e-2;
    real_t kappa = 0.5;
+
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
@@ -127,8 +129,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
-                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
+                  ODESolver::Types.c_str());
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -169,28 +170,7 @@ int main(int argc, char *argv[])
    // 4. Define the ODE solver used for time integration. Several implicit
    //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
    //    explicit Runge-Kutta methods are available.
-   ODESolver *ode_solver;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
+   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
 
    // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -376,7 +356,6 @@ int main(int argc, char *argv[])
    }
 
    // 12. Free the used memory.
-   delete ode_solver;
    delete pmesh;
 
    return 0;

examples/ex18.cpp

@@ -90,8 +90,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
+                  ODESolver::ExplicitTypes.c_str());
    args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step. Positive number skips CFL timestep calculation.");
@@ -125,18 +124,7 @@ int main(int argc, char *argv[])
 
    // 3. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         return 3;
-   }
+   unique_ptr<ODESolver> ode_solver = ODESolver::SelectExplicit(ode_solver_type);
 
    // 4. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
@@ -304,8 +292,5 @@ int main(int argc, char *argv[])
    const real_t error = sol.ComputeLpError(2, u0);
    cout << "Solution error: " << error << endl;
 
-   // Free the used memory.
-   delete ode_solver;
-
    return 0;
 }

examples/ex18p.cpp

@@ -99,8 +99,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
+                  ODESolver::ExplicitTypes.c_str());
    args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step. Positive number skips CFL timestep calculation.");
@@ -148,18 +147,7 @@ int main(int argc, char *argv[])
 
    // 3. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         return 3;
-   }
+   unique_ptr<ODESolver> ode_solver = ODESolver::SelectExplicit(ode_solver_type);
 
    // 4. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
@@ -360,8 +348,5 @@ int main(int argc, char *argv[])
       cout << "Solution error: " << error << endl;
    }
 
-   // Free the used memory.
-   delete ode_solver;
-
    return 0;
 }

examples/ex23.cpp

@@ -201,9 +201,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: [0--10] - GeneralizedAlpha(0.1 * s),\n\t"
-                  "\t   11 - Average Acceleration, 12 - Linear Acceleration\n"
-                  "\t   13 - CentralDifference, 14 - FoxGoodwin");
+                  SecondOrderODESolver::Types.c_str());
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -238,32 +236,7 @@ int main(int argc, char *argv[])
 
    // 3. Define the ODE solver used for time integration. Several second order
    //    time integrators are available.
-   SecondOrderODESolver *ode_solver;
-   switch (ode_solver_type)
-   {
-      // Implicit methods
-      case 0: ode_solver = new GeneralizedAlpha2Solver(0.0); break;
-      case 1: ode_solver = new GeneralizedAlpha2Solver(0.1); break;
-      case 2: ode_solver = new GeneralizedAlpha2Solver(0.2); break;
-      case 3: ode_solver = new GeneralizedAlpha2Solver(0.3); break;
-      case 4: ode_solver = new GeneralizedAlpha2Solver(0.4); break;
-      case 5: ode_solver = new GeneralizedAlpha2Solver(0.5); break;
-      case 6: ode_solver = new GeneralizedAlpha2Solver(0.6); break;
-      case 7: ode_solver = new GeneralizedAlpha2Solver(0.7); break;
-      case 8: ode_solver = new GeneralizedAlpha2Solver(0.8); break;
-      case 9: ode_solver = new GeneralizedAlpha2Solver(0.9); break;
-      case 10: ode_solver = new GeneralizedAlpha2Solver(1.0); break;
-
-      case 11: ode_solver = new AverageAccelerationSolver(); break;
-      case 12: ode_solver = new LinearAccelerationSolver(); break;
-      case 13: ode_solver = new CentralDifferenceSolver(); break;
-      case 14: ode_solver = new FoxGoodwinSolver(); break;
-
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
+   SecondOrderODESolver *ode_solver= SecondOrderODESolver::Select(ode_solver_type);
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a

examples/ex38.cpp

@@ -3,18 +3,18 @@
 // Compile with: make ex38
 //
 // Sample runs:
-// (since all sample runs require LAPACK, the * symbol is used to exclude them
-// from the automatically generated internal MFEM tests).
+// (since all sample runs require LAPACK or ALGOIM, the * symbol is used to
+//  exclude them from the automatically generated internal MFEM tests).
 //              * ex38
 //              * ex38 -i volumetric1d
 //              * ex38 -i surface2d
-//              * ex38 -i surface2d -o 4 -r 5
+//              * ex38 -i surface2d -o 4 -r 5 -m 1
 //              * ex38 -i volumetric2d
-//              * ex38 -i volumetric2d -o 4 -r 5
+//              * ex38 -i volumetric2d -o 4 -r 5 -m 1
 //              * ex38 -i surface3d
-//              * ex38 -i surface3d -o 4 -r 5
+//              * ex38 -i surface3d -o 3 -r 4 -m 1
 //              * ex38 -i volumetric3d
-//              * ex38 -i volumetric3d -o 4 -r 5
+//              * ex38 -i volumetric3d -o 3 -r 4 -m 1
 //
 // Description: This example code demonstrates the use of MFEM to integrate
 //              functions over implicit interfaces and subdomains bounded by
@@ -71,7 +71,7 @@ real_t integrand(const Vector& X)
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return 1.;
+         return pow(X(0), 2.);
       case IntegrationType::Surface2D:
          return 3. * pow(X(0), 2.) - pow(X(1), 2.);
       case IntegrationType::Volumetric2D:
@@ -91,7 +91,7 @@ real_t Surface()
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return 1.;
+         return .3025;
       case IntegrationType::Surface2D:
          return 2. * M_PI;
       case IntegrationType::Volumetric2D:
@@ -111,7 +111,7 @@ real_t Volume()
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return .55;
+         return pow(.55, 3.) / 3.;
       case IntegrationType::Surface2D:
          return NAN;
       case IntegrationType::Volumetric2D:
@@ -125,7 +125,6 @@ real_t Volume()
    }
 }
 
-#ifdef MFEM_USE_LAPACK
 /**
  @brief Class for surface IntegrationRule
 
@@ -135,11 +134,14 @@ real_t Volume()
 class SIntegrationRule : public IntegrationRule
 {
 protected:
-   /// @brief Space Dimension of the IntegrationRule
+   /// method 0 is moments-based, 1 is Algoim.
+   int method, ir_order, ls_order;
+   Coefficient &level_set;
+   /// Space Dimension of the IntegrationRule
    int dim;
-   /// @brief Column-wise matrix of the quadtrature weights
+   /// Column-wise matrix of the quadtrature weights
    DenseMatrix Weights;
-   /// @brief Column-wise matrix of the transformation weights of the normal
+   /// Column-wise matrix of the transformation weights of the normal
    DenseMatrix SurfaceWeights;
 
 public:
@@ -153,15 +155,21 @@ public:
     @param [in] lsOrder Polynomial degree for approx of level-set function
     @param [in] mesh Pointer to the mesh that is used
    */
-   SIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+   SIntegrationRule(int method_, int Order,
+                    Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+      : method(method_), ir_order(Order), ls_order(lsOrder),
+        level_set(LvlSet), dim(mesh->Dimension())
    {
-      dim = mesh->Dimension();
+      // Nothing gets pre-computed for Algoim.
+      if (method == 1) { return; }
+
+#ifdef MFEM_USE_LAPACK
+      MomentFittingIntRules mf_ir(ir_order, level_set, ls_order);
 
       IsoparametricTransformation Tr;
-      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
       mesh->GetElementTransformation(0, &Tr);
       IntegrationRule ir;
-      MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+      mf_ir.GetSurfaceIntegrationRule(Tr, ir);
       if (dim >1)
       {
          Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
@@ -172,7 +180,7 @@ public:
       }
       SurfaceWeights.SetSize(ir.GetNPoints(), mesh->GetNE());
       Vector w;
-      MFIRs.GetSurfaceWeights(Tr, ir, w);
+      mf_ir.GetSurfaceWeights(Tr, ir, w);
       SurfaceWeights.SetCol(0, w);
       SetSize(ir.GetNPoints());
 
@@ -198,8 +206,8 @@ public:
       for (int elem = 1; elem < mesh->GetNE(); elem++)
       {
          mesh->GetElementTransformation(elem, &Tr);
-         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
-         MFIRs.GetSurfaceWeights(Tr, ir, w);
+         mf_ir.GetSurfaceIntegrationRule(Tr, ir);
+         mf_ir.GetSurfaceWeights(Tr, ir, w);
          SurfaceWeights.SetCol(elem, w);
 
          for (int ip = 0; ip < GetNPoints(); ip++)
@@ -215,48 +223,48 @@ public:
             }
          }
       }
+#else
+      MFEM_ABORT("Moment-fitting requires MFEM to be built with LAPACK!");
+#endif
    }
 
    /**
     @brief Set the weights for the given element and multiply them with the
     transformation of the interface
     */
-   void SetElementinclSurfaceWeight(int Element)
+   void SetElementAndSurfaceWeight(ElementTransformation &Tr)
    {
-      if (dim == 1)
+      if (method == 1)
       {
-         IntegrationPoint &intp = IntPoint(0);
-         intp.x = Weights(0, Element);
-         intp.weight = Weights(1, Element);
-         cout << intp.x << " " << Element << endl;
-      }
-      else
+#ifdef MFEM_USE_ALGOIM
+         AlgoimIntegrationRules a_ir(ir_order, level_set, ls_order);
+         a_ir.GetSurfaceIntegrationRule(Tr, *this);
+         Vector w;
+         a_ir.GetSurfaceWeights(Tr, *this, w);
          for (int ip = 0; ip < GetNPoints(); ip++)
          {
-            IntegrationPoint &intp = IntPoint(ip);
-            intp.weight = Weights(ip, Element) * SurfaceWeights(ip, Element);
+            IntPoint(ip).weight *= w(ip);
          }
-   }
+         return;
+#else
+         MFEM_ABORT("MFEM is not built with Algoim support!");
+#endif
+      }
 
-   /// @brief Set the weights for the given element
-   void SetElement(int Element)
-   {
       if (dim == 1)
       {
-         IntegrationPoint &intp = IntPoint(0);
-         intp.x = Weights(0, Element);
-         intp.weight = Weights(1, Element);
+         IntPoint(0).x = Weights(0, Tr.ElementNo);
+         IntPoint(0).weight = Weights(1, Tr.ElementNo);
       }
       else
+      {
          for (int ip = 0; ip < GetNPoints(); ip++)
          {
-            IntegrationPoint &intp = IntPoint(ip);
-            intp.weight = Weights(ip, Element);
+            IntPoint(ip).weight = Weights(ip, Tr.ElementNo) *
+                                  SurfaceWeights(ip, Tr.ElementNo);
          }
+      }
    }
-
-   /// @brief Destructor of SIntegrationRule
-   ~SIntegrationRule() {}
 };
 
 /**
@@ -268,9 +276,12 @@ public:
 class CIntegrationRule : public IntegrationRule
 {
 protected:
-   /// @brief Space Dimension of the IntegrationRule
+   /// method 0 is moments-based, 1 is Algoim.
+   int method, ir_order, ls_order;
+   Coefficient &level_set;
+   /// Space Dimension of the IntegrationRule
    int dim;
-   /// @brief Column-wise matrix of the quadtrature weights
+   /// Column-wise matrix of the quadtrature positions and weights.
    DenseMatrix Weights;
 
 public:
@@ -284,15 +295,21 @@ public:
     @param [in] lsOrder Polynomial degree for approx of level-set function
     @param [in] mesh Pointer to the mesh that is used
    */
-   CIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+   CIntegrationRule(int method_, int Order,
+                    Coefficient &LvlSet, int lsOrder, Mesh *mesh)
+      : method(method_), ir_order(Order), ls_order(lsOrder),
+        level_set(LvlSet), dim(mesh->Dimension())
    {
-      dim = mesh->Dimension();
+      // Nothing gets pre-computed for Algoim.
+      if (method == 1) { return; }
+
+#ifdef MFEM_USE_LAPACK
+      MomentFittingIntRules mf_ir(ir_order, level_set, ls_order);
 
       IsoparametricTransformation Tr;
-      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
       mesh->GetElementTransformation(0, &Tr);
       IntegrationRule ir;
-      MFIRs.GetVolumeIntegrationRule(Tr, ir);
+      mf_ir.GetVolumeIntegrationRule(Tr, ir);
       if (dim > 1)
       {
          Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
@@ -324,9 +341,9 @@ public:
       for (int elem = 1; elem < mesh->GetNE(); elem++)
       {
          mesh->GetElementTransformation(elem, &Tr);
-         MFIRs.GetVolumeIntegrationRule(Tr, ir);
+         mf_ir.GetVolumeIntegrationRule(Tr, ir);
 
-         for (int ip = 0; ip < GetNPoints(); ip++)
+         for (int ip = 0; ip < ir.GetNPoints(); ip++)
          {
             if (dim > 1)
             {
@@ -339,29 +356,39 @@ public:
             }
          }
       }
+#else
+      MFEM_ABORT("Moment-fitting requires MFEM to be built with LAPACK!");
+#endif
    }
 
    /// @brief Set the weights for the given element
-   void SetElement(int Element)
+   void SetElement(ElementTransformation &Tr)
    {
-      if (dim == 1)
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            IntegrationPoint &intp = IntPoint(ip);
-            intp.x = Weights(2 * ip, Element);
-            intp.weight = Weights(2 * ip + 1, Element);
-         }
-      else
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            IntegrationPoint &intp = IntPoint(ip);
-            intp.weight = Weights(ip, Element);
-         }
-   }
+      if (method == 1)
+      {
+#ifdef MFEM_USE_ALGOIM
+         AlgoimIntegrationRules a_ir(ir_order, level_set, ls_order);
+         a_ir.GetVolumeIntegrationRule(Tr, *this);
+         return;
+#else
+         MFEM_ABORT("MFEM is not built with Algoim support!");
+#endif
+      }
 
-   /// @brief Destructor of CIntegrationRule
-   ~CIntegrationRule() {}
+      for (int ip = 0; ip < GetNPoints(); ip++)
+      {
+         IntegrationPoint &intp = IntPoint(ip);
+         if (dim == 1)
+         {
+            intp.x = Weights(2 * ip, Tr.ElementNo);
+            intp.weight = Weights(2 * ip + 1, Tr.ElementNo);
+         }
+         else { intp.weight = Weights(ip, Tr.ElementNo); }
+      }
+   }
 };
+
+
 /**
  @brief Class for surface linearform integrator
 
@@ -418,7 +445,7 @@ public:
       elvect = 0.;
 
       // Update the surface integration rule for the current element
-      SIntRule->SetElementinclSurfaceWeight(Tr.ElementNo);
+      SIntRule->SetElementAndSurfaceWeight(Tr);
 
       for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
       {
@@ -428,6 +455,8 @@ public:
          add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
       }
    }
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
 /**
@@ -486,7 +515,7 @@ public:
       elvect = 0.;
 
       // Update the subdomain integration rule
-      CIntRule->SetElement(Tr.ElementNo);
+      CIntRule->SetElement(Tr);
 
       for (int ip = 0; ip < CIntRule->GetNPoints(); ip++)
       {
@@ -497,18 +526,17 @@ public:
          add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
       }
    }
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
 };
-#endif // MFEM_USE_LAPACK
 
 int main(int argc, char *argv[])
 {
-#ifndef MFEM_USE_LAPACK
-   cout << "MFEM must be built with LAPACK for this example." << endl;
-   return MFEM_SKIP_RETURN_VALUE;
-#else
+#if defined(MFEM_USE_LAPACK) || defined(MFEM_USE_ALGOIM)
    // 1. Parse he command-line options.
    int ref_levels = 3;
    int order = 2;
+   int method = 0;
    const char *inttype = "surface2d";
    bool visualization = true;
    itype = IntegrationType::Surface2D;
@@ -516,6 +544,8 @@ int main(int argc, char *argv[])
    OptionsParser args(argc, argv);
    args.AddOption(&order, "-o", "--order", "Order of quadrature rule");
    args.AddOption(&ref_levels, "-r", "--refine", "Number of meh refinements");
+   args.AddOption(&method, "-m", "--method",
+                  "Cut integration method: 0 for moments-based, 1 for Algoim.");
    args.AddOption(&inttype, "-i", "--integrationtype",
                   "IntegrationType to demonstrate");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -550,7 +580,7 @@ int main(int argc, char *argv[])
    }
 
    // 2. Construct and refine the mesh.
-   Mesh *mesh;
+   Mesh *mesh = nullptr;
    if (itype == IntegrationType::Volumetric1D)
    {
       mesh = new Mesh("../data/inline-segment.mesh");
@@ -598,13 +628,14 @@ int main(int argc, char *argv[])
    // 5. Define the necessary Integration rules on element 0.
    IsoparametricTransformation Tr;
    mesh->GetElementTransformation(0, &Tr);
-   SIntegrationRule* sir = new SIntegrationRule(order, levelset, 2, mesh);
+   SIntegrationRule* sir = new SIntegrationRule(method, order,
+                                                levelset, 2, mesh);
    CIntegrationRule* cir = NULL;
    if (itype == IntegrationType::Volumetric1D
        || itype == IntegrationType::Volumetric2D
        || itype == IntegrationType::Volumetric3D)
    {
-      cir = new CIntegrationRule(order, levelset, 2, mesh);
+      cir = new CIntegrationRule(method, order, levelset, 2, mesh);
    }
 
    // 6. Define and assemble the linear forms on the finite element space.
@@ -647,11 +678,11 @@ int main(int argc, char *argv[])
       cout << "Number of div free basis functions: " << nbasis << endl;
       cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
    }
-   cout << scientific << setprecision(2);
+   cout << scientific << setprecision(10);
    cout << "============================================" << endl;
    cout << "Computed value of surface integral: " << surface.Sum() << endl;
    cout << "True value of surface integral:     " << Surface() << endl;
-   cout << "Absolute Error (Surface):            ";
+   cout << "Absolute Error (Surface):           ";
    cout << abs(surface.Sum() - Surface()) << endl;
    cout << "Relative Error (Surface):           ";
    cout << abs(surface.Sum() - Surface()) / Surface() << endl;
@@ -662,7 +693,7 @@ int main(int argc, char *argv[])
       cout << "--------------------------------------------" << endl;
       cout << "Computed value of volume integral:  " << volume.Sum() << endl;
       cout << "True value of volume integral:      " << Volume() << endl;
-      cout << "Absolute Error (Volume):             ";
+      cout << "Absolute Error (Volume):            ";
       cout << abs(volume.Sum() - Volume()) << endl;
       cout << "Relative Error (Volume):            ";
       cout << abs(volume.Sum() - Volume()) / Volume() << endl;
@@ -691,5 +722,8 @@ int main(int argc, char *argv[])
    delete fespace;
    delete mesh;
    return EXIT_SUCCESS;
-#endif //MFEM_USE_LAPACK
+#else
+   cout << "MFEM must be built with LAPACK or ALGOIM for this example." << endl;
+   return MFEM_SKIP_RETURN_VALUE;
+#endif // MFEM_USE_LAPACK
 }

examples/ex4.cpp

@@ -65,6 +65,7 @@ int main(int argc, char *argv[])
    bool static_cond = false;
    bool hybridization = false;
    bool pa = false;
+   bool ea = false;
    const char *device_config = "cpu";
    bool visualization = 1;
 
@@ -83,18 +84,14 @@ int main(int argc, char *argv[])
                   "--no-hybridization", "Enable hybridization.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
+                  "--no-element-assembly", "Enable Element Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
+   args.ParseCheck();
    kappa = freq * M_PI;
 
    // 2. Enable hardware devices such as GPUs, and programming models such as
@@ -166,6 +163,7 @@ int main(int argc, char *argv[])
    Coefficient *beta  = new ConstantCoefficient(1.0);
    BilinearForm *a = new BilinearForm(fespace);
    if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (ea) { a->SetAssemblyLevel(AssemblyLevel::ELEMENT); }
    a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
    a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
 

examples/ex4p.cpp

@@ -71,6 +71,7 @@ int main(int argc, char *argv[])
    bool static_cond = false;
    bool hybridization = false;
    bool pa = false;
+   bool ea = false;
    const char *device_config = "cpu";
    bool visualization = 1;
 
@@ -89,24 +90,14 @@ int main(int argc, char *argv[])
                   "--no-hybridization", "Enable hybridization.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
+                  "--no-element-assembly", "Enable Element Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
+   args.ParseCheck();
    kappa = freq * M_PI;
 
    // 3. Enable hardware devices such as GPUs, and programming models such as
@@ -194,6 +185,7 @@ int main(int argc, char *argv[])
    Coefficient *beta  = new ConstantCoefficient(1.0);
    ParBilinearForm *a = new ParBilinearForm(fespace);
    if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (ea) { a->SetAssemblyLevel(AssemblyLevel::ELEMENT); }
    a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
    a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
 

examples/ex9.cpp

@@ -9,7 +9,7 @@
 //    ex9 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
-//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 13 -tf 9
+//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 23 -tf 9
 //    ex9 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
@@ -182,12 +182,7 @@ int main(int argc, char *argv[])
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
-                  "            11 - Backward Euler,\n\t"
-                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  ODESolver::Types.c_str());
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -224,28 +219,7 @@ int main(int argc, char *argv[])
 
    // 3. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      // Explicit methods
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      // Implicit (L-stable) methods
-      case 11: ode_solver = new BackwardEulerSolver; break;
-      case 12: ode_solver = new SDIRK23Solver(2); break;
-      case 13: ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         return 3;
-   }
+   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
@@ -440,7 +414,6 @@ int main(int argc, char *argv[])
    }
 
    // 10. Free the used memory.
-   delete ode_solver;
    delete pd;
    delete dc;
 

examples/ex9p.cpp

@@ -9,7 +9,7 @@
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-hexagon.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9
-//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 13 -tf 9
+//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 23 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-mixed.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
@@ -285,12 +285,7 @@ int main(int argc, char *argv[])
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
-                  "            11 - Backward Euler,\n\t"
-                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  ODESolver::Types.c_str());
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -338,31 +333,7 @@ int main(int argc, char *argv[])
 
    // 4. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      // Explicit methods
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      // Implicit (L-stable) methods
-      case 11: ode_solver = new BackwardEulerSolver; break;
-      case 12: ode_solver = new SDIRK23Solver(2); break;
-      case 13: ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         if (Mpi::Root())
-         {
-            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         }
-         delete mesh;
-         return 3;
-   }
+   unique_ptr<ODESolver> ode_solver = ODESolver::Select(ode_solver_type);
 
    // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
@@ -642,7 +613,6 @@ int main(int argc, char *argv[])
    delete m;
    delete fes;
    delete pmesh;
-   delete ode_solver;
    delete pd;
 #ifdef MFEM_USE_ADIOS2
    if (adios2)

examples/sundials/ex10.cpp

@@ -486,7 +486,11 @@ int main(int argc, char *argv[])
          arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
          arkode->Init(*oper);
          arkode->SetSStolerances(reltol, abstol);
+#if MFEM_SUNDIALS_VERSION < 70100
          ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
+#else
+         ARKodeSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
+#endif
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 15)
          {

examples/sundials/ex10p.cpp

@@ -541,7 +541,11 @@ int main(int argc, char *argv[])
          arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
          arkode->Init(*oper);
          arkode->SetSStolerances(reltol, abstol);
+#if MFEM_SUNDIALS_VERSION < 70100
          ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
+#else
+         ARKodeSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
+#endif
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 15)
          {

examples/sundials/ex16.cpp

@@ -447,7 +447,7 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &fes,
                                        const Vector &u,
                                        const Type &ode_expression_type)
    : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
-     fespace(fes), alpha(alpha), kappa(kappa), M(&fespace), z(height)
+     fespace(fes), M(&fespace), alpha(alpha), kappa(kappa), z(height)
 {
    // specify a relative tolerance for all solves with MFEM integrators
    const real_t rel_tol = 1e-8;
@@ -522,7 +522,7 @@ int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
    T = std::unique_ptr<SparseMatrix>(Add(1.0, Mmat, gam, Kmat));
    T_solver.SetOperator(*T);
    *jcur = SUNTRUE; // this should eventually only be set true if K(u) is used
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 
 int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
@@ -544,7 +544,7 @@ int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
    }
    if (T_solver.GetConverged())
    {
-      return SUNLS_SUCCESS;
+      return SUN_SUCCESS;
    }
    else
    {
@@ -555,7 +555,7 @@ int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
 int ConductionOperator::SUNMassSetup()
 {
    // Do nothing b/c mass solver was setup in constructor.
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 
 int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
@@ -565,7 +565,7 @@ int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
    M_solver.Mult(b, x);
    if (M_solver.GetConverged())
    {
-      return SUNLS_SUCCESS;
+      return SUN_SUCCESS;
    }
    else
    {
@@ -577,6 +577,6 @@ int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
 {
    // Compute M x.
    Mmat.Mult(x, v);
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 

examples/sundials/ex16p.cpp

@@ -499,7 +499,7 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &fes,
                                        const Vector &u,
                                        const Type &ode_expression_type)
    : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
-     fespace(fes), alpha(alpha), kappa(kappa), M(&fespace),
+     fespace(fes), M(&fespace), alpha(alpha), kappa(kappa),
      M_solver(fes.GetComm()), T_solver(fes.GetComm()), z(height)
 {
    // specify a relative tolerance for all solves with MFEM integrators
@@ -576,7 +576,7 @@ int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
    T = std::unique_ptr<HypreParMatrix>(Add(1.0, Mmat, gam, Kmat));
    T_solver.SetOperator(*T);
    *jcur = SUNTRUE; // this should eventually only be set true if K(u) is used
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 
 int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
@@ -598,7 +598,7 @@ int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
    }
    if (T_solver.GetConverged())
    {
-      return SUNLS_SUCCESS;
+      return SUN_SUCCESS;
    }
    else
    {
@@ -609,7 +609,7 @@ int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
 int ConductionOperator::SUNMassSetup()
 {
    // Do nothing b/c mass solver was setup in constructor.
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 
 int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
@@ -619,7 +619,7 @@ int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
    M_solver.Mult(b, x);
    if (M_solver.GetConverged())
    {
-      return SUNLS_SUCCESS;
+      return SUN_SUCCESS;
    }
    else
    {
@@ -631,5 +631,5 @@ int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
 {
    // Compute M x.
    Mmat.Mult(x, v);
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }

fem/CMakeLists.txt

@@ -35,6 +35,7 @@ set(SRCS
   integ/bilininteg_mass_ea.cpp
   integ/bilininteg_mixedcurl_pa.cpp
   integ/bilininteg_mixedvecgrad_pa.cpp
+  integ/bilininteg_trace_jump_ea.cpp
   integ/bilininteg_transpose_ea.cpp
   integ/bilininteg_vecdiffusion_mf.cpp
   integ/bilininteg_vecdiffusion_pa.cpp
@@ -46,6 +47,7 @@ set(SRCS
   integ/bilininteg_diffusion_kernels.cpp
   integ/bilininteg_elasticity_kernels.cpp
   integ/bilininteg_hcurl_kernels.cpp
+  integ/bilininteg_hdiv_ea.cpp
   integ/bilininteg_hdiv_kernels.cpp
   integ/bilininteg_hcurlhdiv_kernels.cpp
   integ/bilininteg_mass_kernels.cpp
@@ -80,6 +82,7 @@ set(SRCS
   geom.cpp
   gridfunc.cpp
   hybridization.cpp
+  hybridization_ext.cpp
   intrules.cpp
   intrules_cut.cpp
   ceed/interface/basis.cpp
@@ -188,6 +191,7 @@ set(HDRS
   geom.hpp
   gridfunc.hpp
   hybridization.hpp
+  hybridization_ext.hpp
   intrules.hpp
   intrules_cut.hpp
   kernel_dispatch.hpp

fem/bilinearform.cpp

@@ -71,15 +71,11 @@ BilinearForm::BilinearForm(FiniteElementSpace * f)
    sequence = f->GetSequence();
    mat = mat_e = NULL;
    extern_bfs = 0;
-   element_matrices = NULL;
-   static_cond = NULL;
-   hybridization = NULL;
    precompute_sparsity = 0;
    diag_policy = DIAG_KEEP;
 
    assembly = AssemblyLevel::LEGACY;
    batch = 1;
-   ext = NULL;
 }
 
 BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
@@ -89,15 +85,11 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
    sequence = f->GetSequence();
    mat_e = NULL;
    extern_bfs = 1;
-   element_matrices = NULL;
-   static_cond = NULL;
-   hybridization = NULL;
    precompute_sparsity = ps;
    diag_policy = DIAG_KEEP;
 
    assembly = AssemblyLevel::LEGACY;
    batch = 1;
-   ext = NULL;
 
    // Copy the pointers to the integrators
    domain_integs = bf->domain_integs;
@@ -127,16 +119,16 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
          break;
       case AssemblyLevel::FULL:
          SetDiagonalPolicy( DIAG_ONE ); // Only diagonal policy supported on device
-         ext = new FABilinearFormExtension(this);
+         ext.reset(new FABilinearFormExtension(this));
          break;
       case AssemblyLevel::ELEMENT:
-         ext = new EABilinearFormExtension(this);
+         ext.reset(new EABilinearFormExtension(this));
          break;
       case AssemblyLevel::PARTIAL:
-         ext = new PABilinearFormExtension(this);
+         ext.reset(new PABilinearFormExtension(this));
          break;
       case AssemblyLevel::NONE:
-         ext = new MFBilinearFormExtension(this);
+         ext.reset(new MFBilinearFormExtension(this));
          break;
       default:
          MFEM_ABORT("BilinearForm: unknown assembly level");
@@ -145,14 +137,13 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
 
 void BilinearForm::EnableStaticCondensation()
 {
-   delete static_cond;
    if (assembly != AssemblyLevel::LEGACY)
    {
-      static_cond = NULL;
+      static_cond.reset();
       MFEM_WARNING("Static condensation not supported for this assembly level");
       return;
    }
-   static_cond = new StaticCondensation(fes);
+   static_cond.reset(new StaticCondensation(fes));
    if (static_cond->ReducesTrueVSize())
    {
       bool symmetric = false;      // TODO
@@ -161,8 +152,7 @@ void BilinearForm::EnableStaticCondensation()
    }
    else
    {
-      delete static_cond;
-      static_cond = NULL;
+      static_cond.reset();
    }
 }
 
@@ -170,15 +160,18 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                        BilinearFormIntegrator *constr_integ,
                                        const Array<int> &ess_tdof_list)
 {
-   delete hybridization;
-   if (assembly != AssemblyLevel::LEGACY)
+   if (assembly != AssemblyLevel::LEGACY && assembly != AssemblyLevel::ELEMENT)
    {
       delete constr_integ;
-      hybridization = NULL;
+      hybridization.reset();
       MFEM_WARNING("Hybridization not supported for this assembly level");
       return;
    }
-   hybridization = new Hybridization(fes, constr_space);
+   hybridization.reset(new Hybridization(fes, constr_space));
+   if (assembly == AssemblyLevel::ELEMENT)
+   {
+      hybridization->EnableDeviceExecution();
+   }
    hybridization->SetConstraintIntegrator(constr_integ);
    hybridization->Init(ess_tdof_list);
 }
@@ -231,8 +224,8 @@ void BilinearForm::Finalize (int skip_zeros)
       if (!static_cond) { mat->Finalize(skip_zeros); }
       if (mat_e) { mat_e->Finalize(skip_zeros); }
       if (static_cond) { static_cond->Finalize(); }
-      if (hybridization) { hybridization->Finalize(); }
    }
+   if (hybridization) { hybridization->Finalize(); }
 }
 
 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi)
@@ -465,6 +458,10 @@ void BilinearForm::Assemble(int skip_zeros)
    if (ext)
    {
       ext->Assemble();
+      if (hybridization)
+      {
+         hybridization->AssembleElementMatrices(GetElementMatrices());
+      }
       return;
    }
 
@@ -834,7 +831,19 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
 {
    if (ext)
    {
-      ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+      if (hybridization)
+      {
+         FormSystemMatrix(ess_tdof_list, A);
+         ConstrainedOperator A_constrained(this, ess_tdof_list);
+         A_constrained.EliminateRHS(x, b);
+         hybridization->ReduceRHS(b, B);
+         X.SetSize(B.Size());
+         X = 0.0;
+      }
+      else
+      {
+         ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+      }
       return;
    }
    const SparseMatrix *P = fes->GetConformingProlongation();
@@ -902,7 +911,16 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 {
    if (ext)
    {
-      ext->FormSystemMatrix(ess_tdof_list, A);
+      if (hybridization)
+      {
+         const int remove_zeros = 0;
+         Finalize(remove_zeros);
+         A.Reset(&hybridization->GetMatrix(), false);
+      }
+      else
+      {
+         ext->FormSystemMatrix(ess_tdof_list, A);
+      }
       return;
    }
 
@@ -943,7 +961,7 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 void BilinearForm::RecoverFEMSolution(const Vector &X,
                                       const Vector &b, Vector &x)
 {
-   if (ext)
+   if (ext && !hybridization)
    {
       ext->RecoverFEMSolution(X, b, x);
       return;
@@ -1000,16 +1018,26 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,
 
 void BilinearForm::ComputeElementMatrices()
 {
-   if (element_matrices || domain_integs.Size() == 0 || fes->GetNE() == 0)
+   if (element_matrices) { return; }
+
+   if (auto *ea_ext = dynamic_cast<EABilinearFormExtension*>(ext.get()))
    {
+      element_matrices.reset(new DenseTensor);
+      ea_ext->GetElementMatrices(*element_matrices, ElementDofOrdering::NATIVE, true);
+      return;
+   }
+
+   if (domain_integs.Size() == 0 || fes->GetNE() == 0)
+   {
+      element_matrices.reset(new DenseTensor);
       return;
    }
 
    int num_elements = fes->GetNE();
    int num_dofs_per_el = fes->GetFE(0)->GetDof() * fes->GetVDim();
 
-   element_matrices = new DenseTensor(num_dofs_per_el, num_dofs_per_el,
-                                      num_elements);
+   element_matrices.reset(new DenseTensor(num_dofs_per_el, num_dofs_per_el,
+                                          num_elements));
 
    DenseMatrix tmp;
    IsoparametricTransformation eltrans;
@@ -1040,6 +1068,12 @@ void BilinearForm::ComputeElementMatrices()
    }
 }
 
+const DenseTensor &BilinearForm::GetElementMatrices()
+{
+   ComputeElementMatrices(); // Won't recompute if element_matrices exists
+   return *element_matrices;
+}
+
 void BilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
                                         const Vector &sol, Vector &rhs,
                                         DiagonalPolicy dpolicy)
@@ -1227,15 +1261,13 @@ void BilinearForm::Update(FiniteElementSpace *nfes)
    delete mat_e;
    mat_e = NULL;
    FreeElementMatrices();
-   delete static_cond;
-   static_cond = NULL;
+   static_cond.reset();
 
    if (full_update)
    {
       delete mat;
       mat = NULL;
-      delete hybridization;
-      hybridization = NULL;
+      hybridization.reset();
       sequence = fes->GetSequence();
    }
    else
@@ -1258,9 +1290,6 @@ BilinearForm::~BilinearForm()
 {
    delete mat_e;
    delete mat;
-   delete element_matrices;
-   delete static_cond;
-   delete hybridization;
 
    if (!extern_bfs)
    {
@@ -1272,8 +1301,6 @@ BilinearForm::~BilinearForm()
       for (k=0; k < boundary_face_integs.Size(); k++)
       { delete boundary_face_integs[k]; }
    }
-
-   delete ext;
 }
 
 
@@ -1300,7 +1327,6 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
    mat = NULL;
    mat_e = NULL;
    extern_bfs = 1;
-   ext = NULL;
 
    // Copy the pointers to the integrators
    domain_integs = mbf->domain_integs;
@@ -1330,22 +1356,22 @@ void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
       case AssemblyLevel::LEGACY:
          break;
       case AssemblyLevel::FULL:
-         // ext = new FAMixedBilinearFormExtension(this);
+         // ext.reset(new FAMixedBilinearFormExtension(this));
          // Use the original BilinearForm implementation for now
          break;
       case AssemblyLevel::ELEMENT:
-         mfem_error("Element assembly not supported yet... stay tuned!");
-         // ext = new EAMixedBilinearFormExtension(this);
+         MFEM_ABORT("Element assembly not supported yet... stay tuned!");
+         // ext.reset(new EAMixedBilinearFormExtension(this));
          break;
       case AssemblyLevel::PARTIAL:
-         ext = new PAMixedBilinearFormExtension(this);
+         ext.reset(new PAMixedBilinearFormExtension(this));
          break;
       case AssemblyLevel::NONE:
-         mfem_error("Matrix-free action not supported yet... stay tuned!");
-         // ext = new MFMixedBilinearFormExtension(this);
+         MFEM_ABORT("Matrix-free action not supported yet... stay tuned!");
+         // ext.reset(new MFMixedBilinearFormExtension(this));
          break;
       default:
-         mfem_error("Unknown assembly level");
+         MFEM_ABORT("Unknown assembly level");
    }
 }
 
@@ -2342,7 +2368,6 @@ MixedBilinearForm::~MixedBilinearForm()
       for (i = 0; i < boundary_trace_face_integs.Size(); i++)
       { delete boundary_trace_face_integs[i]; }
    }
-   delete ext;
 }
 
 void DiscreteLinearOperator::SetAssemblyLevel(AssemblyLevel assembly_level)
@@ -2359,16 +2384,16 @@ void DiscreteLinearOperator::SetAssemblyLevel(AssemblyLevel assembly_level)
          // Use the original implementation for now
          break;
       case AssemblyLevel::ELEMENT:
-         mfem_error("Element assembly not supported yet... stay tuned!");
+         MFEM_ABORT("Element assembly not supported yet... stay tuned!");
          break;
       case AssemblyLevel::PARTIAL:
-         ext = new PADiscreteLinearOperatorExtension(this);
+         ext.reset(new PADiscreteLinearOperatorExtension(this));
          break;
       case AssemblyLevel::NONE:
-         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         MFEM_ABORT("Matrix-free action not supported yet... stay tuned!");
          break;
       default:
-         mfem_error("Unknown assembly level");
+         MFEM_ABORT("Unknown assembly level");
    }
 }
 

fem/bilinearform.hpp

@@ -83,7 +83,7 @@ protected:
    /** @brief Extension for supporting Full Assembly (FA),
        Element Assembly (EA),Partial Assembly (PA),
        or Matrix Free assembly (MF). */
-   BilinearFormExtension *ext;
+   std::unique_ptr<BilinearFormExtension> ext;
 
    /** Indicates if the sparse matrix is sorted after assembly when using
        Full Assembly (FA). */
@@ -122,10 +122,10 @@ protected:
    mutable DenseMatrix elemmat;
    mutable Array<int>  vdofs;
 
-   DenseTensor *element_matrices; ///< Owned.
+   std::unique_ptr<DenseTensor> element_matrices;
 
-   StaticCondensation *static_cond; ///< Owned.
-   Hybridization *hybridization; ///< Owned.
+   std::unique_ptr<StaticCondensation> static_cond;
+   std::unique_ptr<Hybridization> hybridization;
 
    /** @brief This data member allows one to specify what should be done to the
        diagonal matrix entries and corresponding RHS values upon elimination of
@@ -148,13 +148,11 @@ protected:
    BilinearForm() : Matrix (0)
    {
       fes = NULL; sequence = -1;
-      mat = mat_e = NULL; extern_bfs = 0; element_matrices = NULL;
-      static_cond = NULL; hybridization = NULL;
+      mat = mat_e = NULL; extern_bfs = 0;
       precompute_sparsity = 0;
       diag_policy = DIAG_KEEP;
       assembly = AssemblyLevel::LEGACY;
       batch = 1;
-      ext = NULL;
    }
 
 private:
@@ -214,7 +212,7 @@ public:
    /// Returns the assembly level
    AssemblyLevel GetAssemblyLevel() const { return assembly; }
 
-   Hybridization *GetHybridization() const { return hybridization; }
+   Hybridization *GetHybridization() const { return hybridization.get(); }
 
    /** @brief Enable the use of static condensation. For details see the
        description for class StaticCondensation in fem/staticcond.hpp This
@@ -224,7 +222,7 @@ public:
 
    /** @brief Check if static condensation was actually enabled by a previous
        call to EnableStaticCondensation(). */
-   bool StaticCondensationIsEnabled() const { return static_cond; }
+   bool StaticCondensationIsEnabled() const { return static_cond.get() != nullptr; }
 
    /// Return the trace FE space associated with static condensation.
    FiniteElementSpace *SCFESpace() const
@@ -569,12 +567,20 @@ public:
    void RecoverFEMSolution(const Vector &X, const Vector &b,
                            Vector &x) override;
 
-   /// Compute and store internally all element matrices.
+   /// @brief Compute and store internally all element matrices.
+   ///
+   /// If AssemblyLevel::ELEMENT is selected with SetAssemblyLeve(), this will
+   /// use effecient (device-accelerated) assembly of the element matrices.
    void ComputeElementMatrices();
 
    /// Free the memory used by the element matrices.
-   void FreeElementMatrices()
-   { delete element_matrices; element_matrices = NULL; }
+   void FreeElementMatrices() { element_matrices.reset(); }
+
+   /// @brief Return a DenseTensor containing the assembled element matrices.
+   ///
+   /// If AssemblyLevel::ELEMENT is selected with SetAssemblyLeve(), this will
+   /// use effecient (device-accelerated) assembly of the element matrices.
+   const DenseTensor &GetElementMatrices();
 
    /// Compute the element matrix of the given element
    /** The element matrix is computed by calling the domain integrators
@@ -753,7 +759,7 @@ protected:
 
    /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
        Partial Assembly (PA), or Matrix Free assembly (MF). */
-   MixedBilinearFormExtension *ext;
+   std::unique_ptr<MixedBilinearFormExtension> ext;
 
    /** @brief Indicates the BilinearFormIntegrator%s stored in
        MixedBilinearForm#domain_integs, MixedBilinearForm#boundary_integs,

fem/bilinearform_ext.cpp

@@ -16,6 +16,7 @@
 #include "bilinearform.hpp"
 #include "pbilinearform.hpp"
 #include "pgridfunc.hpp"
+#include "fe/face_map_utils.hpp"
 #include "ceed/interface/util.hpp"
 
 namespace mfem
@@ -864,54 +865,139 @@ void EABilinearFormExtension::Assemble()
    ne = trial_fes->GetMesh()->GetNE();
    elemDofs = trial_fes->GetFE(0)->GetDof();
 
-   ea_data.SetSize(ne*elemDofs*elemDofs, Device::GetMemoryType());
-   ea_data.UseDevice(true);
+   Vector ea_data_tmp;
 
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   if ( integratorCount == 0 )
+   auto add_with_markers = [&](const Vector &ea_1, Vector &ea_2, const int ne_,
+                               const Array<int> &markers, const Array<int> &attrs,
+                               const bool add)
    {
-      ea_data = 0.0;
-   }
-   for (int i = 0; i < integratorCount; ++i)
+      if (ne_ == 0) { return; }
+      const int sz = ea_1.Size() / ne_;
+      const int *d_m = markers.Read();
+      const int *d_a = attrs.Read();
+      const auto d_ea_1 = Reshape(ea_1.Read(), sz, ne_);
+      auto d_ea_2 = Reshape(add ? ea_2.ReadWrite() : ea_2.Write(), sz, ne_);
+
+      mfem::forall(sz*ne_, [=] MFEM_HOST_DEVICE (int idx)
+      {
+         const int i = idx % sz;
+         const int e = idx / sz;
+         const real_t val = d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0.0;
+         if (add)
+         {
+            d_ea_2(i, e) += val;
+         }
+         else
+         {
+            d_ea_2(i, e) = val;
+         }
+      });
+   };
+
    {
-      integrators[i]->AssembleEA(*a->FESpace(), ea_data, i);
+      ea_data.SetSize(ne*elemDofs*elemDofs);
+      ea_data.UseDevice(true);
+      Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+      Array<Array<int>*> &markers_array = *a->GetDBFI_Marker();
+
+      if (integrators.Size() == 0) { ea_data = 0.0; }
+
+      for (int i = 0; i < integrators.Size(); ++i)
+      {
+         const bool add = (i > 0);
+         const Array<int> *markers = markers_array[i];
+         if (markers == nullptr)
+         {
+            integrators[i]->AssembleEA(*a->FESpace(), ea_data, add);
+         }
+         else
+         {
+            ea_data_tmp.SetSize(ea_data.Size());
+            integrators[i]->AssembleEA(*a->FESpace(), ea_data_tmp, false);
+            add_with_markers(ea_data_tmp, ea_data, ne, *markers,
+                             elem_attributes, add);
+         }
+      }
    }
 
    faceDofs = trial_fes ->
               GetTraceElement(0, trial_fes->GetMesh()->GetFaceGeometry(0)) ->
               GetDof();
 
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Element assembly does not support AddBoundaryIntegrator yet.");
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int intFaceIntegratorCount = intFaceIntegrators.Size();
-   if (intFaceIntegratorCount>0)
    {
-      nf_int = trial_fes->GetNFbyType(FaceType::Interior);
-      ea_data_int.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
-      ea_data_ext.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
-   }
-   for (int i = 0; i < intFaceIntegratorCount; ++i)
-   {
-      intFaceIntegrators[i]->AssembleEAInteriorFaces(*a->FESpace(),
-                                                     ea_data_int,
-                                                     ea_data_ext,
-                                                     i);
+      Array<BilinearFormIntegrator*> &bdr_integs = *a->GetBBFI();
+      Array<Array<int>*> &markers_array = *a->GetBBFI_Marker();
+      const int n_bdr_integs = bdr_integs.Size();
+      if (n_bdr_integs > 0)
+      {
+         nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
+         ea_data_bdr.SetSize(nf_bdr*faceDofs*faceDofs);
+      }
+      for (int i = 0; i < n_bdr_integs; ++i)
+      {
+         const bool add = (i > 0);
+         const Array<int> *markers = markers_array[i];
+         if (markers == nullptr)
+         {
+            bdr_integs[i]->AssembleEABoundary(*a->FESpace(), ea_data_bdr, add);
+         }
+         else
+         {
+            ea_data_tmp.SetSize(ea_data_bdr.Size());
+            bdr_integs[i]->AssembleEABoundary(*a->FESpace(), ea_data_tmp, add);
+            add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers,
+                             bdr_attributes, add);
+         }
+      }
    }
 
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int boundFaceIntegratorCount = bdrFaceIntegrators.Size();
-   if (boundFaceIntegratorCount>0)
    {
-      nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
-      ea_data_bdr.SetSize(nf_bdr*faceDofs*faceDofs, Device::GetMemoryType());
-      ea_data_bdr = 0.0;
+      Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
+      const int intFaceIntegratorCount = intFaceIntegrators.Size();
+      if (intFaceIntegratorCount>0)
+      {
+         nf_int = trial_fes->GetNFbyType(FaceType::Interior);
+         ea_data_int.SetSize(2*nf_int*faceDofs*faceDofs);
+         ea_data_ext.SetSize(2*nf_int*faceDofs*faceDofs);
+      }
+      for (int i = 0; i < intFaceIntegratorCount; ++i)
+      {
+         const bool add = (i > 0);
+         intFaceIntegrators[i]->AssembleEAInteriorFaces(*a->FESpace(),
+                                                        ea_data_int,
+                                                        ea_data_ext,
+                                                        add);
+      }
    }
-   for (int i = 0; i < boundFaceIntegratorCount; ++i)
+
    {
-      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr,i);
+      Array<BilinearFormIntegrator*> &bdr_face_integs = *a->GetBFBFI();
+      Array<Array<int>*> &markers_array = *a->GetBFBFI_Marker();
+      const int n_bdr_face_integs = bdr_face_integs.Size();
+      if (n_bdr_face_integs > 0)
+      {
+         nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
+         ea_data_bdr.SetSize(nf_bdr*faceDofs*faceDofs);
+      }
+      for (int i = 0; i < n_bdr_face_integs; ++i)
+      {
+         const bool add = (i > 0);
+         const Array<int> *markers = markers_array[i];
+         if (markers == nullptr)
+         {
+            bdr_face_integs[i]->AssembleEABoundaryFaces(
+               *a->FESpace(), ea_data_bdr, add);
+         }
+         else
+         {
+            ea_data_tmp.SetSize(ea_data_bdr.Size());
+            bdr_face_integs[i]->AssembleEABoundaryFaces(*a->FESpace(),
+                                                        ea_data_tmp,
+                                                        add);
+            add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers,
+                             bdr_attributes, add);
+         }
+      }
    }
 
    if (factorize_face_terms && int_face_restrict_lex)
@@ -1023,34 +1109,29 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
    }
 
    // Treatment of boundary faces
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
+   if (!factorize_face_terms && bdr_face_restrict_lex && ea_data_bdr.Size() > 0)
    {
       // Apply the Boundary Face Restriction
       bdr_face_restrict_lex->Mult(x, bdr_face_X);
-      if (bdr_face_X.Size()>0)
+      bdr_face_Y = 0.0;
+      // Apply the boundary face matrices
+      const int NDOFS = faceDofs;
+      auto X = Reshape(bdr_face_X.Read(), NDOFS, nf_bdr);
+      auto Y = Reshape(bdr_face_Y.ReadWrite(), NDOFS, nf_bdr);
+      auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
+      mfem::forall(nf_bdr*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
       {
-         bdr_face_Y = 0.0;
-         // Apply the boundary face matrices
-         const int NDOFS = faceDofs;
-         auto X = Reshape(bdr_face_X.Read(), NDOFS, nf_bdr);
-         auto Y = Reshape(bdr_face_Y.ReadWrite(), NDOFS, nf_bdr);
-         auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         mfem::forall(nf_bdr*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
+         const int f = glob_j/NDOFS;
+         const int j = glob_j%NDOFS;
+         real_t res = 0.0;
+         for (int i = 0; i < NDOFS; i++)
          {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            real_t res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A(i, j, f)*X(i, f);
-            }
-            Y(j, f) += res;
-         });
-         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
-      }
+            res += A(i, j, f)*X(i, f);
+         }
+         Y(j, f) += res;
+      });
+      // Apply the Boundary Face Restriction transposed
+      bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
    }
 }
 
@@ -1151,35 +1232,164 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
    }
 
    // Treatment of boundary faces
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
+   if (!factorize_face_terms && bdr_face_restrict_lex && ea_data_bdr.Size() > 0)
    {
       // Apply the Boundary Face Restriction
       bdr_face_restrict_lex->Mult(x, bdr_face_X);
-      if (bdr_face_X.Size()>0)
+      bdr_face_Y = 0.0;
+      // Apply the boundary face matrices transposed
+      const int NDOFS = faceDofs;
+      auto X = Reshape(bdr_face_X.Read(), NDOFS, nf_bdr);
+      auto Y = Reshape(bdr_face_Y.ReadWrite(), NDOFS, nf_bdr);
+      auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
+      mfem::forall(nf_bdr*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
       {
-         bdr_face_Y = 0.0;
-         // Apply the boundary face matrices transposed
-         const int NDOFS = faceDofs;
-         auto X = Reshape(bdr_face_X.Read(), NDOFS, nf_bdr);
-         auto Y = Reshape(bdr_face_Y.ReadWrite(), NDOFS, nf_bdr);
-         auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         mfem::forall(nf_bdr*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
+         const int f = glob_j/NDOFS;
+         const int j = glob_j%NDOFS;
+         real_t res = 0.0;
+         for (int i = 0; i < NDOFS; i++)
          {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            real_t res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A(j, i, f)*X(i, f);
-            }
-            Y(j, f) += res;
-         });
-         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+            res += A(j, i, f)*X(i, f);
+         }
+         Y(j, f) += res;
+      });
+      // Apply the Boundary Face Restriction transposed
+      bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+   }
+}
+
+void EABilinearFormExtension::GetElementMatrices(
+   DenseTensor &element_matrices, ElementDofOrdering ordering, bool add_bdr)
+{
+   // Ensure the EA data is assembled
+   if (ea_data.Size() == 0) { Assemble(); }
+
+   const int ndofs = elemDofs;
+   element_matrices.SetSize(ndofs, ndofs, ne);
+   const int N = element_matrices.TotalSize();
+
+   const auto d_ea_data = Reshape(ea_data.Read(), ndofs, ndofs, ne);
+   auto d_element_matrices = Reshape(element_matrices.Write(),
+                                     ndofs, ndofs,
+                                     ne);
+
+   const int *d_dof_map = nullptr;
+   Array<int> dof_map;
+   if (ordering == ElementDofOrdering::NATIVE)
+   {
+      const TensorBasisElement* tbe =
+         dynamic_cast<const TensorBasisElement*>(trial_fes->GetFE(0));
+      if (tbe)
+      {
+         // Deep copy to avoid issues with host device (see similar comment in
+         // HybridizationExtension::ConstructC).
+         dof_map = tbe->GetDofMap();
+         d_dof_map = dof_map.Read();
       }
    }
+
+   if (d_dof_map)
+   {
+      // Reordering required
+      mfem::forall(N, [=] MFEM_HOST_DEVICE (int idx)
+      {
+         const int e = idx / ndofs / ndofs;
+         const int i = idx % ndofs;
+         const int j = (idx / ndofs) % ndofs;
+         const int ii_s = d_dof_map[i];
+         const int ii = (ii_s >= 0) ? ii_s : -1 - ii_s;
+         const int s_i = (ii_s >= 0) ? 1 : -1;
+         const int jj_s = d_dof_map[j];
+         const int jj = (jj_s >= 0) ? jj_s : -1 - jj_s;
+         const int s_j = (jj_s >= 0) ? 1 : -1;
+         d_element_matrices(ii, jj, e) = s_i*s_j*d_ea_data(j, i, e);
+      });
+   }
+   else
+   {
+      // No reordering required
+      mfem::forall(N, [=] MFEM_HOST_DEVICE (int idx)
+      {
+         const int e = idx / ndofs / ndofs;
+         const int i = idx % ndofs;
+         const int j = (idx / ndofs) % ndofs;
+         d_element_matrices(i, j, e) = d_ea_data(j, i, e);
+      });
+   }
+
+   if (add_bdr && ea_data_bdr.Size() > 0)
+   {
+      const int ndof_face = faceDofs;
+      const auto d_ea_bdr = Reshape(ea_data_bdr.Read(),
+                                    ndof_face, ndof_face, nf_bdr);
+
+      // Get all the local face maps (mapping from lexicographic face index to
+      // lexicographic volume index, depending on the local face index).
+      const Mesh &mesh = *trial_fes->GetMesh();
+      const int dim = mesh.Dimension();
+      const int n_faces_per_el = 2*dim; // assuming tensor product
+      Array<int> face_maps(ndof_face * n_faces_per_el);
+      for (int lf_i = 0; lf_i < n_faces_per_el; ++lf_i)
+      {
+         Array<int> face_map(ndof_face);
+         trial_fes->GetFE(0)->GetFaceMap(lf_i, face_map);
+         for (int i = 0; i < ndof_face; ++i)
+         {
+            face_maps[i + lf_i*ndof_face] = face_map[i];
+         }
+      }
+
+      Array<int> face_info(nf_bdr * 2);
+      {
+         int fidx = 0;
+         for (int f = 0; f < mesh.GetNumFaces(); ++f)
+         {
+            Mesh::FaceInformation finfo = mesh.GetFaceInformation(f);
+            if (!finfo.IsBoundary()) { continue; }
+            face_info[0 + fidx*2] = finfo.element[0].local_face_id;
+            face_info[1 + fidx*2] = finfo.element[0].index;
+            fidx++;
+         }
+      }
+
+      const auto d_face_maps = Reshape(face_maps.Read(), ndof_face, n_faces_per_el);
+      const auto d_face_info = Reshape(face_info.Read(), 2, nf_bdr);
+
+      const bool reorder = (ordering == ElementDofOrdering::NATIVE);
+
+      mfem::forall_2D(nf_bdr, ndof_face, ndof_face, [=] MFEM_HOST_DEVICE (int f)
+      {
+         const int lf_i = d_face_info(0, f);
+         const int e = d_face_info(1, f);
+         // Loop over face indices in "native ordering"
+         MFEM_FOREACH_THREAD(i_lex_face, x, ndof_face)
+         {
+            // Convert from lexicographic face DOF to volume DOF
+            const int i_lex = d_face_maps(i_lex_face, lf_i);
+
+            const int ii_s = d_dof_map[i_lex];
+            const int ii = (ii_s >= 0) ? ii_s : -1 - ii_s;
+
+            const int i = reorder ? ii : i_lex;
+            const int s_i = (ii_s < 0 && reorder) ? -1 : 1;
+
+            MFEM_FOREACH_THREAD(j_lex_face, y, ndof_face)
+            {
+               // Convert from lexicographic face DOF to volume DOF
+               const int j_lex = d_face_maps(j_lex_face, lf_i);
+
+               const int jj_s = d_dof_map[j_lex];
+               const int jj = (jj_s >= 0) ? jj_s : -1 - jj_s;
+
+               const int j = reorder ? jj : j_lex;
+               const int s_j = (jj_s < 0 && reorder) ? -1 : 1;
+
+               AtomicAdd(d_element_matrices(i, j, e),
+                         s_i*s_j*d_ea_bdr(i_lex_face, j_lex_face, f));
+            }
+         }
+      });
+   }
 }
 
 // Data and methods for fully-assembled bilinear forms

fem/bilinearform_ext.hpp

@@ -154,6 +154,17 @@ public:
    void Assemble() override;
    void Mult(const Vector &x, Vector &y) const override;
    void MultTranspose(const Vector &x, Vector &y) const override;
+
+   /// @brief Populates @a element_matrices with the element matrices.
+   ///
+   /// The element matrices are converted from row-major (how they are stored in
+   /// @a ea_data) to column-major format.
+   ///
+   /// If @a ordering is ElementDofOrdering::NATIVE, then the matrices are
+   /// reordered from the lexicographic ordering used internally.
+   void GetElementMatrices(DenseTensor &element_matrices,
+                           ElementDofOrdering ordering,
+                           bool add_bdr);
 };
 
 /// Data and methods for fully-assembled bilinear forms

fem/bilininteg.cpp

@@ -72,6 +72,14 @@ void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
               "   is not implemented for this class.");
 }
 
+void BilinearFormIntegrator::AssembleEABoundary(const FiniteElementSpace &fes,
+                                                Vector &emat,
+                                                const bool add)
+{
+   MFEM_ABORT("BilinearFormIntegrator::AssembleEABoundary(...)\n"
+              "   is not implemented for this class.");
+}
+
 void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
                                                      &fes,
                                                      Vector &ea_data_int,
@@ -82,6 +90,16 @@ void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
               "   is not implemented for this class.");
 }
 
+void BilinearFormIntegrator::AssembleEAInteriorFaces(
+   const FiniteElementSpace &trial_fes,
+   const FiniteElementSpace &test_fes,
+   Vector &emat,
+   const bool add)
+{
+   MFEM_ABORT("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
+              "   is not implemented for this class.");
+}
+
 void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
                                                      &fes,
                                                      Vector &ea_data_bdr,
@@ -855,6 +873,34 @@ const IntegrationRule &GradientIntegrator::GetRule(const FiniteElement
 }
 
 
+DiffusionIntegrator::DiffusionIntegrator(const IntegrationRule *ir)
+   : BilinearFormIntegrator(ir),
+     Q(nullptr), VQ(nullptr), MQ(nullptr), maps(nullptr), geom(nullptr)
+{
+   static Kernels kernels;
+}
+
+DiffusionIntegrator::DiffusionIntegrator(Coefficient &q,
+                                         const IntegrationRule *ir)
+   : DiffusionIntegrator(ir)
+{
+   Q = &q;
+}
+
+DiffusionIntegrator::DiffusionIntegrator(VectorCoefficient &q,
+                                         const IntegrationRule *ir)
+   : DiffusionIntegrator(ir)
+{
+   VQ = &q;
+}
+
+DiffusionIntegrator::DiffusionIntegrator(MatrixCoefficient &q,
+                                         const IntegrationRule *ir)
+   : DiffusionIntegrator(ir)
+{
+   MQ = &q;
+}
+
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -1310,6 +1356,17 @@ const IntegrationRule &DiffusionIntegrator::GetRule(
    return IntRules.Get(trial_fe.GetGeomType(), order);
 }
 
+MassIntegrator::MassIntegrator(const IntegrationRule *ir)
+   : BilinearFormIntegrator(ir), Q(nullptr), maps(nullptr), geom(nullptr)
+{
+   static Kernels kernels;
+}
+
+MassIntegrator::MassIntegrator(Coefficient &q, const IntegrationRule *ir)
+   : MassIntegrator(ir)
+{
+   Q = &q;
+}
 
 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,

fem/bilininteg.hpp

@@ -124,11 +124,25 @@ public:
    /// Assemble diagonal and add it to Vector @a diag.
    virtual void AssembleDiagonalMF(Vector &diag);
 
+   virtual void AssembleEABoundary(const FiniteElementSpace &fes,
+                                   Vector &ea_data_bdr,
+                                   const bool add = true);
+
    virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                         Vector &ea_data_int,
                                         Vector &ea_data_ext,
                                         const bool add = true);
 
+   /// @brief Method defining element assembly for mixed trace integrators.
+   ///
+   /// This is the element assembly analogue of AssembleFaceMatrix(const
+   /// FiniteElement&, const FiniteElement&, const FiniteElement&,
+   /// FaceElementTransformations&, DenseMatrix&).
+   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &trial_fes,
+                                        const FiniteElementSpace &test_fes,
+                                        Vector &emat,
+                                        const bool add = true);
+
    virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
                                         Vector &ea_data_bdr,
                                         const bool add = true);
@@ -383,6 +397,7 @@ public:
    void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
                    const bool add) override;
 
+   using BilinearFormIntegrator::AssembleEAInteriorFaces;
    void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                 Vector &ea_data_int,
                                 Vector &ea_data_ext,
@@ -494,6 +509,7 @@ public:
    void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
                    const bool add) override;
 
+   using BilinearFormIntegrator::AssembleEAInteriorFaces;
    void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                 Vector &ea_data_int,
                                 Vector &ea_data_ext,
@@ -2156,7 +2172,7 @@ public:
 
    MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
    MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
-   static struct Kernels { Kernels(); } kernels;
+   struct Kernels { Kernels(); };
 
 protected:
    Coefficient *Q;
@@ -2234,26 +2250,16 @@ private:
 
 public:
    /// Construct a diffusion integrator with coefficient Q = 1
-   DiffusionIntegrator(const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(const IntegrationRule *ir = nullptr);
 
    /// Construct a diffusion integrator with a scalar coefficient q
-   DiffusionIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(&q), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr);
 
    /// Construct a diffusion integrator with a vector coefficient q
-   DiffusionIntegrator(VectorCoefficient &q,
-                       const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(&q), MQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(VectorCoefficient &q, const IntegrationRule *ir = nullptr);
 
    /// Construct a diffusion integrator with a matrix coefficient q
-   DiffusionIntegrator(MatrixCoefficient &q,
-                       const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(&q), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(MatrixCoefficient &q, const IntegrationRule *ir = nullptr);
 
    /** Given a particular Finite Element computes the element stiffness matrix
        elmat. */
@@ -2344,6 +2350,8 @@ protected:
    const FaceGeometricFactors *face_geom; ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
 
+   void AssembleEA_(Vector &ea, const bool add);
+
 public:
 
    using ApplyKernelType = void(*)(const int, const Array<real_t>&,
@@ -2356,15 +2364,13 @@ public:
 
    MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
    MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
-   static struct Kernels { Kernels(); } kernels;
+   struct Kernels { Kernels(); };
 
 public:
-   MassIntegrator(const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
+   MassIntegrator(const IntegrationRule *ir = nullptr);
 
    /// Construct a mass integrator with coefficient q
-   MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL) { }
+   MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL);
 
    /** Given a particular Finite Element computes the element mass matrix
        elmat. */
@@ -2386,7 +2392,10 @@ public:
    void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
                    const bool add) override;
 
-   void AssembleDiagonalPA(Vector &diag) override;
+   virtual void AssembleEABoundary(const FiniteElementSpace &fes, Vector &emat,
+                                   const bool add) override;
+
+   virtual void AssembleDiagonalPA(Vector &diag) override;
 
    void AssembleDiagonalMF(Vector &diag) override;
 
@@ -2886,12 +2895,14 @@ public:
                                ElementTransformation &Trans,
                                DenseMatrix &elmat) override;
 
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssemblePA(const FiniteElementSpace &trial_fes,
-                   const FiniteElementSpace &test_fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultTransposePA(const Vector &x, Vector &y) const override;
-   void AssembleDiagonalPA(Vector& diag) override;
+   virtual void AssemblePA(const FiniteElementSpace &fes) override;
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes) override;
+   virtual void AddMultPA(const Vector &x, Vector &y) const override;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const override;
+   virtual void AssembleDiagonalPA(Vector& diag) override;
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                           const bool add) override;
 
    const Coefficient *GetCoefficient() const { return Q; }
 };
@@ -2950,11 +2961,6 @@ class DivDivIntegrator: public BilinearFormIntegrator
 protected:
    Coefficient *Q;
 
-   using BilinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AssembleDiagonalPA(Vector& diag) override;
-
 private:
 #ifndef MFEM_THREAD_SAFE
    Vector divshape, te_divshape;
@@ -2981,6 +2987,13 @@ public:
                                ElementTransformation &Trans,
                                DenseMatrix &elmat) override;
 
+   using BilinearFormIntegrator::AssemblePA;
+   void AssemblePA(const FiniteElementSpace &fes) override;
+   void AddMultPA(const Vector &x, Vector &y) const override;
+   void AssembleDiagonalPA(Vector& diag) override;
+   void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                   const bool add) override;
+
    const Coefficient *GetCoefficient() const { return Q; }
 };
 
@@ -3282,6 +3295,7 @@ public:
 
    void AddMultPA(const Vector&, Vector&) const override;
 
+   using BilinearFormIntegrator::AssembleEAInteriorFaces;
    void AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                 Vector &ea_data_int,
                                 Vector &ea_data_ext,
@@ -3590,6 +3604,12 @@ public:
                            const FiniteElement &test_fe2,
                            FaceElementTransformations &Trans,
                            DenseMatrix &elmat) override;
+
+   using BilinearFormIntegrator::AssembleEAInteriorFaces;
+   void AssembleEAInteriorFaces(const FiniteElementSpace &trial_fes,
+                                const FiniteElementSpace &test_fes,
+                                Vector &emat,
+                                const bool add = true) override;
 };
 
 /** Integrator for the DPG form:$ \langle v, w \rangle $ over a face (the interface) where
@@ -3722,14 +3742,37 @@ private:
     the range space. Otherwise, a dof projection matrix is constructed. */
 class IdentityInterpolator : public DiscreteInterpolator
 {
+protected:
+   const int vdim;
+
 public:
-   IdentityInterpolator(): dofquad_fe(NULL) { }
+   /** @brief Construct an identity interpolator.
+
+       @param[in]  vdim_  Vector dimension (number of components) in the domain
+                          and range FE spaces.
+   */
+   IdentityInterpolator(int vdim_ = 1) : vdim(vdim_) { }
 
    void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                const FiniteElement &ran_fe,
                                ElementTransformation &Trans,
                                DenseMatrix &elmat) override
-   { ran_fe.Project(dom_fe, Trans, elmat); }
+   {
+      if (vdim == 1)
+      {
+         ran_fe.Project(dom_fe, Trans, elmat);
+         return;
+      }
+      DenseMatrix elmat_block;
+      ran_fe.Project(dom_fe, Trans, elmat_block);
+      elmat.SetSize(vdim*elmat_block.Height(), vdim*elmat_block.Width());
+      elmat = 0_r;
+      for (int i = 0; i < vdim; i++)
+      {
+         elmat.SetSubMatrix(i*elmat_block.Height(), i*elmat_block.Width(),
+                            elmat_block);
+      }
+   }
 
    using BilinearFormIntegrator::AssemblePA;
    void AssemblePA(const FiniteElementSpace &trial_fes,
@@ -3738,11 +3781,9 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultTransposePA(const Vector &x, Vector &y) const override;
 
-   virtual ~IdentityInterpolator() { delete dofquad_fe; }
-
 private:
    /// 1D finite element that generates and owns the 1D DofToQuad maps below
-   FiniteElement *dofquad_fe;
+   std::unique_ptr<FiniteElement> dofquad_fe;
 
    const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
    const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
@@ -3752,6 +3793,16 @@ private:
 };
 
 
+/** @brief Class identical to IdentityInterpolator with the exception that it
+    requires the vector dimension (number of components) to be specified during
+    construction. */
+class VectorIdentityInterpolator : public IdentityInterpolator
+{
+public:
+   VectorIdentityInterpolator(int vdim_) : IdentityInterpolator(vdim_) { }
+};
+
+
 /** Class for constructing the (local) discrete curl matrix which can be used
     as an integrator in a DiscreteLinearOperator object to assemble the global
     discrete curl matrix. */

fem/coefficient.hpp

@@ -798,6 +798,12 @@ public:
    /// Sets coefficient in the vector.
    void Set(int i, Coefficient *c, bool own=true);
 
+   /// Set ownership of the i'th coefficient
+   void SetOwnership(int i, bool own) { ownCoeff[i] = own; }
+
+   /// Get ownership of the i'th coefficient
+   bool GetOwnership(int i) const { return ownCoeff[i]; }
+
    /// Evaluates i'th component of the vector of coefficients and returns the
    /// value.
    real_t Eval(int i, ElementTransformation &T, const IntegrationPoint &ip)
@@ -1320,6 +1326,12 @@ public:
        can be overridden with the @a own parameter. */
    void Set(int i, int j, Coefficient * c, bool own=true);
 
+   /// Set ownership of the coefficient at (i,j) in the matrix
+   void SetOwnership(int i, int j, bool own) { ownCoeff[i*width+j] = own; }
+
+   /// Get ownership of the coefficient at (i,j) in the matrix
+   bool GetOwnership(int i, int j) const { return ownCoeff[i*width+j]; }
+
    using MatrixCoefficient::Eval;
 
    /// Evaluate coefficient located at (i,j) in the matrix using integration
@@ -1360,6 +1372,12 @@ public:
        can be overridden with the @a own parameter. */
    void Set(int i, VectorCoefficient * c, bool own=true);
 
+   /// Set ownership of the i'th coefficient
+   void SetOwnership(int i, bool own) { ownCoeff[i] = own; }
+
+   /// Get ownership of the i'th coefficient
+   bool GetOwnership(int i) const { return ownCoeff[i]; }
+
    using MatrixCoefficient::Eval;
 
    /// Evaluate coefficient located at the i-th row of the matrix using integration

fem/complex_fem.cpp

@@ -1245,7 +1245,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
          hypre_ParCSRMatrix *Aih = *Ah;
          Ah->HypreReadWrite();
          const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(GetHypreMemoryClass(), n);
+            ess_tdof_list.GetMemory().Read(GetHypreForallMemoryClass(), n);
          HYPRE_Int *d_diag_i = Aih->diag->i;
          real_t *d_diag_data = Aih->diag->data;
          mfem::hypre_forall(n, [=] MFEM_HOST_DEVICE (int k)

fem/fe/face_map_utils.hpp

@@ -69,16 +69,16 @@ inline int ToLexOrdering2D(const int face_id, const int size1d, const int i)
 }
 
 /// @brief Given a face DOF index on a shared face, ordered lexicographically
-/// relative to element 1, return the corresponding face DOF index ordered
-/// lexicographically relative to element 2.
+/// relative to element the element (where the local face is face_id), and
+/// return the corresponding face DOF index ordered lexicographically relative
+/// to the face itself.
 MFEM_HOST_DEVICE
-inline int PermuteFace2D(const int face_id1, const int face_id2,
-                         const int orientation, const int size1d,
-                         const int index)
+inline int PermuteFace2D(const int face_id, const int orientation,
+                         const int size1d, const int index)
 {
    int new_index;
    // Convert from element 1 lex ordering to native ordering
-   if (face_id1 == 2 || face_id1 == 3)
+   if (face_id == 2 || face_id == 3)
    {
       new_index = size1d-1-index;
    }
@@ -91,7 +91,18 @@ inline int PermuteFace2D(const int face_id1, const int face_id2,
    {
       new_index = size1d-1-new_index;
    }
-   // Covert to element 2 lex ordering
+   return new_index;
+}
+
+/// @brief Given a face DOF index on a shared face, ordered lexicographically
+/// relative to element 1, return the corresponding face DOF index ordered
+/// lexicographically relative to element 2.
+MFEM_HOST_DEVICE
+inline int PermuteFace2D(const int face_id1, const int face_id2,
+                         const int orientation, const int size1d,
+                         const int index)
+{
+   const int new_index = PermuteFace2D(face_id1, orientation, size1d, index);
    return ToLexOrdering2D(face_id2, size1d, new_index);
 }
 
@@ -116,26 +127,22 @@ inline int ToLexOrdering3D(const int face_id, const int size1d, const int i,
    }
 }
 
-/// @brief Given the index of a face DOF in lexicographic ordering relative
-/// element 1, permute the index so that it is lexicographically ordered
-/// relative to element 2.
-///
-/// The given face corresponds to local face index @a face_id1 relative to
-/// element 1, and @a face_id2 (with @a orientation) relative to element 2.
+/// @brief Given the index of a face DOF in lexicographic ordering relative the
+/// element (where the local face id is @a face_id), permute the index so that
+/// it is lexicographically ordered relative to the face itself.
 MFEM_HOST_DEVICE
-inline int PermuteFace3D(const int face_id1, const int face_id2,
-                         const int orientation,
+inline int PermuteFace3D(const int face_id, const int orientation,
                          const int size1d, const int index)
 {
    int i=0, j=0, new_i=0, new_j=0;
    i = index%size1d;
    j = index/size1d;
    // Convert from lex ordering
-   if (face_id1==3 || face_id1==4)
+   if (face_id==3 || face_id==4)
    {
       i = size1d-1-i;
    }
-   else if (face_id1==0)
+   else if (face_id==0)
    {
       j = size1d-1-j;
    }
@@ -175,6 +182,23 @@ inline int PermuteFace3D(const int face_id1, const int face_id2,
          new_j = (size1d-1-j);
          break;
    }
+   return new_i + new_j*size1d;
+}
+
+/// @brief Given the index of a face DOF in lexicographic ordering relative
+/// element 1, permute the index so that it is lexicographically ordered
+/// relative to element 2.
+///
+/// The given face corresponds to local face index @a face_id1 relative to
+/// element 1, and @a face_id2 (with @a orientation) relative to element 2.
+MFEM_HOST_DEVICE
+inline int PermuteFace3D(const int face_id1, const int face_id2,
+                         const int orientation,
+                         const int size1d, const int index)
+{
+   const int new_index = PermuteFace3D(face_id1, orientation, size1d, index);
+   const int new_i = new_index%size1d;
+   const int new_j = new_index/size1d;
    return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
 }
 

fem/fespace.cpp

@@ -1417,6 +1417,10 @@ const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
             res = new NCL2FaceRestriction(*this, f_ordering, type, m);
          }
       }
+      else if (dynamic_cast<const DG_Interface_FECollection*>(fec))
+      {
+         res = new L2InterfaceFaceRestriction(*this, f_ordering, type);
+      }
       else
       {
          res = new ConformingFaceRestriction(*this, f_ordering, type);

fem/gslib.cpp

@@ -37,7 +37,7 @@ FindPointsGSLIB::FindPointsGSLIB()
    : mesh(NULL),
      fec_map_lin(NULL),
      fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
-     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
+     dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
    mesh_split.SetSize(4);
@@ -85,7 +85,7 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
    : mesh(NULL),
      fec_map_lin(NULL),
      fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
-     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
+     dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
    mesh_split.SetSize(4);
@@ -307,6 +307,7 @@ void FindPointsGSLIB::FreeData()
    }
    if (fec_map_lin) { delete fec_map_lin; fec_map_lin = NULL; }
    setupflag = false;
+   points_cnt = -1;
 }
 
 void FindPointsGSLIB::SetupSplitMeshes()
@@ -897,7 +898,8 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
       int gf_order_h1 = std::max(gf_order, 1); // H1 should be at least order 1
       H1_FECollection fec(gf_order_h1, dim);
       const int ncomp = field_in.FESpace()->GetVDim();
-      FiniteElementSpace fes(mesh, &fec, ncomp);
+      FiniteElementSpace fes(mesh, &fec, ncomp,
+                             field_in.FESpace()->GetOrdering());
       GridFunction field_in_h1(&fes);
 
       if (avgtype == AvgType::ARITHMETIC)
@@ -927,7 +929,7 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
       {
          for (int i = 0; i < indl2.Size(); i++)
          {
-            int idx = field_in.FESpace()->GetOrdering() == Ordering::byNODES ?
+            int idx = field_in_h1.FESpace()->GetOrdering() == Ordering::byNODES?
                       indl2[i] + j*points_cnt:
                       indl2[i]*ncomp + j;
             field_out(idx) = field_out_l2(idx);
@@ -1172,7 +1174,7 @@ void FindPointsGSLIB::DistributePointInfoToOwningMPIRanks(
    Array<unsigned int> &recv_elem, Vector &recv_ref,
    Array<unsigned int> &recv_code)
 {
-   MFEM_VERIFY(points_cnt,
+   MFEM_VERIFY(points_cnt >= 0,
                "Invalid size. Please make sure to call FindPoints method "
                "before calling this function.");
 

fem/hybridization.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "hybridization.hpp"
+#include "hybridization_ext.hpp"
 #include "gridfunc.hpp"
 
 #ifdef MFEM_USE_MPI
@@ -29,36 +30,21 @@ namespace mfem
 
 Hybridization::Hybridization(FiniteElementSpace *fespace,
                              FiniteElementSpace *c_fespace)
-   : fes(fespace), c_fes(c_fespace), c_bfi(NULL), extern_bdr_constr_integs(0),
-     Ct(NULL), H(NULL), Af_data(NULL), Af_ipiv(NULL)
+   : fes(*fespace), c_fes(*c_fespace)
 {
 #ifdef MFEM_USE_MPI
-   pC = P_pc = NULL;
    pH.SetType(Operator::Hypre_ParCSR);
 #endif
 }
 
-Hybridization::~Hybridization()
+void Hybridization::EnableDeviceExecution()
 {
-#ifdef MFEM_USE_MPI
-   delete P_pc;
-   delete pC;
-#endif
-   delete [] Af_ipiv;
-   delete [] Af_data;
-   delete H;
-   delete Ct;
-   delete c_bfi;
-   if (!extern_bdr_constr_integs)
-   {
-      for (int k=0; k < boundary_constraint_integs.Size(); k++)
-      { delete boundary_constraint_integs[k]; }
-   }
+   ext.reset(new HybridizationExtension(*this));
 }
 
 void Hybridization::ConstructC()
 {
-   const int NE = fes->GetNE();
+   const int NE = fes.GetNE();
    int num_hat_dofs = hat_offsets[NE];
    Array<int> vdofs, c_vdofs;
 
@@ -73,7 +59,7 @@ void Hybridization::ConstructC()
 
    int c_num_face_nbr_dofs = 0;
 #ifdef MFEM_USE_MPI
-   ParFiniteElementSpace *c_pfes = dynamic_cast<ParFiniteElementSpace*>(c_fes);
+   ParFiniteElementSpace *c_pfes = dynamic_cast<ParFiniteElementSpace*>(&c_fes);
    ParMesh *pmesh = c_pfes ? c_pfes->GetParMesh() : NULL;
    HYPRE_BigInt num_shared_slave_faces = 0, glob_num_shared_slave_faces = 0;
    if (c_pfes)
@@ -105,15 +91,15 @@ void Hybridization::ConstructC()
    }
 #endif
 
-   const int c_vsize = c_fes->GetVSize();
-   Ct = new SparseMatrix(num_hat_dofs, c_vsize + c_num_face_nbr_dofs);
+   const int c_vsize = c_fes.GetVSize();
+   Ct.reset(new SparseMatrix(num_hat_dofs, c_vsize + c_num_face_nbr_dofs));
 
    if (c_bfi)
    {
       const int skip_zeros = 1;
       DenseMatrix elmat;
       FaceElementTransformations *FTr;
-      Mesh *mesh = fes->GetMesh();
+      Mesh *mesh = fes.GetMesh();
       int num_faces = mesh->GetNumFaces();
       for (int i = 0; i < num_faces; i++)
       {
@@ -133,17 +119,17 @@ void Hybridization::ConstructC()
          {
             vdofs[s1+j] = o2 + j;
          }
-         c_fes->GetFaceVDofs(i, c_vdofs);
-         c_bfi->AssembleFaceMatrix(*c_fes->GetFaceElement(i),
-                                   *fes->GetFE(FTr->Elem1No),
-                                   *fes->GetFE(FTr->Elem2No),
+         c_fes.GetFaceVDofs(i, c_vdofs);
+         c_bfi->AssembleFaceMatrix(*c_fes.GetFaceElement(i),
+                                   *fes.GetFE(FTr->Elem1No),
+                                   *fes.GetFE(FTr->Elem2No),
                                    *FTr, elmat);
          // zero-out small elements in elmat
          elmat.Threshold(mtol * elmat.MaxMaxNorm());
          Ct->AddSubMatrix(vdofs, c_vdofs, elmat, skip_zeros);
       }
 
-      if (boundary_constraint_integs.Size())
+      if (!boundary_constraint_integs.empty())
       {
          const FiniteElement *fe1, *fe2;
          const FiniteElement *face_el;
@@ -152,14 +138,14 @@ void Hybridization::ConstructC()
          Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                     mesh->bdr_attributes.Max() : 0);
          bdr_attr_marker = 0;
-         for (int k = 0; k < boundary_constraint_integs.Size(); k++)
+         for (size_t k = 0; k < boundary_constraint_integs.size(); k++)
          {
-            if (boundary_constraint_integs_marker[k] == NULL)
+            if (boundary_constraint_integs_marker[k].IsEmpty())
             {
                bdr_attr_marker = 1;
                break;
             }
-            Array<int> &bdr_marker = *boundary_constraint_integs_marker[k];
+            Array<int> &bdr_marker = boundary_constraint_integs_marker[k];
             MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                         "invalid boundary marker for boundary face integrator #"
                         << k << ", counting from zero");
@@ -169,7 +155,7 @@ void Hybridization::ConstructC()
             }
          }
 
-         for (int i = 0; i < fes->GetNBE(); i++)
+         for (int i = 0; i < fes.GetNBE(); i++)
          {
             const int bdr_attr = mesh->GetBdrAttribute(i);
             if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -186,17 +172,17 @@ void Hybridization::ConstructC()
                vdofs[j] = o1 + j;
             }
             int iface = mesh->GetBdrElementFaceIndex(i);
-            c_fes->GetFaceVDofs(iface, c_vdofs);
-            face_el = c_fes->GetFaceElement(iface);
-            fe1 = fes -> GetFE (FTr -> Elem1No);
+            c_fes.GetFaceVDofs(iface, c_vdofs);
+            face_el = c_fes.GetFaceElement(iface);
+            fe1 = fes.GetFE(FTr -> Elem1No);
             // The fe2 object is really a dummy and not used on the boundaries,
             // but we can't dereference a NULL pointer, and we don't want to
             // actually make a fake element.
             fe2 = fe1;
-            for (int k = 0; k < boundary_constraint_integs.Size(); k++)
+            for (size_t k = 0; k < boundary_constraint_integs.size(); k++)
             {
-               if (boundary_constraint_integs_marker[k] &&
-                   (*boundary_constraint_integs_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_constraint_integs_marker[k].Size() &&
+                   boundary_constraint_integs_marker[k][bdr_attr-1] == 0) { continue; }
 
                boundary_constraint_integs[k]->AssembleFaceMatrix(*face_el, *fe1, *fe2, *FTr,
                                                                  elmat);
@@ -220,8 +206,8 @@ void Hybridization::ConstructC()
             {
                FTr = pmesh->GetFaceElementTransformations(face_no);
                MFEM_ASSERT(FTr->Elem2No < 0, "");
-               face_fe = c_fes->GetFaceElement(face_no);
-               c_fes->GetFaceVDofs(face_no, c_vdofs);
+               face_fe = c_fes.GetFaceElement(face_no);
+               c_fes.GetFaceVDofs(face_no, c_vdofs);
             }
             else
             {
@@ -242,7 +228,7 @@ void Hybridization::ConstructC()
             {
                vdofs[j] = o1 + j;
             }
-            fe = fes->GetFE(FTr->Elem1No);
+            fe = fes.GetFE(FTr->Elem1No);
             c_bfi->AssembleFaceMatrix(*face_fe, *fe, *fe, *FTr, elmat);
             // zero-out small elements in elmat
             elmat.Threshold(mtol * elmat.MaxMaxNorm());
@@ -272,12 +258,12 @@ void Hybridization::ConstructC()
                                Ct->GetI(), Ct_J.GetData(), Ct->GetData(),
                                Ct_rows, c_pfes->GetDofOffsets());
             Ct_J.DeleteAll();
-            pC = pCt.Transpose();
+            pC.reset(pCt.Transpose());
          }
          if (pmesh->Nonconforming())
          {
             // TODO - Construct P_pc directly in the pH format
-            P_pc = c_pfes->GetPartialConformingInterpolation();
+            P_pc.reset(c_pfes->GetPartialConformingInterpolation());
          }
       }
 #endif
@@ -294,15 +280,21 @@ void Hybridization::Init(const Array<int> &ess_tdof_list)
 {
    if (Ct) { return; }
 
+   if (ext)
+   {
+      ext->Init(ess_tdof_list);
+      return;
+   }
+
    // count the number of dofs in the discontinuous version of fes:
-   const int NE = fes->GetNE();
+   const int NE = fes.GetNE();
    Array<int> vdofs;
    int num_hat_dofs = 0;
    hat_offsets.SetSize(NE+1);
    hat_offsets[0] = 0;
    for (int i = 0; i < NE; i++)
    {
-      fes->GetElementVDofs(i, vdofs);
+      fes.GetElementVDofs(i, vdofs);
       num_hat_dofs += vdofs.Size();
       hat_offsets[i+1] = num_hat_dofs;
    }
@@ -318,7 +310,7 @@ void Hybridization::Init(const Array<int> &ess_tdof_list)
       C->PrintMatlab(C_file);
       delete C;
 
-      const SparseMatrix *P = fes->GetConformingProlongation();
+      const SparseMatrix *P = fes.GetConformingProlongation();
       if (P)
       {
          std::ofstream P_file("P_matrix.txt");
@@ -333,11 +325,11 @@ void Hybridization::Init(const Array<int> &ess_tdof_list)
    hat_dofs_marker.SetSize(num_hat_dofs);
    Array<int> free_tdof_marker;
 #ifdef MFEM_USE_MPI
-   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace*>(fes);
+   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace*>(&fes);
    free_tdof_marker.SetSize(pfes ? pfes->TrueVSize() :
-                            fes->GetConformingVSize());
+                            fes.GetConformingVSize());
 #else
-   free_tdof_marker.SetSize(fes->GetConformingVSize());
+   free_tdof_marker.SetSize(fes.GetConformingVSize());
 #endif
    free_tdof_marker = 1;
    for (int i = 0; i < ess_tdof_list.Size(); i++)
@@ -348,38 +340,38 @@ void Hybridization::Init(const Array<int> &ess_tdof_list)
 #ifdef MFEM_USE_MPI
    if (!pfes)
    {
-      const SparseMatrix *cP = fes->GetConformingProlongation();
+      const SparseMatrix *cP = fes.GetConformingProlongation();
       if (!cP)
       {
          free_vdofs_marker.MakeRef(free_tdof_marker);
       }
       else
       {
-         free_vdofs_marker.SetSize(fes->GetVSize());
+         free_vdofs_marker.SetSize(fes.GetVSize());
          cP->BooleanMult(free_tdof_marker, free_vdofs_marker);
       }
    }
    else
    {
       HypreParMatrix *P = pfes->Dof_TrueDof_Matrix();
-      free_vdofs_marker.SetSize(fes->GetVSize());
+      free_vdofs_marker.SetSize(fes.GetVSize());
       P->BooleanMult(1, free_tdof_marker, 0, free_vdofs_marker);
    }
 #else
-   const SparseMatrix *cP = fes->GetConformingProlongation();
+   const SparseMatrix *cP = fes.GetConformingProlongation();
    if (!cP)
    {
       free_vdofs_marker.MakeRef(free_tdof_marker);
    }
    else
    {
-      free_vdofs_marker.SetSize(fes->GetVSize());
+      free_vdofs_marker.SetSize(fes.GetVSize());
       cP->BooleanMult(free_tdof_marker, free_vdofs_marker);
    }
 #endif
    for (int i = 0; i < NE; i++)
    {
-      fes->GetElementVDofs(i, vdofs);
+      fes.GetElementVDofs(i, vdofs);
       FiniteElementSpace::AdjustVDofs(vdofs);
       for (int j = 0; j < vdofs.Size(); j++)
       {
@@ -442,18 +434,18 @@ void Hybridization::Init(const Array<int> &ess_tdof_list)
 #undef MFEM_DEBUG_HERE
 #endif
 
-   Af_data = new real_t[Af_offsets[NE]];
-   Af_ipiv = new int[Af_f_offsets[NE]];
+   Af_data.SetSize(Af_offsets[NE]);
+   Af_ipiv.SetSize(Af_f_offsets[NE]);
 
 #ifdef MFEM_DEBUG
    // check that Ref = 0
-   const SparseMatrix *R = fes->GetRestrictionMatrix();
+   const SparseMatrix *R = fes.GetRestrictionMatrix();
    if (!R) { return; }
-   Array<int> vdof_marker(fes->GetVSize()); // 0 - f, 1 - e
+   Array<int> vdof_marker(fes.GetVSize()); // 0 - f, 1 - e
    vdof_marker = 0;
    for (int i = 0; i < NE; i++)
    {
-      fes->GetElementVDofs(i, vdofs);
+      fes.GetElementVDofs(i, vdofs);
       FiniteElementSpace::AdjustVDofs(vdofs);
       for (int j = 0; j < vdofs.Size(); j++)
       {
@@ -519,11 +511,17 @@ void Hybridization::GetBDofs(int el, int &num_idofs, Array<int> &b_dofs) const
 
 void Hybridization::AssembleMatrix(int el, const DenseMatrix &A)
 {
+   if (ext)
+   {
+      ext->AssembleMatrix(el, A);
+      return;
+   }
+
    Array<int> i_dofs, b_dofs;
 
    GetIBDofs(el, i_dofs, b_dofs);
 
-   DenseMatrix A_ii(Af_data + Af_offsets[el], i_dofs.Size(), i_dofs.Size());
+   DenseMatrix A_ii(&Af_data[Af_offsets[el]], i_dofs.Size(), i_dofs.Size());
    DenseMatrix A_ib(A_ii.Data() + i_dofs.Size()*i_dofs.Size(),
                     i_dofs.Size(), b_dofs.Size());
    DenseMatrix A_bi(A_ib.Data() + i_dofs.Size()*b_dofs.Size(),
@@ -557,12 +555,32 @@ void Hybridization::AssembleMatrix(int el, const DenseMatrix &A)
    }
 }
 
+void Hybridization::AssembleElementMatrices(const class DenseTensor &el_mats)
+{
+   if (ext)
+   {
+      ext->AssembleElementMatrices(el_mats);
+      return;
+   }
+
+   for (int e = 0; e < el_mats.SizeK(); ++e)
+   {
+      AssembleMatrix(e, el_mats(e));
+   }
+}
+
 void Hybridization::AssembleBdrMatrix(int bdr_el, const DenseMatrix &A)
 {
+   if (ext)
+   {
+      ext->AssembleBdrMatrix(bdr_el, A);
+      return;
+   }
+
    // Not tested.
 #ifdef MFEM_DEBUG
    Array<int> vdofs, bvdofs;
-   fes->GetBdrElementVDofs(bdr_el, bvdofs);
+   fes.GetBdrElementVDofs(bdr_el, bvdofs);
 #endif
 
    int el;
@@ -570,19 +588,19 @@ void Hybridization::AssembleBdrMatrix(int bdr_el, const DenseMatrix &A)
    Array<int> i_dofs, b_dofs, e2f;
 
    {
-      int info, vdim = fes->GetVDim();
+      int info, vdim = fes.GetVDim();
       Array<int> lvdofs;
-      Mesh *mesh = fes->GetMesh();
+      Mesh *mesh = fes.GetMesh();
       mesh->GetBdrElementAdjacentElement(bdr_el, el, info);
       e2f.SetSize(hat_offsets[el+1]-hat_offsets[el], -1);
       lvdofs.Reserve(A.Height());
-      fes->FEColl()->SubDofOrder(mesh->GetElementBaseGeometry(el),
-                                 mesh->Dimension()-1, info, lvdofs);
+      fes.FEColl()->SubDofOrder(mesh->GetElementBaseGeometry(el),
+                                mesh->Dimension()-1, info, lvdofs);
       // Convert local element dofs to local element vdofs.
       Ordering::DofsToVDofs<Ordering::byNODES>(e2f.Size()/vdim, vdim, lvdofs);
       MFEM_ASSERT(lvdofs.Size() == A.Height(), "internal error");
 #ifdef MFEM_DEBUG
-      fes->GetElementVDofs(el, vdofs);
+      fes.GetElementVDofs(el, vdofs);
       for (int i = 0; i < lvdofs.Size(); i++)
       {
          int bd = lvdofs[i];
@@ -601,7 +619,7 @@ void Hybridization::AssembleBdrMatrix(int bdr_el, const DenseMatrix &A)
 
    GetIBDofs(el, i_dofs, b_dofs);
 
-   DenseMatrix A_ii(Af_data + Af_offsets[el], i_dofs.Size(), i_dofs.Size());
+   DenseMatrix A_ii(&Af_data[Af_offsets[el]], i_dofs.Size(), i_dofs.Size());
    DenseMatrix A_ib(A_ii.Data() + i_dofs.Size()*i_dofs.Size(),
                     i_dofs.Size(), b_dofs.Size());
    DenseMatrix A_bi(A_ib.Data() + i_dofs.Size()*b_dofs.Size(),
@@ -647,15 +665,24 @@ void Hybridization::AssembleBdrMatrix(int bdr_el, const DenseMatrix &A)
 
 void Hybridization::ComputeH()
 {
+   if (ext)
+   {
+      ext->ConstructH();
+      return;
+   }
+
    const int skip_zeros = 1;
    Array<int> c_dof_marker(Ct->Width());
    Array<int> b_dofs, c_dofs;
-   const int NE = fes->GetNE();
+   const int NE = fes.GetNE();
    DenseMatrix Cb_t, Sb_inv_Cb_t, Hb;
 #ifndef MFEM_USE_MPI
-   H = new SparseMatrix(Ct->Width());
+   H.reset(new SparseMatrix(Ct->Width()));
 #else
-   H = pC ? NULL : new SparseMatrix(Ct->Width());
+   if (!pC)
+   {
+      H.reset(new SparseMatrix(Ct->Width()));
+   }
    // V = Sb^{-1} Cb^T, for parallel non-conforming meshes
    SparseMatrix *V = pC ? new SparseMatrix(Ct->Height(), Ct->Width()) : NULL;
 #endif
@@ -667,7 +694,7 @@ void Hybridization::ComputeH()
       int i_dofs_size;
       GetBDofs(el, i_dofs_size, b_dofs);
 
-      LUFactors LU_ii(Af_data + Af_offsets[el], Af_ipiv + Af_f_offsets[el]);
+      LUFactors LU_ii(&Af_data[Af_offsets[el]], Af_ipiv + Af_f_offsets[el]);
       real_t *A_ib_data = LU_ii.data + i_dofs_size*i_dofs_size;
       real_t *A_bi_data = A_ib_data + i_dofs_size*b_dofs.Size();
       LUFactors LU_bb(A_bi_data + i_dofs_size*b_dofs.Size(),
@@ -737,7 +764,7 @@ void Hybridization::ComputeH()
 #ifndef MFEM_USE_MPI
    H->Finalize(skip_zeros, fix_empty_rows);
 #else
-   ParFiniteElementSpace *c_pfes = dynamic_cast<ParFiniteElementSpace*>(c_fes);
+   ParFiniteElementSpace *c_pfes = dynamic_cast<ParFiniteElementSpace*>(&c_fes);
    if (!pC)
    {
       H->Finalize(skip_zeros, fix_empty_rows);
@@ -745,12 +772,11 @@ void Hybridization::ComputeH()
 
       OperatorHandle pP(pH.Type()), dH(pH.Type());
       // TODO - construct P_pc / Dof_TrueDof_Matrix directly in the pH format
-      pP.ConvertFrom(P_pc ? P_pc : c_pfes->Dof_TrueDof_Matrix());
+      pP.ConvertFrom(P_pc ? P_pc.get() : c_pfes->Dof_TrueDof_Matrix());
       dH.MakeSquareBlockDiag(c_pfes->GetComm(),c_pfes->GlobalVSize(),
-                             c_pfes->GetDofOffsets(), H);
+                             c_pfes->GetDofOffsets(), H.get());
       pH.MakePtAP(dH, pP);
-      delete H;
-      H = NULL;
+      H.reset();
    }
    else
    {
@@ -758,7 +784,7 @@ void Hybridization::ComputeH()
       V->Finalize();
       Array<HYPRE_BigInt> V_J(V->NumNonZeroElems());
       MFEM_ASSERT(c_pfes, "");
-      const int c_vsize = c_fes->GetVSize();
+      const int c_vsize = c_fes.GetVSize();
       HYPRE_BigInt c_ldof_offset = c_pfes->GetMyDofOffset();
       const HYPRE_BigInt *c_face_nbr_glob_ldof = c_pfes->GetFaceNbrGlobalDofMap();
       int *J = V->GetJ();
@@ -779,11 +805,11 @@ void Hybridization::ComputeH()
          // safely delete V_J and V:
          V_J.DeleteAll();
          delete V;
-         lpH = ParMult(pC, &pV);
+         lpH = ParMult(pC.get(), &pV);
       }
       OperatorHandle pP(pH.Type()), plpH(pH.Type());
       // TODO - construct P_pc directly in the pH format
-      pP.ConvertFrom(P_pc);
+      pP.ConvertFrom(P_pc.get());
       plpH.ConvertFrom(lpH);
       MFEM_VERIFY(pH.Type() != Operator::PETSC_MATIS, "To be implemented");
       pH.MakePtAP(plpH, pP);
@@ -806,18 +832,18 @@ void Hybridization::MultAfInv(const Vector &b, const Vector &lambda, Vector &bf,
 {
    // b1 = Rf^t b (assuming that Ref = 0)
    Vector b1;
-   const SparseMatrix *R = fes->GetRestrictionMatrix();
+   const SparseMatrix *R = fes.GetRestrictionMatrix();
    if (!R)
    {
       b1.SetDataAndSize(b.GetData(), b.Size());
    }
    else
    {
-      b1.SetSize(fes->GetVSize());
+      b1.SetSize(fes.GetVSize());
       R->MultTranspose(b, b1);
    }
 
-   const int NE = fes->GetMesh()->GetNE();
+   const int NE = fes.GetMesh()->GetNE();
    Array<int> vdofs, i_dofs, b_dofs;
    Vector el_vals, bf_i, i_vals, b_vals;
    bf.SetSize(hat_offsets[NE]);
@@ -825,7 +851,7 @@ void Hybridization::MultAfInv(const Vector &b, const Vector &lambda, Vector &bf,
    {
 #ifdef MFEM_USE_MPI
       ParFiniteElementSpace *c_pfes =
-         dynamic_cast<ParFiniteElementSpace*>(c_fes);
+         dynamic_cast<ParFiniteElementSpace*>(&c_fes);
       if (!c_pfes)
       {
          Ct->Mult(lambda, bf);
@@ -833,7 +859,7 @@ void Hybridization::MultAfInv(const Vector &b, const Vector &lambda, Vector &bf,
       else
       {
          Vector L(c_pfes->GetVSize());
-         (P_pc ? P_pc : c_pfes->GetProlongationMatrix())->Mult(lambda, L);
+         (P_pc ? P_pc.get() : c_pfes->GetProlongationMatrix())->Mult(lambda, L);
          pC ? pC->MultTranspose(L, bf) : Ct->Mult(L, bf);
       }
 #else
@@ -845,7 +871,7 @@ void Hybridization::MultAfInv(const Vector &b, const Vector &lambda, Vector &bf,
    vdof_marker = false;
    for (int i = 0; i < NE; i++)
    {
-      fes->GetElementVDofs(i, vdofs);
+      fes.GetElementVDofs(i, vdofs);
       b1.GetSubVector(vdofs, el_vals);
       for (int j = 0; j < vdofs.Size(); j++)
       {
@@ -863,7 +889,9 @@ void Hybridization::MultAfInv(const Vector &b, const Vector &lambda, Vector &bf,
       el_vals.GetSubVector(i_dofs, i_vals);
       el_vals.GetSubVector(b_dofs, b_vals);
 
-      LUFactors LU_ii(Af_data + Af_offsets[i], Af_ipiv + Af_f_offsets[i]);
+      real_t *Af_data_ptr = const_cast<real_t*>(&Af_data[Af_offsets[i]]);
+      int *Af_ipiv_ptr = const_cast<int*>(&Af_ipiv[Af_f_offsets[i]]);
+      LUFactors LU_ii(Af_data_ptr, Af_ipiv_ptr);
       real_t *U_ib = LU_ii.data + i_dofs.Size()*i_dofs.Size();
       real_t *L_bi = U_ib + i_dofs.Size()*b_dofs.Size();
       LUFactors LU_bb(L_bi + b_dofs.Size()*i_dofs.Size(),
@@ -884,13 +912,19 @@ void Hybridization::MultAfInv(const Vector &b, const Vector &lambda, Vector &bf,
 
 void Hybridization::ReduceRHS(const Vector &b, Vector &b_r) const
 {
+   if (ext)
+   {
+      ext->ReduceRHS(b, b_r);
+      return;
+   }
+
    // bf = Af^{-1} Rf^t b
    Vector bf;
    MultAfInv(b, b, bf, 0);
 
    // b_r = Cf bf
 #ifdef MFEM_USE_MPI
-   ParFiniteElementSpace *c_pfes = dynamic_cast<ParFiniteElementSpace*>(c_fes);
+   ParFiniteElementSpace *c_pfes = dynamic_cast<ParFiniteElementSpace*>(&c_fes);
    if (!c_pfes)
    {
       b_r.SetSize(Ct->Width());
@@ -908,7 +942,7 @@ void Hybridization::ReduceRHS(const Vector &b, Vector &b_r) const
          Ct->MultTranspose(bf, bl);
       }
       b_r.SetSize(pH.Ptr()->Height());
-      (P_pc ? P_pc : c_pfes->GetProlongationMatrix())->MultTranspose(bl, b_r);
+      (P_pc ? P_pc.get() : c_pfes->GetProlongationMatrix())->MultTranspose(bl, b_r);
    }
 #else
    b_r.SetSize(Ct->Width());
@@ -919,28 +953,34 @@ void Hybridization::ReduceRHS(const Vector &b, Vector &b_r) const
 void Hybridization::ComputeSolution(const Vector &b, const Vector &sol_r,
                                     Vector &sol) const
 {
-   // bf = Af^{-1} ( Rf^t - Cf^t sol_r )
+   if (ext)
+   {
+      ext->ComputeSolution(b, sol_r, sol);
+      return;
+   }
+
+   // bf = Af^{-1} ( Rf^t b - Cf^t sol_r )
    Vector bf;
    MultAfInv(b, sol_r, bf, 1);
 
    // sol = Rf bf
    GridFunction s;
-   const SparseMatrix *R = fes->GetRestrictionMatrix();
+   const SparseMatrix *R = fes.GetRestrictionMatrix();
    if (!R)
    {
-      MFEM_ASSERT(sol.Size() == fes->GetVSize(), "");
-      s.MakeRef(fes, sol, 0);
+      MFEM_ASSERT(sol.Size() == fes.GetVSize(), "");
+      s.MakeRef(&fes, sol, 0);
    }
    else
    {
-      s.SetSpace(fes);
+      s.SetSpace(&fes);
       R->MultTranspose(sol, s);
    }
-   const int NE = fes->GetMesh()->GetNE();
+   const int NE = fes.GetMesh()->GetNE();
    Array<int> vdofs;
    for (int i = 0; i < NE; i++)
    {
-      fes->GetElementVDofs(i, vdofs);
+      fes.GetElementVDofs(i, vdofs);
       for (int j = hat_offsets[i]; j < hat_offsets[i+1]; j++)
       {
          if (hat_dofs_marker[j] == 1) { continue; } // skip essential b.c.
@@ -957,11 +997,15 @@ void Hybridization::ComputeSolution(const Vector &b, const Vector &sol_r,
 
 void Hybridization::Reset()
 {
-   delete H;
-   H = NULL;
+   H.reset();
 #ifdef MFEM_USE_MPI
    pH.Clear();
 #endif
+   if (ext) { ext->Reset(); }
 }
 
+// Set to default in cpp file because of use of incomplete type
+// (HybridizationExtension) in unique_ptr member data.
+Hybridization::~Hybridization() = default;
+
 }

fem/hybridization.hpp

@@ -15,10 +15,26 @@
 #include "../config/config.hpp"
 #include "fespace.hpp"
 #include "bilininteg.hpp"
+#include <memory>
 
 namespace mfem
 {
 
+template <typename T>
+std::shared_ptr<T> Owning(T *t) { return std::shared_ptr<T>(t); }
+
+template <typename T>
+std::shared_ptr<T> NonOwning(T *t)
+{
+   return std::shared_ptr<T>(t, [](T*) {});
+}
+
+template <typename T>
+std::shared_ptr<T> OptionallyOwning(T *t, bool own)
+{
+   return own ? Owning(t) : NonOwning(t);
+}
+
 /** @brief Auxiliary class Hybridization, used to implement BilinearForm
     hybridization.
 
@@ -60,34 +76,43 @@ namespace mfem
     (weak) continuity constraints between neighboring elements. */
 class Hybridization
 {
+   friend class HybridizationExtension;
 protected:
-   FiniteElementSpace *fes, *c_fes;
-   BilinearFormIntegrator *c_bfi;
-
-   /// Set of constraint boundary face integrators to be applied.
-   Array<BilinearFormIntegrator*> boundary_constraint_integs;
-   Array<Array<int>*>             boundary_constraint_integs_marker;
-   /// Indicates if the boundary_constraint_integs integrators are owned externally
-   bool extern_bdr_constr_integs;
-
-   SparseMatrix *Ct, *H;
+   FiniteElementSpace &fes; ///< The finite element space.
+   FiniteElementSpace &c_fes; ///< The constraint finite element space.
+   /// Extension for device execution.
+   std::unique_ptr<class HybridizationExtension> ext;
+   /// The constraint integrator.
+   std::unique_ptr<BilinearFormIntegrator> c_bfi;
+   /// The constraint boundary face integrators.
+   std::vector<std::shared_ptr<BilinearFormIntegrator>> boundary_constraint_integs;
+   /// Boundary markers for constraint face integrators.
+   std::vector<Array<int>> boundary_constraint_integs_marker;
+   /// The constraint matrix.
+   std::unique_ptr<SparseMatrix> Ct;
+   /// The Schur complement system for the Lagrange multiplier.
+   std::unique_ptr<SparseMatrix> H;
 
    Array<int> hat_offsets, hat_dofs_marker;
    Array<int> Af_offsets, Af_f_offsets;
-   real_t *Af_data;
-   int *Af_ipiv;
+   Array<real_t> Af_data;
+   Array<int> Af_ipiv;
 
 #ifdef MFEM_USE_MPI
-   HypreParMatrix *pC, *P_pc; // for parallel non-conforming meshes
+   std::unique_ptr<HypreParMatrix> pC, P_pc; // for parallel non-conforming meshes
    OperatorHandle pH;
 #endif
 
+   /// Construct the constraint matrix.
    void ConstructC();
 
+   /// Returns the local indices of the i-dofs and b-dofs of element @a el.
    void GetIBDofs(int el, Array<int> &i_dofs, Array<int> &b_dofs) const;
 
+   /// Returns global indices of the b-dofs of element @a el.
    void GetBDofs(int el, int &num_idofs, Array<int> &b_dofs) const;
 
+   /// Construct the Schur complement system.
    void ComputeH();
 
    // Compute depending on mode:
@@ -101,43 +126,58 @@ protected:
                   int mode) const;
 
 public:
-   /// Constructor
+   /// Constructor.
    Hybridization(FiniteElementSpace *fespace, FiniteElementSpace *c_fespace);
-   /// Destructor
+
+   /// Destructor.
    ~Hybridization();
 
-   /** Set the integrator that will be used to construct the constraint matrix
-       C. The Hybridization object assumes ownership of the integrator, i.e. it
-       will delete the integrator when destroyed. */
+   /// Turns on device execution.
+   void EnableDeviceExecution();
+
+   /// @brief Set the integrator that will be used to construct the constraint
+   /// matrix C.
+   ///
+   /// The Hybridization object assumes ownership of the integrator, i.e. it
+   /// will delete the integrator when destroyed.
    void SetConstraintIntegrator(BilinearFormIntegrator *c_integ)
-   { delete c_bfi; c_bfi = c_integ; }
+   { c_bfi.reset(c_integ); }
 
-   /** Add the boundary face integrator that will be used to construct the
-       constraint matrix C. The Hybridization object assumes ownership of the
-       integrator, i.e. it will delete the integrator when destroyed. */
-   void AddBdrConstraintIntegrator(BilinearFormIntegrator *c_integ)
+   /// @brief Add a boundary face integrator that will be used to construct the
+   /// constraint matrix C.
+   ///
+   /// The integrator will apply to the boundaries specified using the marker
+   /// array @a bdr_marker. If @a bdr_marker is empty (its default value) then
+   /// the integrator will be applied on all boundaries.
+   void AddBdrConstraintIntegrator(
+      const std::shared_ptr<BilinearFormIntegrator> &c_integ,
+      const Array<int> &bdr_marker = Array<int>())
    {
-      boundary_constraint_integs.Append(c_integ);
-      boundary_constraint_integs_marker.Append(
-         NULL); // NULL marker means apply everywhere
+      boundary_constraint_integs.push_back(c_integ);
+      boundary_constraint_integs_marker.push_back(bdr_marker);
    }
+
+   /// @brief Add the boundary face integrator that will be used to construct
+   /// the constraint matrix C.
+   ///
+   /// If @a own is true (its default value), then the Hybridization object
+   /// assumes ownership of the integrator.
+   ///
+   /// @sa AddBdrConstraintIntegrator().
    void AddBdrConstraintIntegrator(BilinearFormIntegrator *c_integ,
-                                   Array<int> &bdr_marker)
+                                   const Array<int> &bdr_marker = Array<int>(),
+                                   bool own = true)
    {
-      boundary_constraint_integs.Append(c_integ);
-      boundary_constraint_integs_marker.Append(&bdr_marker);
+      AddBdrConstraintIntegrator(OptionallyOwning(c_integ, own), bdr_marker);
    }
 
-   /// Access all integrators added with AddBdrConstraintIntegrator().
-   Array<BilinearFormIntegrator*> *GetBCBFI() { return &boundary_constraint_integs; }
+   /// Access the integrators added with AddBdrConstraintIntegrator().
+   const std::vector<std::shared_ptr<BilinearFormIntegrator>>
+   &GetBdrConstraintIntegrators() const { return boundary_constraint_integs; }
 
-   /// Access all boundary markers added with AddBdrConstraintIntegrator().
-   /** If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBCBFI_Marker() { return &boundary_constraint_integs_marker; }
-
-   /// Indicate that boundary constraint integrators are not owned
-   void UseExternalBdrConstraintIntegrators() { extern_bdr_constr_integs = true; }
+   /// Access the boundary markers added with AddBdrConstraintIntegrator().
+   const std::vector<Array<int>> &GetBdrConstraintIntegratorMarker() const
+   { return boundary_constraint_integs_marker; }
 
    /// Prepare the Hybridization object for assembly.
    void Init(const Array<int> &ess_tdof_list);
@@ -145,6 +185,9 @@ public:
    /// Assemble the element matrix A into the hybridized system matrix.
    void AssembleMatrix(int el, const DenseMatrix &A);
 
+   /// Assemble all of the element matrices given in the form of a DenseTensor.
+   void AssembleElementMatrices(const class DenseTensor &el_mats);
+
    /// Assemble the boundary element matrix A into the hybridized system matrix.
    void AssembleBdrMatrix(int bdr_el, const DenseMatrix &A);
 
@@ -158,30 +201,34 @@ public:
    /// Return the parallel hybridized matrix.
    HypreParMatrix &GetParallelMatrix() { return *pH.Is<HypreParMatrix>(); }
 
-   /** @brief Return the parallel hybridized matrix in the format specified by
-       SetOperatorType(). */
+   /// @brief Return the parallel hybridized matrix in the format specified by
+   /// SetOperatorType().
    void GetParallelMatrix(OperatorHandle &H_h) const { H_h = pH; }
 
    /// Set the operator type id for the parallel hybridized matrix/operator.
    void SetOperatorType(Operator::Type tid) { pH.SetType(tid); }
 #endif
 
-   /** Perform the reduction of the given r.h.s. vector, b, to a r.h.s vector,
-       b_r, for the hybridized system. */
+   /// @brief Perform the reduction of the given right-hand side @a b to a
+   /// right-hand side vector @a b_r for the hybridized system.
    void ReduceRHS(const Vector &b, Vector &b_r) const;
 
-   /** Reconstruct the solution of the original system, sol, from solution of
-       the hybridized system, sol_r, and the original r.h.s. vector, b.
-       It is assumed that the vector sol has the right essential b.c. */
+   /// @brief Reconstruct the solution of the original system @a sol from
+   /// solution of the hybridized system @a sol_r and the original right-hand
+   /// side @a b.
+   ///
+   /// It is assumed that the vector sol has the correct essential boundary
+   /// conditions.
    void ComputeSolution(const Vector &b, const Vector &sol_r,
                         Vector &sol) const;
 
-   /** @brief Destroy the current hybridization matrix while preserving the
-       computed constraint matrix and the set of essential true dofs. After
-       Reset(), a new hybridized matrix can be assembled via AssembleMatrix()
-       and Finalize(). The Mesh and FiniteElementSpace objects are assumed to be
-       un-modified. If that is not the case, a new Hybridization object must be
-       created. */
+   /// @brief Destroy the current hybridization matrix while preserving the
+   /// computed constraint matrix and the set of essential true dofs.
+   ///
+   /// After Reset(), a new hybridized matrix can be assembled via
+   /// AssembleMatrix() and Finalize(). The Mesh and FiniteElementSpace objects
+   /// are assumed to be unmodified. If that is not the case, a new
+   /// Hybridization object must be created.
    void Reset();
 };
 

fem/hybridization_ext.hpp

@@ -0,0 +1,119 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_HYBRIDIZATION_EXT
+#define MFEM_HYBRIDIZATION_EXT
+
+#include "../config/config.hpp"
+#include "../general/array.hpp"
+#include "../linalg/vector.hpp"
+
+namespace mfem
+{
+
+/// @brief Extension class supporting Hybridization on device (GPU).
+///
+/// Similar to BilinearFormExtension and LinearFormExtension, this extension
+/// class provides device execution capabilities for the Hybridization class.
+///
+/// As with the other extension classes, a limitation of this class is that it
+/// requires meshes consisting only of tensor-product elements, and finite
+/// element spaces without variable polynomial degrees.
+class HybridizationExtension
+{
+   friend class Hybridization;
+public:
+   enum DofType : char
+   {
+      ESSENTIAL,
+      BOUNDARY,
+      INTERIOR
+   };
+protected:
+   class Hybridization &h; ///< The associated Hybridization object.=
+   int num_hat_dofs; ///< Number of Lagrange multipliers.
+   mutable Vector tmp1, tmp2; ///< Temporary vectors.
+
+   Array<int> hat_dof_gather_map;
+   Array<DofType> hat_dof_marker;
+
+   Array<int> el_to_face;
+   Array<int> face_to_el;
+   Vector Ct_mat; ///< Constraint matrix (transposed) stored element-wise.
+
+   Array<int> idofs, bdofs;
+
+   Vector Ahat, Ahat_ii, Ahat_ib, Ahat_bi, Ahat_bb;
+   Array<int> Ahat_ii_piv, Ahat_bb_piv;
+
+public:
+   /// Construct the constraint matrix.
+   void ConstructC();
+
+   template <int MID, int MBD>
+   void FactorElementMatrices(Vector &AhatInvCt_mat);
+
+   /// Form the Schur complement matix $H$.
+   void ConstructH();
+
+   /// Compute the action of C^t x.
+   void MultCt(const Vector &x, Vector &y) const;
+
+   /// Compute the action of C x.
+   void MultC(const Vector &x, Vector &y) const;
+
+   /// @brief Assemble the element matrix A into the hybridized system matrix.
+   ///
+   /// @warning Using the interface will be very slow. AssembleElementMatrices()
+   /// should be used instead.
+   void AssembleMatrix(int el, const class DenseMatrix &elmat);
+
+   /// @brief Assemble the boundary element matrix A into the hybridized system
+   /// matrix.
+   ///
+   /// @warning Using the interface will be very slow. AssembleElementMatrices()
+   /// should be used instead.
+   void AssembleBdrMatrix(int bdr_el, const class DenseMatrix &elmat);
+
+   /// Invert and store the element matrices Ahat.
+   void AssembleElementMatrices(const class DenseTensor &el_mats);
+
+   /// Apply the action of R mapping from "hat DOFs" to T-vector
+   void MultR(const Vector &b, Vector &b_hat) const;
+
+   /// Apply the action of R^t mapping into the "hat DOF" space.
+   void MultRt(const Vector &b, Vector &b_hat) const;
+
+   /// Apply the elementwise A_hat^{-1}.
+   void MultAhatInv(Vector &x) const;
+
+   /// Constructor.
+   HybridizationExtension(class Hybridization &hybridization_);
+
+   /// Prepare for assembly; form the constraint matrix.
+   void Init(const Array<int> &ess_tdof_list);
+
+   /// @brief Given a right-hand side on the original space, compute the
+   /// corresponding right-hand side for the Lagrange multipliers.
+   void ReduceRHS(const Vector &b, Vector &b_r) const;
+
+   /// @brief Given Lagrange multipliers @a sol_r and the original right-hand
+   /// side @a b, recover the solution @a sol on the original finite element
+   /// space.
+   void ComputeSolution(const Vector &b, const Vector &sol_r, Vector &sol) const;
+
+   /// Destroys the stored element matrices.
+   void Reset() { Ahat = 0.0; }
+};
+
+}
+
+#endif

fem/integ/bilininteg_diffusion_kernels.cpp

@@ -16,7 +16,6 @@ namespace mfem
 
 // PA Diffusion Integrator
 
-DiffusionIntegrator::Kernels DiffusionIntegrator::kernels;
 DiffusionIntegrator::Kernels::Kernels()
 {
    // 2D

fem/integ/bilininteg_hdiv_ea.cpp

@@ -0,0 +1,336 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+
+namespace mfem
+{
+
+// For H(div) mass, Bo and Bc are the basis evaluation operators, and the
+// pa_data corresponds to a (potentially symmetric) matrix coefficient.
+// coeff_dim must be 3 or 4 depending on symmetry.
+//
+// For div-div, Bc is the derivative evaluation operator, and pa_data
+// corresponds to a scalar coefficient. coeff_dim must be 1.
+//
+// These two integrators are distinguished using coeff_dim.
+template<int T_D1D = 0, int T_Q1D = 0>
+static void EAHdivAssemble2D(const int NE,
+                             const Array<real_t> &Bo_,
+                             const Array<real_t> &Bc_,
+                             const int coeff_dim,
+                             const Vector &pa_data,
+                             Vector &ea_data,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D, "");
+   const int NDOF = 2*(D1D-1)*D1D;
+   const auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   const auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, coeff_dim, NE);
+   const bool symmetric = (coeff_dim == 3);
+   auto M = Reshape(add ? ea_data.ReadWrite() : ea_data.Write(), NDOF, NDOF, NE);
+   mfem::forall_2D(NE, NDOF, 1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
+      // Load Bo and Bc matrices into registers
+      real_t r_Bo[MQ1][MD1];
+      real_t r_Bc[MQ1][MD1];
+      for (int d = 0; d < D1D; d++)
+      {
+         for (int q = 0; q < Q1D; q++)
+         {
+            if (d < D1D - 1) { r_Bo[q][d] = Bo(q,d); }
+            r_Bc[q][d] = Bc(q,d);
+         }
+      }
+      // Store PA data in shared memory
+      MFEM_SHARED real_t s_D[4][MQ1][MQ1];
+      MFEM_FOREACH_THREAD(idx_q, x, Q1D*Q1D)
+      {
+         const int qx = idx_q % Q1D;
+         const int qy = idx_q / Q1D;
+         if (coeff_dim == 1)
+         {
+            const real_t val = D(qx, qy, 0, e);
+            for (int i = 0; i < 4; ++i) { s_D[i][qx][qy] = val; }
+         }
+         else
+         {
+            s_D[0][qx][qy] = D(qx, qy, 0, e);
+            s_D[1][qx][qy] = D(qx, qy, 1, e);
+            s_D[2][qx][qy] = (symmetric) ? s_D[1][qx][qy] : D(qx, qy, 2, e);
+            s_D[3][qx][qy] = (symmetric) ? D(qx, qy, 2, e) : D(qx, qy, 3, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Assemble (one row per thread)
+      MFEM_FOREACH_THREAD(idx_i, x, NDOF)
+      {
+         const int ic = idx_i / D1D / (D1D-1);
+         const int idx_ii = idx_i % (D1D * (D1D-1));
+         const int ix = (ic == 0) ? idx_ii%D1D : idx_ii%(D1D-1);
+         const int iy = (ic == 0) ? idx_ii/D1D : idx_ii/(D1D-1);
+
+         const real_t (&Bi1)[MQ1][MD1] = (ic == 0) ? r_Bc : r_Bo;
+         const real_t (&Bi2)[MQ1][MD1] = (ic == 0) ? r_Bo : r_Bc;
+
+         for (int idx_j = 0; idx_j < NDOF; ++idx_j)
+         {
+            const int jc = idx_j / (D1D*(D1D-1));
+            const int idx_jj = idx_j % (D1D * (D1D-1));
+            const int jx = (jc == 0) ? idx_jj%D1D : idx_jj%(D1D-1);
+            const int jy = (jc == 0) ? idx_jj/D1D : idx_jj/(D1D-1);
+
+            const real_t (&Bj1)[MQ1][MD1] = (jc == 0) ? r_Bc : r_Bo;
+            const real_t (&Bj2)[MQ1][MD1] = (jc == 0) ? r_Bo : r_Bc;
+
+            real_t val = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double coeff = s_D[ic + jc*2][qx][qy];
+                  val += coeff*Bi1[qx][ix]*Bi2[qy][iy]*Bj1[qx][jx]*Bj2[qy][jy];
+               }
+            }
+            if (add)
+            {
+               M(idx_i, idx_j, e) += val;
+            }
+            else
+            {
+               M(idx_i, idx_j, e) = val;
+            }
+         }
+      }
+   });
+}
+
+// For H(div) mass, Bo and Bc are the basis evaluation operators, and the
+// pa_data corresponds to a (potentially symmetric) matrix coefficient.
+// coeff_dim must be 6 or 9 depending on symmetry.
+//
+// For div-div, Bc is the derivative evaluation operator, and pa_data
+// corresponds to a scalar coefficient. coeff_dim must be 1.
+//
+// These two integrators are distinguished using coeff_dim.
+template<int T_D1D = 0, int T_Q1D = 0>
+static void EAHdivAssemble3D(const int NE,
+                             const Array<real_t> &Bo_,
+                             const Array<real_t> &Bc_,
+                             const int coeff_dim,
+                             const Vector &pa_data,
+                             Vector &ea_data,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D, "");
+   const int NDOF_C = (D1D-1)*(D1D-1)*D1D;
+   const int NDOF = 3*NDOF_C;
+   const auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   const auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, coeff_dim, NE);
+   const bool symmetric = (coeff_dim == 6);
+   auto M = Reshape(add ? ea_data.ReadWrite() : ea_data.Write(), NDOF, NDOF, NE);
+   mfem::forall_2D(NE, NDOF, 1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
+      // Load Bo and Bc matrices into registers
+      real_t r_Bo[MQ1][MD1];
+      real_t r_Bc[MQ1][MD1];
+      for (int d = 0; d < D1D; d++)
+      {
+         for (int q = 0; q < Q1D; q++)
+         {
+            if (d < D1D - 1) { r_Bo[q][d] = Bo(q,d); }
+            r_Bc[q][d] = Bc(q,d);
+         }
+      }
+      // Store PA data in shared memory
+      MFEM_SHARED real_t s_D[9][MQ1][MQ1][MQ1];
+      MFEM_FOREACH_THREAD(idx_q, x, Q1D*Q1D*Q1D)
+      {
+         const int qx = idx_q % Q1D;
+         const int qy = (idx_q / Q1D) % Q1D;
+         const int qz = (idx_q / Q1D) / Q1D;
+         if (coeff_dim == 1)
+         {
+            const real_t val = D(qx,qy,qz,0,e);
+            for (int i = 0; i < 9; ++i) { s_D[i][qx][qy][qz] = val; }
+         }
+         else
+         {
+            s_D[0][qx][qy][qz] = D(qx,qy,qz,0,e);
+            s_D[1][qx][qy][qz] = D(qx,qy,qz,1,e);
+            s_D[2][qx][qy][qz] = D(qx,qy,qz,2,e);
+            s_D[3][qx][qy][qz] = symmetric ? s_D[1][qx][qy][qz] : D(qx,qy,qz,3,e);
+            s_D[4][qx][qy][qz] = symmetric ? D(qx,qy,qz,3,e) : D(qx,qy,qz,4,e);
+            s_D[5][qx][qy][qz] = symmetric ? D(qx,qy,qz,4,e) : D(qx,qy,qz,5,e);
+            s_D[6][qx][qy][qz] = symmetric ? s_D[2][qx][qy][qz] : D(qx,qy,qz,6,e);
+            s_D[7][qx][qy][qz] = symmetric ? s_D[5][qx][qy][qz] : D(qx,qy,qz,7,e);
+            s_D[8][qx][qy][qz] = symmetric ? D(qx,qy,qz,5,e) : D(qx,qy,qz,8,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Assemble (one row per thread)
+      MFEM_FOREACH_THREAD(idx_i, x, NDOF)
+      {
+         const int ic = idx_i / NDOF_C;
+         const int idx_ii = idx_i % NDOF_C;
+
+         const int nx_i = (ic == 0) ? D1D : D1D-1;
+         const int ny_i = (ic == 1) ? D1D : D1D-1;
+
+         const int ix = idx_ii % nx_i;
+         const int iy = (idx_ii / nx_i) % ny_i;
+         const int iz = (idx_ii / nx_i) / ny_i;
+
+         const real_t (&Bi1)[MQ1][MD1] = (ic == 0) ? r_Bc : r_Bo;
+         const real_t (&Bi2)[MQ1][MD1] = (ic == 1) ? r_Bc : r_Bo;
+         const real_t (&Bi3)[MQ1][MD1] = (ic == 2) ? r_Bc : r_Bo;
+
+         for (int idx_j = 0; idx_j < NDOF; ++idx_j)
+         {
+            const int jc = idx_j / NDOF_C;
+            const int idx_jj = idx_j % NDOF_C;
+
+            const int nx_j = (jc == 0) ? D1D : D1D-1;
+            const int ny_j = (jc == 1) ? D1D : D1D-1;
+
+            const int jx = idx_jj % nx_j;
+            const int jy = (idx_jj / nx_j) % ny_j;
+            const int jz = (idx_jj / nx_j) / ny_j;
+
+            const real_t (&Bj1)[MQ1][MD1] = (jc == 0) ? r_Bc : r_Bo;
+            const real_t (&Bj2)[MQ1][MD1] = (jc == 1) ? r_Bc : r_Bo;
+            const real_t (&Bj3)[MQ1][MD1] = (jc == 2) ? r_Bc : r_Bo;
+
+            real_t val = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     const double coeff = s_D[ic + jc*3][qx][qy][qz];
+                     val += coeff*Bi1[qx][ix]*Bi2[qy][iy]*Bi3[qz][iz]*
+                            Bj1[qx][jx]*Bj2[qy][jy]*Bj3[qz][jz];
+                  }
+               }
+            }
+            if (add)
+            {
+               M(idx_i, idx_j, e) += val;
+            }
+            else
+            {
+               M(idx_i, idx_j, e) = val;
+            }
+         }
+      }
+   });
+}
+
+void VectorFEMassIntegrator::AssembleEA(const FiniteElementSpace &fes,
+                                        Vector &ea_data,
+                                        const bool add)
+{
+   AssemblePA(fes);
+
+   if (trial_fetype != mfem::FiniteElement::DIV ||
+       test_fetype != mfem::FiniteElement::DIV)
+   {
+      MFEM_ABORT("Unsupported kernel.");
+   }
+
+   const Array<real_t> &Bo = mapsO->B;
+   const Array<real_t> &Bc = mapsC->B;
+
+   if (dim == 2)
+   {
+      const int coeff_dim = symmetric ? 3 : 4;
+      auto kernel = EAHdivAssemble2D<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x22: kernel = EAHdivAssemble2D<2,2>; break;
+         case 0x33: kernel = EAHdivAssemble2D<3,3>; break;
+         case 0x44: kernel = EAHdivAssemble2D<4,4>; break;
+         case 0x55: kernel = EAHdivAssemble2D<5,5>; break;
+      }
+      return kernel(ne,Bo,Bc,coeff_dim,pa_data,ea_data,add,dofs1D,quad1D);
+   }
+   else if (dim == 3)
+   {
+      const int coeff_dim = symmetric ? 6 : 9;
+      auto kernel = EAHdivAssemble3D<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x23: kernel = EAHdivAssemble3D<2,3>; break;
+         case 0x34: kernel = EAHdivAssemble3D<3,4>; break;
+         case 0x45: kernel = EAHdivAssemble3D<4,5>; break;
+         case 0x56: kernel = EAHdivAssemble3D<5,6>; break;
+      }
+      return kernel(ne,Bo,Bc,coeff_dim,pa_data,ea_data,add,dofs1D,quad1D);
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void DivDivIntegrator::AssembleEA(const FiniteElementSpace &fes,
+                                  Vector &ea_data,
+                                  const bool add)
+{
+   AssemblePA(fes);
+
+   const Array<real_t> &Bo = mapsO->B;
+   const Array<real_t> &Gc = mapsC->G;
+
+   if (dim == 2)
+   {
+      auto kernel = EAHdivAssemble2D<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x22: kernel = EAHdivAssemble2D<2,2>; break;
+         case 0x33: kernel = EAHdivAssemble2D<3,3>; break;
+         case 0x44: kernel = EAHdivAssemble2D<4,4>; break;
+         case 0x55: kernel = EAHdivAssemble2D<5,5>; break;
+      }
+      return kernel(ne,Bo,Gc,1,pa_data,ea_data,add,dofs1D,quad1D);
+   }
+   else if (dim == 3)
+   {
+      auto kernel = EAHdivAssemble3D<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x23: kernel = EAHdivAssemble3D<2,3>; break;
+         case 0x34: kernel = EAHdivAssemble3D<3,4>; break;
+         case 0x45: kernel = EAHdivAssemble3D<4,5>; break;
+         case 0x56: kernel = EAHdivAssemble3D<5,6>; break;
+      }
+      return kernel(ne,Bo,Gc,1,pa_data,ea_data,add,dofs1D,quad1D);
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+}

fem/integ/bilininteg_interp_pa.cpp

@@ -1819,10 +1819,12 @@ void IdentityInterpolator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    MFEM_VERIFY(trial_el->GetOrder() == test_el->GetOrder(), "");
 
+   MFEM_VERIFY(vdim == 1, "vdim != 1 with PA is not supported yet!");
+
    ne = trial_fes.GetNE();
 
    const int order = trial_el->GetOrder();
-   dofquad_fe = new H1_SegmentElement(order);
+   dofquad_fe.reset(new H1_SegmentElement(order));
    mfem::QuadratureFunctions1D qf1d;
    mfem::IntegrationRule closed_ir;
    closed_ir.SetSize(order + 1);

fem/integ/bilininteg_mass_ea.cpp

@@ -12,248 +12,67 @@
 #include "../../general/forall.hpp"
 #include "../bilininteg.hpp"
 #include "../gridfunc.hpp"
+#include "bilininteg_mass_kernels.hpp"
 
 namespace mfem
 {
 
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble1D(const int NE,
-                             const Array<real_t> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const bool add,
-                             const int d1d = 0,
-                             const int q1d = 0)
+void MassIntegrator::AssembleEA_(Vector &ea_data,
+                                 const bool add)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(), D1D, D1D, NE);
-   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   using internal::EAMassAssemble1D;
+   using internal::EAMassAssemble2D;
+   using internal::EAMassAssemble3D;
+
+   const Array<real_t> &B = maps->B;
+   if (dim == 1)
    {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t r_Bi[MQ1];
-      real_t r_Bj[MQ1];
-      for (int q = 0; q < Q1D; q++)
+      auto kernel = EAMassAssemble1D<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
       {
-         r_Bi[q] = B(q,MFEM_THREAD_ID(x));
-         r_Bj[q] = B(q,MFEM_THREAD_ID(y));
+         case 0x22: kernel = EAMassAssemble1D<2,2>; break;
+         case 0x33: kernel = EAMassAssemble1D<3,3>; break;
+         case 0x44: kernel = EAMassAssemble1D<4,4>; break;
+         case 0x55: kernel = EAMassAssemble1D<5,5>; break;
+         case 0x66: kernel = EAMassAssemble1D<6,6>; break;
+         case 0x77: kernel = EAMassAssemble1D<7,7>; break;
+         case 0x88: kernel = EAMassAssemble1D<8,8>; break;
+         case 0x99: kernel = EAMassAssemble1D<9,9>; break;
       }
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(j1,y,D1D)
-         {
-            real_t val = 0.0;
-            for (int k1 = 0; k1 < Q1D; ++k1)
-            {
-               val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
-            }
-            if (add)
-            {
-               M(i1, j1, e) += val;
-            }
-            else
-            {
-               M(i1, j1, e) = val;
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble2D(const int NE,
-                             const Array<real_t> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const bool add,
-                             const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
-   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(), D1D, D1D, D1D, D1D,
-                    NE);
-   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+      return kernel(ne,B,pa_data,ea_data,add,dofs1D,quad1D);
+   }
+   else if (dim == 2)
    {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t r_B[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
+      auto kernel = EAMassAssemble2D<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
       {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-         }
+         case 0x22: kernel = EAMassAssemble2D<2,2>; break;
+         case 0x33: kernel = EAMassAssemble2D<3,3>; break;
+         case 0x44: kernel = EAMassAssemble2D<4,4>; break;
+         case 0x55: kernel = EAMassAssemble2D<5,5>; break;
+         case 0x66: kernel = EAMassAssemble2D<6,6>; break;
+         case 0x77: kernel = EAMassAssemble2D<7,7>; break;
+         case 0x88: kernel = EAMassAssemble2D<8,8>; break;
+         case 0x99: kernel = EAMassAssemble2D<9,9>; break;
       }
-      MFEM_SHARED real_t s_D[MQ1][MQ1];
-      MFEM_FOREACH_THREAD(k1,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(k2,y,Q1D)
-         {
-            s_D[k1][k2] = D(k1,k2,e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            for (int j1 = 0; j1 < D1D; ++j1)
-            {
-               for (int j2 = 0; j2 < D1D; ++j2)
-               {
-                  real_t val = 0.0;
-                  for (int k1 = 0; k1 < Q1D; ++k1)
-                  {
-                     for (int k2 = 0; k2 < Q1D; ++k2)
-                     {
-                        val += r_B[k1][i1] * r_B[k1][j1]
-                               * r_B[k2][i2] * r_B[k2][j2]
-                               * s_D[k1][k2];
-                     }
-                  }
-                  if (add)
-                  {
-                     M(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     M(i1, i2, j1, j2, e) = val;
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble3D(const int NE,
-                             const Array<real_t> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const bool add,
-                             const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
-   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(), D1D, D1D, D1D, D1D,
-                    D1D, D1D, NE);
-   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+      return kernel(ne,B,pa_data,ea_data,add,dofs1D,quad1D);
+   }
+   else if (dim == 3)
    {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int DQ = T_D1D * T_Q1D;
-
-      // For quadratic and lower it's better to use registers but for higher-order you start to
-      // spill and it's better to use shared memory
-      constexpr bool USE_REG = DQ != 0 && DQ <= 12;
-      constexpr int MD1r = USE_REG ? MD1 : 1;
-      constexpr int MQ1r = USE_REG ? MQ1 : 1;
-      constexpr int MD1s = USE_REG ? 1 : MD1;
-      constexpr int MQ1s = USE_REG ? 1 : MQ1;
-
-      MFEM_SHARED real_t s_B[MQ1s][MD1s];
-      real_t r_B[MQ1r][MD1r];
-      real_t (*l_B)[MD1] = nullptr;
-      if (USE_REG)
+      auto kernel = EAMassAssemble3D<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
       {
-         for (int d = 0; d < D1D; d++)
-         {
-            for (int q = 0; q < Q1D; q++)
-            {
-               r_B[q][d] = B(q,d);
-            }
-         }
-         l_B = (real_t (*)[MD1])r_B;
+         case 0x23: kernel = EAMassAssemble3D<2,3>; break;
+         case 0x34: kernel = EAMassAssemble3D<3,4>; break;
+         case 0x45: kernel = EAMassAssemble3D<4,5>; break;
+         case 0x56: kernel = EAMassAssemble3D<5,6>; break;
+         case 0x67: kernel = EAMassAssemble3D<6,7>; break;
+         case 0x78: kernel = EAMassAssemble3D<7,8>; break;
+         case 0x89: kernel = EAMassAssemble3D<8,9>; break;
       }
-      else
-      {
-         if (MFEM_THREAD_ID(z) == 0)
-         {
-            MFEM_FOREACH_THREAD(d,x,D1D)
-            {
-               MFEM_FOREACH_THREAD(q,y,Q1D)
-               {
-                  s_B[q][d] = B(q,d);
-               }
-            }
-         }
-         l_B = (real_t (*)[MD1])s_B;
-      }
-
-      MFEM_SHARED real_t s_D[MQ1][MQ1][MQ1];
-      MFEM_FOREACH_THREAD(k1,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(k2,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(k3,z,Q1D)
-            {
-               s_D[k1][k2][k3] = D(k1,k2,k3,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(i3,z,D1D)
-            {
-               for (int j1 = 0; j1 < D1D; ++j1)
-               {
-                  for (int j2 = 0; j2 < D1D; ++j2)
-                  {
-                     for (int j3 = 0; j3 < D1D; ++j3)
-                     {
-                        real_t val = 0.0;
-                        for (int k1 = 0; k1 < Q1D; ++k1)
-                        {
-                           for (int k2 = 0; k2 < Q1D; ++k2)
-                           {
-                              for (int k3 = 0; k3 < Q1D; ++k3)
-                              {
-                                 val += l_B[k1][i1] * l_B[k1][j1]
-                                        * l_B[k2][i2] * l_B[k2][j2]
-                                        * l_B[k3][i3] * l_B[k3][j3]
-                                        * s_D[k1][k2][k3];
-                              }
-                           }
-                        }
-                        if (add)
-                        {
-                           M(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           M(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
-                     }
-                  }
-               }
-            }
-         }
-      }
-   });
+      return kernel(ne,B,pa_data,ea_data,add,dofs1D,quad1D);
+   }
+   MFEM_ABORT("Unknown kernel.");
 }
 
 void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
@@ -261,56 +80,15 @@ void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
                                 const bool add)
 {
    AssemblePA(fes);
-   ne = fes.GetMesh()->GetNE();
-   const Array<real_t> &B = maps->B;
-   if (dim == 1)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data,add);
-         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data,add);
-         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data,add);
-         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data,add);
-         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data,add);
-         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data,add);
-         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data,add);
-         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
-      }
-   }
-   else if (dim == 2)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data,add);
-         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data,add);
-         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data,add);
-         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data,add);
-         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data,add);
-         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data,add);
-         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data,add);
-         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data,add);
-         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data,add);
-         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data,add);
-         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data,add);
-         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data,add);
-         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data,add);
-         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
+   if (ne > 0) { AssembleEA_(ea_data, add); }
+}
+
+void MassIntegrator::AssembleEABoundary(const FiniteElementSpace &fes,
+                                        Vector &ea_data,
+                                        const bool add)
+{
+   AssemblePABoundary(fes);
+   if (ne > 0) { AssembleEA_(ea_data, add); }
 }
 
 }

fem/integ/bilininteg_mass_kernels.cpp

@@ -14,7 +14,6 @@
 namespace mfem
 {
 
-MassIntegrator::Kernels MassIntegrator::kernels;
 MassIntegrator::Kernels::Kernels()
 {
    // 2D

fem/integ/bilininteg_mass_kernels.hpp

@@ -1132,6 +1132,244 @@ inline void SmemPAMassApply3D(const int NE,
    });
 }
 
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssemble1D(const int NE,
+                             const Array<real_t> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(), D1D, D1D, NE);
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      MFEM_FOREACH_THREAD(i1,x,D1D)
+      {
+         real_t r_Bi[MQ1];
+         for (int q = 0; q < Q1D; q++) { r_Bi[q] = B(q,i1); }
+         MFEM_FOREACH_THREAD(j1,y,D1D)
+         {
+            real_t r_Bj[MQ1];
+            for (int q = 0; q < Q1D; q++) { r_Bj[q] = B(q,j1); }
+
+            real_t val = 0.0;
+            for (int k1 = 0; k1 < Q1D; ++k1)
+            {
+               val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
+            }
+            if (add)
+            {
+               M(i1, j1, e) += val;
+            }
+            else
+            {
+               M(i1, j1, e) = val;
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssemble2D(const int NE,
+                             const Array<real_t> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(), D1D, D1D, D1D, D1D,
+                    NE);
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      real_t r_B[MQ1][MD1];
+      for (int d = 0; d < D1D; d++)
+      {
+         for (int q = 0; q < Q1D; q++)
+         {
+            r_B[q][d] = B(q,d);
+         }
+      }
+      MFEM_SHARED real_t s_D[MQ1][MQ1];
+      MFEM_FOREACH_THREAD(k1,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(k2,y,Q1D)
+         {
+            s_D[k1][k2] = D(k1,k2,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(i1,x,D1D)
+      {
+         MFEM_FOREACH_THREAD(i2,y,D1D)
+         {
+            for (int j1 = 0; j1 < D1D; ++j1)
+            {
+               for (int j2 = 0; j2 < D1D; ++j2)
+               {
+                  real_t val = 0.0;
+                  for (int k1 = 0; k1 < Q1D; ++k1)
+                  {
+                     for (int k2 = 0; k2 < Q1D; ++k2)
+                     {
+                        val += r_B[k1][i1] * r_B[k1][j1]
+                               * r_B[k2][i2] * r_B[k2][j2]
+                               * s_D[k1][k2];
+                     }
+                  }
+                  if (add)
+                  {
+                     M(i1, i2, j1, j2, e) += val;
+                  }
+                  else
+                  {
+                     M(i1, i2, j1, j2, e) = val;
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssemble3D(const int NE,
+                             const Array<real_t> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(), D1D, D1D, D1D, D1D,
+                    D1D, D1D, NE);
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int DQ = T_D1D * T_Q1D;
+
+      // For quadratic and lower it's better to use registers but for higher-order you start to
+      // spill and it's better to use shared memory
+      constexpr bool USE_REG = DQ != 0 && DQ <= 12;
+      constexpr int MD1r = USE_REG ? MD1 : 1;
+      constexpr int MQ1r = USE_REG ? MQ1 : 1;
+      constexpr int MD1s = USE_REG ? 1 : MD1;
+      constexpr int MQ1s = USE_REG ? 1 : MQ1;
+
+      MFEM_SHARED real_t s_B[MQ1s][MD1s];
+      real_t r_B[MQ1r][MD1r];
+      real_t (*l_B)[MD1] = nullptr;
+      if (USE_REG)
+      {
+         for (int d = 0; d < D1D; d++)
+         {
+            for (int q = 0; q < Q1D; q++)
+            {
+               r_B[q][d] = B(q,d);
+            }
+         }
+         l_B = (real_t (*)[MD1])r_B;
+      }
+      else
+      {
+         if (MFEM_THREAD_ID(z) == 0)
+         {
+            MFEM_FOREACH_THREAD(d,x,D1D)
+            {
+               MFEM_FOREACH_THREAD(q,y,Q1D)
+               {
+                  s_B[q][d] = B(q,d);
+               }
+            }
+         }
+         l_B = (real_t (*)[MD1])s_B;
+      }
+
+      MFEM_SHARED real_t s_D[MQ1][MQ1][MQ1];
+      MFEM_FOREACH_THREAD(k1,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(k2,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(k3,z,Q1D)
+            {
+               s_D[k1][k2][k3] = D(k1,k2,k3,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(i1,x,D1D)
+      {
+         MFEM_FOREACH_THREAD(i2,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(i3,z,D1D)
+            {
+               for (int j1 = 0; j1 < D1D; ++j1)
+               {
+                  for (int j2 = 0; j2 < D1D; ++j2)
+                  {
+                     for (int j3 = 0; j3 < D1D; ++j3)
+                     {
+                        real_t val = 0.0;
+                        for (int k1 = 0; k1 < Q1D; ++k1)
+                        {
+                           for (int k2 = 0; k2 < Q1D; ++k2)
+                           {
+                              for (int k3 = 0; k3 < Q1D; ++k3)
+                              {
+                                 val += l_B[k1][i1] * l_B[k1][j1]
+                                        * l_B[k2][i2] * l_B[k2][j2]
+                                        * l_B[k3][i3] * l_B[k3][j3]
+                                        * s_D[k1][k2][k3];
+                              }
+                           }
+                        }
+                        if (add)
+                        {
+                           M(i1, i2, i3, j1, j2, j3, e) += val;
+                        }
+                        else
+                        {
+                           M(i1, i2, i3, j1, j2, j3, e) = val;
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
 } // namespace internal
 
 namespace

fem/integ/bilininteg_mass_pa.cpp

@@ -29,7 +29,8 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    // Assuming the same element type
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
+   ne = fes.GetMesh()->GetNE();
+   if (ne == 0) { return; }
    const FiniteElement &el = *fes.GetFE(0);
    ElementTransformation *T0 = mesh->GetElementTransformation(0);
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
@@ -50,7 +51,6 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    }
    int map_type = el.GetMapType();
    dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
    nq = ir->GetNPoints();
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS, mt);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -123,14 +123,14 @@ void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
    // Assuming the same element type
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNBE() == 0) { return; }
+   ne = mesh->GetNFbyType(FaceType::Boundary);
+   if (ne == 0) { return; }
    const FiniteElement &el = *fes.GetBE(0);
    ElementTransformation *T0 = mesh->GetBdrElementTransformation(0);
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
 
    int map_type = el.GetMapType();
    dim = el.GetDim(); // Dimension of the boundary element, *not* the mesh
-   ne = fes.GetMesh()->GetNFbyType(FaceType::Boundary);
    nq = ir->GetNPoints();
    face_geom = mesh->GetFaceGeometricFactors(*ir, GeometricFactors::DETERMINANTS,
                                              FaceType::Boundary, mt);

fem/integ/bilininteg_trace_jump_ea.cpp

@@ -0,0 +1,158 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../../general/forall.hpp"
+#include "../fe/face_map_utils.hpp"
+#include "../bilininteg.hpp"
+#include "bilininteg_mass_kernels.hpp"
+
+namespace mfem
+{
+
+void NormalTraceJumpIntegrator::AssembleEAInteriorFaces(
+   const FiniteElementSpace &trial_fes,
+   const FiniteElementSpace &test_fes,
+   Vector &emat,
+   const bool add)
+{
+   Mesh &mesh = *trial_fes.GetMesh();
+   const int dim = mesh.Dimension();
+   const FaceType ftype = FaceType::Interior;
+   const int nf = mesh.GetNFbyType(ftype);
+
+   const Geometry::Type geom = mesh.GetFaceGeometry(0);
+   const int trial_order = trial_fes.GetMaxElementOrder();
+   const int test_order = test_fes.GetMaxElementOrder();
+   const int qorder = test_order + trial_order - 1;
+   const IntegrationRule &ir = IntRule ? *IntRule : IntRules.Get(geom, qorder);
+   const int nquad = ir.Size();
+
+   Vector pa_data(nquad * nf);
+   {
+      const auto d_w = ir.GetWeights().Read();
+      auto d_pa_data = Reshape(pa_data.Write(), nquad, nf);
+      mfem::forall(nquad * nf, [=] MFEM_HOST_DEVICE (int idx)
+      {
+         const int q = idx % nquad;
+         const int f = idx / nquad;
+         d_pa_data(q, f) = d_w[q];
+      });
+   }
+
+   const FiniteElement &trial_face_el = *trial_fes.GetFaceElement(0);
+   const auto maps = &trial_face_el.GetDofToQuad(ir, DofToQuad::TENSOR);
+   const int ndof_face = trial_face_el.GetDof();
+
+   const Array<real_t> &B = maps->B;
+   const int d1d = maps->ndof;
+   const int q1d = maps->nqpt;
+
+   Vector mass_emat(ndof_face*ndof_face*nf);
+
+   // Note: dim is the element dimension, and we integrate over the faces (one
+   // dimension less)
+   if (dim == 2)
+   {
+      internal::EAMassAssemble1D(nf, B, pa_data, mass_emat, false, d1d, q1d);
+   }
+   else if (dim == 3)
+   {
+      internal::EAMassAssemble2D(nf, B, pa_data, mass_emat, false, d1d, q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unknown kernel.");
+   }
+
+   const FiniteElement &test_el = *test_fes.GetFE(0);
+   const int n_faces_per_el = 2*dim; // assuming tensor product
+   // Get all the local face maps (mapping from lexicographic face index to
+   // lexicographic volume index, depending on the local face index).
+   Array<int> face_maps(ndof_face * n_faces_per_el);
+   for (int lf_i = 0; lf_i < n_faces_per_el; ++lf_i)
+   {
+      Array<int> face_map(ndof_face);
+      test_el.GetFaceMap(lf_i, face_map);
+      for (int i = 0; i < ndof_face; ++i)
+      {
+         face_maps[i + lf_i*ndof_face] = face_map[i];
+      }
+   }
+
+   Array<int> face_info(nf * 4);
+   {
+      int fidx = 0;
+      for (int f = 0; f < mesh.GetNumFaces(); ++f)
+      {
+         Mesh::FaceInformation finfo = mesh.GetFaceInformation(f);
+         if (!finfo.IsInterior()) { continue; }
+         face_info[0 + fidx*4] = finfo.element[0].local_face_id;
+         face_info[1 + fidx*4] = finfo.element[0].orientation;
+         face_info[2 + fidx*4] = finfo.element[1].local_face_id;
+         face_info[3 + fidx*4] = finfo.element[1].orientation;
+         fidx++;
+      }
+   }
+
+   const int ndof_vol = test_el.GetDof();
+   const auto d_face_maps = Reshape(face_maps.Read(), ndof_face, n_faces_per_el);
+   const auto d_face_info = Reshape(face_info.Read(), 2, 2, nf);
+
+   real_t *d_emat;
+   if (add)
+   {
+      d_emat = emat.ReadWrite();
+   }
+   if (!add)
+   {
+      d_emat = emat.Write();
+      mfem::forall(emat.Size(), [=] MFEM_HOST_DEVICE (int i) { d_emat[i] = 0.0; });
+   }
+
+   const auto face_mats = Reshape(mass_emat.Read(), ndof_face, ndof_face, nf);
+   auto el_mats = Reshape(d_emat, ndof_vol, ndof_face, 2, nf);
+
+   auto permute_face = [=] MFEM_HOST_DEVICE(int local_face_id, int orient,
+                                            int size1d, int index)
+   {
+      if (dim == 2)
+      {
+         return internal::PermuteFace2D(local_face_id, orient, size1d, index);
+      }
+      else // dim == 3
+      {
+         return internal::PermuteFace3D(local_face_id, orient, size1d, index);
+      }
+   };
+
+   mfem::forall_3D(nf, ndof_face, ndof_face, 2, [=] MFEM_HOST_DEVICE (int f)
+   {
+      MFEM_FOREACH_THREAD(el_i, z, 2)
+      {
+         const int lf_i = d_face_info(0, el_i, f);
+         const int orient = d_face_info(1, el_i, f);
+         // Loop over face indices in "native ordering"
+         MFEM_FOREACH_THREAD(i_lex, x, ndof_face)
+         {
+            // Convert to lexicographic relative to the face itself
+            const int i_face = permute_face(lf_i, orient, d1d, i_lex);
+            // Convert from lexicographic face DOF to volume DOF
+            const int i = d_face_maps(i_lex, lf_i);
+            MFEM_FOREACH_THREAD(j, y, ndof_face)
+            {
+               el_mats(i, j, el_i, f) += face_mats(i_face, j, f);
+            }
+         }
+      }
+   });
+}
+
+}

fem/intrules_cut.cpp

@@ -31,6 +31,172 @@ void CutIntegrationRules::SetLevelSetProjectionOrder(int order)
    lsOrder = order;
 }
 
+#ifdef MFEM_USE_ALGOIM
+void AlgoimIntegrationRules::GetSurfaceIntegrationRule(ElementTransformation
+                                                       &Tr,
+                                                       IntegrationRule &result)
+{
+   GenerateLSVector(Tr,LvlSet);
+
+   const int dim=pe->GetDim();
+   int np1d=CutIntegrationRules::Order/2+1;
+   if (dim==2)
+   {
+      LevelSet2D ls(pe,lsvec);
+      auto q = Algoim::quadGen<2>(ls,Algoim::BoundingBox<real_t,2>(0.0,1.0),
+                                  2, -1, np1d);
+      result.SetSize(q.nodes.size());
+      result.SetOrder(CutIntegrationRules::Order);
+      for (size_t i=0; i<q.nodes.size(); i++)
+      {
+         IntegrationPoint& ip=result.IntPoint(i);
+         ip.Set2w(q.nodes[i].x(0),q.nodes[i].x(1),q.nodes[i].w);
+      }
+   }
+   else
+   {
+      LevelSet3D ls(pe,lsvec);
+      auto q = Algoim::quadGen<3>(ls,Algoim::BoundingBox<real_t,3>(0.0,1.0),
+                                  3, -1, np1d);
+
+      result.SetSize(q.nodes.size());
+      result.SetOrder(CutIntegrationRules::Order);
+      for (size_t i=0; i<q.nodes.size(); i++)
+      {
+         IntegrationPoint& ip=result.IntPoint(i);
+         ip.Set(q.nodes[i].x(0),q.nodes[i].x(1),q.nodes[i].x(2),q.nodes[i].w);
+      }
+   }
+
+}
+
+void AlgoimIntegrationRules::GetVolumeIntegrationRule(ElementTransformation &Tr,
+                                                      IntegrationRule &result,
+                                                      const IntegrationRule *sir)
+{
+   GenerateLSVector(Tr,LvlSet);
+
+   const int dim=pe->GetDim();
+   int np1d=CutIntegrationRules::Order/2+1;
+   if (dim==2)
+   {
+      LevelSet2D ls(pe,lsvec);
+      auto q = Algoim::quadGen<2>(ls,Algoim::BoundingBox<real_t,2>(0.0,1.0),
+                                  -1, -1, np1d);
+      result.SetSize(q.nodes.size());
+      result.SetOrder(CutIntegrationRules::Order);
+      for (size_t i=0; i<q.nodes.size(); i++)
+      {
+         IntegrationPoint& ip=result.IntPoint(i);
+         ip.Set2w(q.nodes[i].x(0),q.nodes[i].x(1),q.nodes[i].w);
+      }
+   }
+   else
+   {
+      LevelSet3D ls(pe,lsvec);
+      auto q = Algoim::quadGen<3>(ls,Algoim::BoundingBox<real_t,3>(0.0,1.0),
+                                  -1, -1, np1d);
+
+      result.SetSize(q.nodes.size());
+      result.SetOrder(CutIntegrationRules::Order);
+      for (size_t i=0; i<q.nodes.size(); i++)
+      {
+         IntegrationPoint& ip=result.IntPoint(i);
+         ip.Set(q.nodes[i].x(0),q.nodes[i].x(1),q.nodes[i].x(2),q.nodes[i].w);
+      }
+   }
+
+}
+
+void AlgoimIntegrationRules::GetSurfaceWeights(ElementTransformation &Tr,
+                                               const IntegrationRule &sir,
+                                               Vector &weights)
+{
+   GenerateLSVector(Tr,LvlSet);
+
+   DenseMatrix bmat; // gradients of the shape functions in isoparametric space
+   DenseMatrix pmat; // gradients of the shape functions in physical space
+   Vector inormal; // normal to the level set in isoparametric space
+   Vector tnormal; // normal to the level set in physical space
+   bmat.SetSize(pe->GetDof(),pe->GetDim());
+   pmat.SetSize(pe->GetDof(),pe->GetDim());
+   inormal.SetSize(pe->GetDim());
+   tnormal.SetSize(pe->GetDim());
+
+   weights.SetSize(sir.GetNPoints());
+
+   for (int j = 0; j < sir.GetNPoints(); j++)
+   {
+      const IntegrationPoint &ip = sir.IntPoint(j);
+      Tr.SetIntPoint(&ip);
+      pe->CalcDShape(ip,bmat);
+      Mult(bmat, Tr.InverseJacobian(), pmat);
+      // compute the normal to the LS in isoparametric space
+      bmat.MultTranspose(lsvec,inormal);
+      // compute the normal to the LS in physical space
+      pmat.MultTranspose(lsvec,tnormal);
+      weights[j]= tnormal.Norml2() / inormal.Norml2();
+   }
+
+}
+
+void AlgoimIntegrationRules::GenerateLSVector(ElementTransformation &Tr,
+                                              Coefficient* lvlset)
+{
+   //check if the coefficient is already projected
+   if (currentElementNo==Tr.ElementNo)
+   {
+      if (currentLvlSet==lvlset)
+      {
+         if (currentGeometry==Tr.GetGeometryType())
+         {
+            return;
+         }
+      }
+   }
+
+   currentElementNo=Tr.ElementNo;
+
+   if (currentGeometry!=Tr.GetGeometryType())
+   {
+      delete le;
+      delete pe;
+      currentGeometry=Tr.GetGeometryType();
+      if (Tr.GetGeometryType()==Geometry::Type::SQUARE)
+      {
+         pe=new H1Pos_QuadrilateralElement(lsOrder);
+         le=new H1_QuadrilateralElement(lsOrder);
+      }
+      else if (Tr.GetGeometryType()==Geometry::Type::CUBE)
+      {
+         pe=new H1Pos_HexahedronElement(lsOrder);
+         le=new H1_HexahedronElement(lsOrder);
+      }
+      else
+      {
+         MFEM_ABORT("Currently MFEM + Algoim supports only quads and hexes.");
+      }
+
+      T.SetSize(pe->GetDof());
+      pe->Project(*le,Tr,T);
+      //The transformation matrix depends only on the geometry for change of basis
+   }
+
+   currentLvlSet=lvlset;
+   const IntegrationRule &ir=le->GetNodes();
+   lsvec.SetSize(ir.GetNPoints());
+   lsfun.SetSize(ir.GetNPoints());
+   for (int i=0; i<ir.GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(i);
+      Tr.SetIntPoint(&ip);
+      lsfun(i)=lvlset->Eval(Tr,ip);
+   }
+   T.Mult(lsfun,lsvec);
+}
+
+#endif
+
 #ifdef MFEM_USE_LAPACK
 
 void MomentFittingIntRules::InitSurface(int order, Coefficient& levelset,
@@ -175,6 +341,7 @@ void MomentFittingIntRules::ComputeFaceWeights(ElementTransformation& Tr)
          local_mesh.GetElementTransformation(0, &faceTrafo);
 
          // The 3D face integrals are computed as 2D volumetric integrals.
+         // The 2D face integrals are computed as 1D volumetric integrals.
          MomentFittingIntRules FaceRules(Order, *LvlSet, lsOrder);
          IntegrationRule FaceRule;
          FaceRules.GetVolumeIntegrationRule(faceTrafo, FaceRule);
@@ -254,8 +421,56 @@ void MomentFittingIntRules::ComputeSurfaceWeights1D(ElementTransformation& Tr)
    }
 }
 
-void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr,
-                                                   const IntegrationRule* sir)
+double bisect(ElementTransformation &Tr, Coefficient *LvlSet)
+{
+   IntegrationPoint intp;
+
+   IntegrationPoint ip0;
+   ip0.x = 0.;
+   IntegrationPoint ip1;
+   ip1.x = 1.;
+   Tr.SetIntPoint(&ip0);
+   if (LvlSet->Eval(Tr, ip0) * LvlSet->Eval(Tr, ip1) < 0.)
+   {
+      IntegrationPoint ip2;
+      ip2.x = .5;
+      while (LvlSet->Eval(Tr, ip2) > 1e-12
+             || LvlSet->Eval(Tr, ip2) < -1e-12)
+      {
+         if (LvlSet->Eval(Tr, ip0) * LvlSet->Eval(Tr, ip2) < 0.)
+         {
+            ip1.x = ip2.x;
+         }
+         else
+         {
+            ip0.x = ip2.x;
+         }
+
+         ip2.x = (ip1.x + ip0.x) / 2.;
+      }
+      intp.x = ip2.x;
+      intp.weight = 1. / Tr.Weight();
+   }
+   else if (LvlSet->Eval(Tr, ip0) > 0. && LvlSet->Eval(Tr, ip1) <= 1e-12)
+   {
+      intp.x = 1.;
+      intp.weight = 1. / Tr.Weight();
+   }
+   else if (LvlSet->Eval(Tr, ip1) > 0. && LvlSet->Eval(Tr, ip0) <= 1e-12)
+   {
+      intp.x = 0.;
+      intp.weight = 1. / Tr.Weight();
+   }
+   else
+   {
+      intp.x = .5;
+      intp.weight = 0.;
+   }
+
+   return intp.x;
+}
+
+void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr)
 {
    IntegrationRules irs(0, Quadrature1D::GaussLegendre);
    IntegrationRule ir2 = irs.Get(Geometry::SEGMENT, ir.GetOrder());
@@ -271,7 +486,7 @@ void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr,
       real_t length;
       if (LvlSet->Eval(Tr, ip0) > 0.)
       {
-         length = sir->IntPoint(0).x;
+         length = bisect(Tr, LvlSet);
          for (int ip = 0; ip < ir.GetNPoints(); ip++)
          {
             IntegrationPoint &intp = ir.IntPoint(ip);
@@ -281,11 +496,11 @@ void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr,
       }
       else
       {
-         length = 1. - sir->IntPoint(0).x;
+         length = 1. - bisect(Tr, LvlSet);
          for (int ip = 0; ip < ir.GetNPoints(); ip++)
          {
             IntegrationPoint &intp = ir.IntPoint(ip);
-            intp.x = sir->IntPoint(ip).x + ir2.IntPoint(ip).x * length;
+            intp.x = bisect(Tr, LvlSet) + ir2.IntPoint(ip).x * length;
             intp.weight = ir2.IntPoint(ip).weight * length;
          }
       }
@@ -1491,26 +1706,29 @@ void MomentFittingIntRules::GetVolumeIntegrationRule(ElementTransformation& Tr,
    }
 
    IntegrationRule SIR;
-   if (sir == NULL)
-   {
-      Order++;
-      GetSurfaceIntegrationRule(Tr, SIR);
-      Order--;
-   }
-   else if ((sir->GetOrder() - 1) != ir.GetOrder())
-   {
-      Order++;
-      GetSurfaceIntegrationRule(Tr, SIR);
-      Order--;
-   }
-   else
-   {
-      SIR = *sir;
-   }
 
    if (Tr.GetDimension() == 1)
    {
-      ComputeVolumeWeights1D(Tr, &SIR);
+      Clear();
+      InitVolume(Order, *LvlSet, lsOrder, Tr);
+   }
+   else if (sir == NULL)
+   {
+      Order++;
+      GetSurfaceIntegrationRule(Tr, SIR);
+      Order--;
+   }
+   else if (sir->GetOrder() - 1 != ir.GetOrder())
+   {
+      Order++;
+      GetSurfaceIntegrationRule(Tr, SIR);
+      Order--;
+   }
+   else { SIR = *sir; }
+
+   if (Tr.GetDimension() == 1)
+   {
+      ComputeVolumeWeights1D(Tr);
    }
    else if (Tr.GetDimension() == 2)
    {

fem/intrules_cut.hpp

@@ -18,6 +18,16 @@
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 
+
+#ifdef MFEM_USE_ALGOIM
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+#include <algoim_quad.hpp>
+#pragma GCC diagnostic pop
+#endif
+
 namespace mfem
 {
 /**
@@ -116,6 +126,349 @@ public:
    virtual ~CutIntegrationRules() {}
 };
 
+#ifdef MFEM_USE_ALGOIM
+// define templated element bases
+namespace TmplPoly_1D
+{
+
+/// Templated version of CalcBinomTerms
+template<typename float_type>
+void CalcBinomTerms(const int p, const float_type x, const float_type y,
+                    float_type* u)
+{
+   if (p == 0)
+   {
+      u[0] = float_type(1.);
+   }
+   else
+   {
+      int i;
+      const int *b = Poly_1D::Binom(p);
+      float_type z = x;
+      for (i = 1; i < p; i++)
+      {
+         u[i] = b[i]*z;
+         z *= x;
+      }
+      u[p] = z;
+      z = y;
+      for (i--; i > 0; i--)
+      {
+         u[i] *= z;
+         z *= y;
+      }
+      u[0] = z;
+   }
+}
+
+/// Templated version of CalcBinomTerms
+template<typename float_type>
+void CalcBinomTerms(const int p, const float_type x, const float_type y,
+                    float_type* u, float_type* d)
+{
+   if (p == 0)
+   {
+      u[0] = float_type(1.);
+      d[0] = float_type(0.);
+   }
+   else
+   {
+      int i;
+      const int *b = Poly_1D::Binom(p);
+      const float_type xpy = x + y, ptx = p*x;
+      float_type z = float_type(1.);
+
+      for (i = 1; i < p; i++)
+      {
+         d[i] = b[i]*z*(i*xpy - ptx);
+         z *= x;
+         u[i] = b[i]*z;
+      }
+      d[p] = p*z;
+      u[p] = z*x;
+      z = float_type(1.);
+      for (i--; i > 0; i--)
+      {
+         d[i] *= z;
+         z *= y;
+         u[i] *= z;
+      }
+      d[0] = -p*z;
+      u[0] = z*y;
+   }
+
+}
+
+/// Templated evaluation of Bernstein basis
+template <typename float_type>
+void CalcBernstein(const int p, const float_type x, float_type *u)
+{
+   CalcBinomTerms(p, x, 1. - x, u);
+}
+
+
+/// Templated evaluation of Bernstein basis
+template <typename float_type>
+void CalcBernstein(const int p, const float_type x,
+                   float_type *u, float_type *d)
+{
+   CalcBinomTerms(p, x, 1. - x, u, d);
+}
+
+
+}
+
+class AlgoimIntegrationRules : public CutIntegrationRules
+{
+public:
+
+   /** @brief Constructor to set up the generated cut IntegrationRules.
+
+       @param [in] order  Order of the constructed IntegrationRule.
+       @param [in] lvlset Coefficient whose zero level set specifies the cut.
+       @param [in] lsO    Polynomial degree for projecting the level-set
+                          Coefficient to a GridFunction, which is used to
+                          compute gradients and normals. */
+   AlgoimIntegrationRules(int order, Coefficient &lvlset, int lsO = 2)
+      : CutIntegrationRules(order, lvlset, lsO)
+   {
+      pe=nullptr;
+      le=nullptr;
+      currentLvlSet=nullptr;
+      currentGeometry=Geometry::Type::INVALID;
+      currentElementNo = -1;
+   }
+
+   virtual ~AlgoimIntegrationRules()
+   {
+      delete pe;
+      delete le;
+   }
+
+   virtual void SetOrder(int order) override
+   {
+      MFEM_VERIFY(order > 0, "Invalid input");
+      Order = order;
+      delete pe;
+      delete le;
+      pe=nullptr;
+      le=nullptr;
+      currentLvlSet=nullptr;
+      currentGeometry=Geometry::Type::INVALID;
+      currentElementNo=-1;
+   }
+
+   virtual void SetLevelSetProjectionOrder(int order) override
+   {
+      MFEM_VERIFY(order > 0, "Invalid input");
+      lsOrder = order;
+      delete pe;
+      delete le;
+      pe=nullptr;
+      le=nullptr;
+      currentLvlSet=nullptr;
+      currentGeometry=Geometry::Type::INVALID;
+      currentElementNo=-1;
+   }
+
+
+   /**
+   @brief Construct a cut-surface IntegrationRule.
+
+   Construct an IntegrationRule to integrate on the surface given by the
+   already specified level set function, for the element given by @a Tr.
+
+   @param [in]  Tr     Specifies the IntegrationRule's associated mesh element.
+   @param [out] result IntegrationRule on the cut-surface
+   */
+   virtual
+   void GetSurfaceIntegrationRule(ElementTransformation &Tr,
+                                  IntegrationRule &result) override;
+
+   /**
+   @brief Construct a cut-volume IntegrationRule.
+
+   Construct an IntegrationRule to integrate in the subdomain given by the
+   positive values of the already specified level set function, for the element
+   given by @a Tr.
+
+   @param [in]  Tr     Specifies the IntegrationRule's associated mesh element.
+   @param [out] result IntegrationRule for the cut-volume
+   @param [in]  sir    Corresponding IntegrationRule for the surface, which can
+                       be used to avoid computations.
+   */
+   virtual
+   void GetVolumeIntegrationRule(ElementTransformation &Tr,
+                                 IntegrationRule &result,
+                                 const IntegrationRule *sir = nullptr) override;
+
+
+   /**
+   @brief Compute transformation quadrature weights for surface integration.
+
+   Compute the transformation weights for integration over the cut-surface in
+   reference space.
+
+   @param [in]  Tr      Specifies the IntegrationRule's associated element.
+   @param [in]  sir     IntegrationRule defining the IntegrationPoints
+   @param [out] weights Vector containing the transformation weights.
+   */
+   virtual
+   void GetSurfaceWeights(ElementTransformation &Tr,
+                          const IntegrationRule &sir,
+                          Vector &weights) override;
+
+private:
+
+   /// projects the lvlset coefficient onto the lsvec,
+   /// i.e., represent the level-set using Bernstein bases
+   void GenerateLSVector(ElementTransformation &Tr, Coefficient* lvlset);
+
+
+   /// Lagrange finite element used for converting coefficients to positive basis
+   FiniteElement* le;
+   PositiveTensorFiniteElement *pe;
+   DenseMatrix T; //Projection matrix from nodal basis to positive basis
+   Vector lsvec; // level-set in Bernstein basis
+   Vector lsfun; // level-set in nodal basis
+   Geometry::Type currentGeometry; // the current element geometry
+   Coefficient* currentLvlSet; //the current level-set coefficient
+   int currentElementNo; //the current element No
+
+   /// 3D level-set function object required by Algoim.
+   struct LevelSet3D
+   {
+      /// Constructor for 3D level-set function object required by Algoim.
+      LevelSet3D(PositiveTensorFiniteElement* el_, Vector& lsfun_)
+         : el(el_), lsfun(lsfun_) { }
+
+      /// Returns the value of the LSF for point x.
+      template<typename T>
+      T operator() (const blitz::TinyVector<T,3>& x) const
+      {
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T u3[el_order+1];
+         TmplPoly_1D::CalcBernstein(el_order, x[0], u1);
+         TmplPoly_1D::CalcBernstein(el_order, x[1], u2);
+         TmplPoly_1D::CalcBernstein(el_order, x[2], u3);
+
+         const Array<int>& dof_map=el->GetDofMap();
+
+         T res=T(0.0);
+         for (int oo = 0, kk = 0; kk <= el_order; kk++)
+            for (int jj = 0; jj <= el_order; jj++)
+               for (int ii = 0; ii <= el_order; ii++)
+               {
+                  res=res-u1[ii]*u2[jj]*u3[kk]*lsfun(dof_map[oo++]);
+               }
+         return res;
+      }
+
+      /// Returns the gradients of the LSF for point x.
+      template<typename T>
+      blitz::TinyVector<T,3> grad(const blitz::TinyVector<T,3>& x) const
+      {
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T u3[el_order+1];
+         T d1[el_order+1];
+         T d2[el_order+1];
+         T d3[el_order+1];
+
+         TmplPoly_1D::CalcBernstein(el_order,x[0], u1, d1);
+         TmplPoly_1D::CalcBernstein(el_order,x[1], u2, d2);
+         TmplPoly_1D::CalcBernstein(el_order,x[2], u3, d3);
+
+         blitz::TinyVector<T,3> res(T(0.0),T(0.0),T(0.0));
+
+         const Array<int>& dof_map=el->GetDofMap();
+
+         for (int oo = 0, kk = 0; kk <= el_order; kk++)
+            for (int jj = 0; jj <= el_order; jj++)
+               for (int ii = 0; ii <= el_order; ii++)
+               {
+                  res[0]=res[0]-d1[ii]*u2[jj]*u3[kk]*lsfun(dof_map[oo]);
+                  res[1]=res[1]-u1[ii]*d2[jj]*u3[kk]*lsfun(dof_map[oo]);
+                  res[2]=res[2]-u1[ii]*u2[jj]*d3[kk]*lsfun(dof_map[oo]);
+                  oo++;
+               }
+
+         return res;
+      }
+
+   private:
+      PositiveTensorFiniteElement* el;
+      Vector& lsfun;
+   };
+
+   /// 2D level-set function object required by Algoim.
+   struct LevelSet2D
+   {
+      /// Constructor for 2D level-set function object required by Algoim.
+      LevelSet2D(PositiveTensorFiniteElement* el_, Vector& lsfun_)
+         :el(el_), lsfun(lsfun_) { }
+
+      /// Returns the value of the LSF for point x.
+      template<typename T>
+      T operator() (const blitz::TinyVector<T,2>& x) const
+      {
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         TmplPoly_1D::CalcBernstein(el_order, x[0], u1);
+         TmplPoly_1D::CalcBernstein(el_order, x[1], u2);
+
+         const Array<int>& dof_map=el->GetDofMap();
+
+         T res=T(0.0);
+
+         for (int oo = 0, jj = 0; jj <= el_order; jj++)
+            for (int ii = 0; ii <= el_order; ii++)
+            {
+               res=res-u1[ii]*u2[jj]*lsfun(dof_map[oo++]);
+            }
+         return res;
+      }
+
+      /// Returns the gradients of the LSF for point x.
+      template<typename T>
+      blitz::TinyVector<T,2> grad(const blitz::TinyVector<T,2>& x) const
+      {
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T d1[el_order+1];
+         T d2[el_order+1];
+
+         TmplPoly_1D::CalcBernstein(el_order,x[0], u1, d1);
+         TmplPoly_1D::CalcBernstein(el_order,x[1], u2, d2);
+
+         blitz::TinyVector<T,2> res(T(0.0),T(0.0));
+
+         const Array<int>& dof_map=el->GetDofMap();
+
+         for (int oo = 0, jj = 0; jj <= el_order; jj++)
+            for (int ii = 0; ii <= el_order; ii++)
+            {
+               res[0]=res[0]-(d1[ii]*u2[jj])*lsfun(dof_map[oo]);
+               res[1]=res[1]-(u1[ii]*d2[jj])*lsfun(dof_map[oo]);
+               oo++;
+            }
+
+         return res;
+      }
+
+
+   private:
+      PositiveTensorFiniteElement* el;
+      Vector& lsfun;
+   };
+};
+#endif //MFEM_USE_ALGOIM
+
 #ifdef MFEM_USE_LAPACK
 
 /**
@@ -212,10 +565,8 @@ protected:
     rule.
 
     @param [in] Tr ElementTransformation of the current element
-    @param [in] sir corresponding IntegrationRule on surface
     */
-   void ComputeVolumeWeights1D(ElementTransformation& Tr,
-                               const IntegrationRule* sir);
+   void ComputeVolumeWeights1D(ElementTransformation& Tr);
 
    /**
     @brief Compute 2D quadrature weights

fem/kernel_dispatch.hpp

@@ -78,9 +78,9 @@ namespace mfem
       const char *kernel_name = MFEM_KERNEL_NAME(KernelName);                  \
       using KernelSignature = KernelType;                                      \
       template <MFEM_PARAM_LIST P3>                                            \
-      static KernelSignature Kernel();                                         \
-      static KernelSignature Fallback(MFEM_PARAM_LIST P1);                     \
-      static KernelName &Get()                                                 \
+      static MFEM_EXPORT KernelSignature Kernel();                             \
+      static MFEM_EXPORT KernelSignature Fallback(MFEM_PARAM_LIST P1);         \
+      static MFEM_EXPORT KernelName &Get()                                     \
       { static KernelName table; return table;}                                \
    }
 
@@ -126,9 +126,9 @@ class KernelDispatchTable<Kernels,
          internal::KernelTypeList<Params...>,
          internal::KernelTypeList<OptParams...>>
 {
-   std::unordered_map<std::tuple<Params...>,
-       Signature,
-       KernelDispatchKeyHash<Params...>> table;
+   using TableType = std::unordered_map<std::tuple<Params...>,
+         Signature, KernelDispatchKeyHash<Params...>>;
+   TableType table;
 
 public:
    /// @brief Run the kernel with the given dispatch parameters and arguments.
@@ -162,7 +162,7 @@ public:
       {
          std::tuple<Params...> param_tuple(PARAMS...);
          Kernels::Get().table[param_tuple] =
-            Kernels:: template Kernel<PARAMS...>();
+            Kernels:: template Kernel<PARAMS..., OptParams{}...>();
       };
       // Version with optional parameters
       template <OptParams... OPT_PARAMS>
@@ -176,6 +176,12 @@ public:
          }
       };
    };
+
+   /// Return the dispatch map table
+   static const TableType &GetDispatchTable()
+   {
+      return Kernels::Get().table;
+   }
 };
 
 }

fem/lor/lor_ams.cpp

@@ -242,13 +242,13 @@ void BatchedLOR_AMS::FormGradientMatrix()
 template <typename T>
 static inline const T *HypreRead(const Memory<T> &mem)
 {
-   return mem.Read(GetHypreMemoryClass(), mem.Capacity());
+   return mem.Read(GetHypreForallMemoryClass(), mem.Capacity());
 }
 
 template <typename T>
 static inline T *HypreWrite(Memory<T> &mem)
 {
-   return mem.Write(GetHypreMemoryClass(), mem.Capacity());
+   return mem.Write(GetHypreForallMemoryClass(), mem.Capacity());
 }
 
 void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
@@ -278,10 +278,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
    const int sdim = vert_fes.GetMesh()->SpaceDimension();
    const int ntdofs = R->Height();
 
-   const MemoryClass mc = GetHypreMemoryClass();
-   bool dev = (mc == MemoryClass::DEVICE);
-
-   xyz_tvec = new Vector(ntdofs*sdim);
+   xyz_tvec = new Vector(ntdofs*sdim, GetHypreMemoryType());
 
    auto xyz_tv = Reshape(HypreWrite(xyz_tvec->GetMemory()), ntdofs, sdim);
    const auto xyz_e =
@@ -304,15 +301,12 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
    // Make x, y, z HypreParVectors point to T-vector data
    HYPRE_BigInt glob_size = vert_fes.GlobalTrueVSize();
    HYPRE_BigInt *cols = vert_fes.GetTrueDofOffsets();
-
-   real_t *d_x_ptr = xyz_tv + 0*ntdofs;
-   x = new HypreParVector(vert_fes.GetComm(), glob_size, d_x_ptr, cols, dev);
-   real_t *d_y_ptr = xyz_tv + 1*ntdofs;
-   y = new HypreParVector(vert_fes.GetComm(), glob_size, d_y_ptr, cols, dev);
+   MPI_Comm comm = vert_fes.GetComm();
+   x = new HypreParVector(comm, glob_size, *xyz_tvec, 0*ntdofs, cols);
+   y = new HypreParVector(comm, glob_size, *xyz_tvec, 1*ntdofs, cols);
    if (sdim == 3)
    {
-      real_t *d_z_ptr = xyz_tv + 2*ntdofs;
-      z = new HypreParVector(vert_fes.GetComm(), glob_size, d_z_ptr, cols, dev);
+      z = new HypreParVector(comm, glob_size, *xyz_tvec, 2*ntdofs, cols);
    }
    else
    {

fem/pbilinearform.cpp

@@ -438,9 +438,31 @@ void ParBilinearForm::FormLinearSystem(
    const Array<int> &ess_tdof_list, Vector &x, Vector &b,
    OperatorHandle &A, Vector &X, Vector &B, int copy_interior)
 {
+   const Operator &P = *pfes->GetProlongationMatrix();
+   const SparseMatrix &R = *pfes->GetRestrictionMatrix();
+
    if (ext)
    {
-      ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+      if (hybridization)
+      {
+         HypreParVector true_X(pfes), true_B(pfes);
+         P.MultTranspose(b, true_B);
+         R.Mult(x, true_X);
+
+         FormSystemMatrix(ess_tdof_list, A);
+         ConstrainedOperator *A_constrained;
+         Operator::FormConstrainedSystemOperator(ess_tdof_list, A_constrained);
+         A_constrained->EliminateRHS(true_X, true_B);
+         delete A_constrained;
+         R.MultTranspose(true_B, b);
+         hybridization->ReduceRHS(true_B, B);
+         X.SetSize(B.Size());
+         X = 0.0;
+      }
+      else
+      {
+         ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+      }
       return;
    }
 
@@ -448,9 +470,6 @@ void ParBilinearForm::FormLinearSystem(
    // eliminated part of the matrix.
    FormSystemMatrix(ess_tdof_list, A);
 
-   const Operator &P = *pfes->GetProlongationMatrix();
-   const SparseMatrix &R = *pfes->GetRestrictionMatrix();
-
    // Transform the system and perform the elimination in B, based on the
    // essential BC values from x. Restrict the BC part of x in X, and set the
    // non-BC part to zero. Since there is no good initial guess for the Lagrange
@@ -495,7 +514,16 @@ void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 {
    if (ext)
    {
-      ext->FormSystemMatrix(ess_tdof_list, A);
+      if (hybridization)
+      {
+         const int remove_zeros = 0;
+         Finalize(remove_zeros);
+         hybridization->GetParallelMatrix(A);
+      }
+      else
+      {
+         ext->FormSystemMatrix(ess_tdof_list, A);
+      }
       return;
    }
 
@@ -541,7 +569,7 @@ void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 void ParBilinearForm::RecoverFEMSolution(
    const Vector &X, const Vector &b, Vector &x)
 {
-   if (ext)
+   if (ext && !hybridization)
    {
       ext->RecoverFEMSolution(X, b, x);
       return;

fem/pfespace.cpp

@@ -564,6 +564,10 @@ const FaceRestriction *ParFiniteElementSpace::GetFaceRestriction(
             res = new ParNCL2FaceRestriction(*this, f_ordering, type, m);
          }
       }
+      else if (dynamic_cast<const DG_Interface_FECollection*>(fec))
+      {
+         res = new L2InterfaceFaceRestriction(*this, f_ordering, type);
+      }
       else
       {
          if (Conforming())

fem/qinterp/grad.hpp

@@ -43,12 +43,13 @@ static void Derivatives1D(const int NE,
                           const int q1d)
 {
    MFEM_CONTRACT_VAR(b_);
+   const int SDIM = GRAD_PHYS ? sdim : 1;
    const auto g = Reshape(g_, q1d, d1d);
-   const auto j = Reshape(j_, q1d, sdim, NE);
+   const auto j = Reshape(j_, q1d, SDIM, NE);
    const auto x = Reshape(x_, d1d, vdim, NE);
    auto y = Q_LAYOUT == QVectorLayout::byNODES ?
-            Reshape(y_, q1d, vdim, sdim, NE):
-            Reshape(y_, vdim, sdim, q1d, NE);
+            Reshape(y_, q1d, vdim, SDIM, NE):
+            Reshape(y_, vdim, SDIM, q1d, NE);
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
@@ -63,8 +64,8 @@ static void Derivatives1D(const int NE,
             }
             if (GRAD_PHYS)
             {
-               if (sdim == 1) { du[0] /= j(q, 0, e); }
-               else if (sdim == 2)
+               if (SDIM == 1) { du[0] /= j(q, 0, e); }
+               else if (SDIM == 2)
                {
                   const real_t Jloc[2] = {j(q,0,e), j(q,1,e)};
                   real_t Jinv[3];
@@ -74,7 +75,7 @@ static void Derivatives1D(const int NE,
                   du[0] = U;
                   du[1] = V;
                }
-               else // sdim == 3
+               else // SDIM == 3
                {
                   const real_t Jloc[3] = {j(q,0,e), j(q,1,e), j(q,2,e)};
                   real_t Jinv[3];
@@ -87,7 +88,7 @@ static void Derivatives1D(const int NE,
                   du[2] = W;
                }
             }
-            for (int d = 0; d < sdim; ++d)
+            for (int d = 0; d < SDIM; ++d)
             {
                if (Q_LAYOUT == QVectorLayout::byVDIM)  { y(c, d, q, e) = du[d]; }
                if (Q_LAYOUT == QVectorLayout::byNODES) { y(q, c, d, e) = du[d]; }
@@ -372,14 +373,222 @@ static void Derivatives3D(const int NE,
    });
 }
 
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
+static void CollocatedDerivatives1D(const int NE,
+                                    const real_t *g_,
+                                    const real_t *j_,
+                                    const real_t *x_,
+                                    real_t *y_,
+                                    const int sdim,
+                                    const int vdim,
+                                    const int d1d)
+{
+   Derivatives1D<Q_LAYOUT, GRAD_PHYS>(
+      NE, nullptr, g_, j_, x_, y_, sdim, vdim, d1d, d1d);
+}
+
+// Template compute kernel for derivatives in 2D: tensor product version.
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
+         int T_VDIM = 0, int T_D1D = 0,
+         int T_NBZ = 1>
+static void CollocatedDerivatives2D(const int NE,
+                                    const real_t *g_,
+                                    const real_t *j_,
+                                    const real_t *x_,
+                                    real_t *y_,
+                                    const int sdim = 2,
+                                    const int vdim = 0,
+                                    const int d1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   const int SDIM = GRAD_PHYS ? sdim : 2;
+   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+
+   const auto g = Reshape(g_, D1D, D1D);
+   const auto j = Reshape(j_, D1D, D1D, SDIM, 2, NE);
+   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
+            Reshape(y_, D1D, D1D, VDIM, SDIM, NE):
+            Reshape(y_, VDIM, SDIM, D1D, D1D, NE);
+
+   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED real_t XY[NBZ][MD1*MD1];
+      DeviceTensor<2> X((real_t*)(XY+tidz), D1D, D1D);
+
+      for (int c = 0; c < VDIM; ++c)
+      {
+         kernels::internal::LoadX<MD1,NBZ>(e,D1D,c,x,XY);
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               real_t u = 0.0;
+               real_t v = 0.0;
+               real_t w = 0.0;
+               for (int dxy = 0; dxy < D1D; ++dxy)
+               {
+                  u += X(dxy, dy) * g(dx,dxy);
+                  v += X(dx, dxy) * g(dy,dxy);
+               }
+
+               if (GRAD_PHYS)
+               {
+                  if (SDIM == 2)
+                  {
+                     real_t Jloc[4], Jinv[4];
+                     Jloc[0] = j(dx,dy,0,0,e);
+                     Jloc[1] = j(dx,dy,1,0,e);
+                     Jloc[2] = j(dx,dy,0,1,e);
+                     Jloc[3] = j(dx,dy,1,1,e);
+                     kernels::CalcInverse<2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*u + Jinv[1]*v;
+                     const real_t V = Jinv[2]*u + Jinv[3]*v;
+                     u = U;
+                     v = V;
+                  }
+                  else
+                  {
+                     real_t Jloc[6], Jinv[6];
+                     Jloc[0] = j(dx,dy,0,0,e);
+                     Jloc[1] = j(dx,dy,1,0,e);
+                     Jloc[2] = j(dx,dy,2,0,e);
+                     Jloc[3] = j(dx,dy,0,1,e);
+                     Jloc[4] = j(dx,dy,1,1,e);
+                     Jloc[5] = j(dx,dy,2,1,e);
+                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*u + Jinv[1]*v;
+                     const real_t V = Jinv[2]*u + Jinv[3]*v;
+                     const real_t W = Jinv[4]*u + Jinv[5]*v;
+                     u = U;
+                     v = V;
+                     w = W;
+                  }
+               }
+
+               if (Q_LAYOUT == QVectorLayout::byVDIM)
+               {
+                  y(c,0,dx,dy,e) = u;
+                  y(c,1,dx,dy,e) = v;
+                  if (SDIM == 3) { y(c,2,dx,dy,e) = w; }
+               }
+               if (Q_LAYOUT == QVectorLayout::byNODES)
+               {
+                  y(dx,dy,c,0,e) = u;
+                  y(dx,dy,c,1,e) = v;
+                  if (SDIM == 3) { y(dx,dy,c,2,e) = w; }
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+// Template compute kernel for derivatives in 3D: tensor product version.
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
+         int T_VDIM = 0, int T_D1D = 0>
+static void CollocatedDerivatives3D(const int NE,
+                                    const real_t *g_,
+                                    const real_t *j_,
+                                    const real_t *x_,
+                                    real_t *y_,
+                                    const int sdim = 3,
+                                    const int vdim = 0,
+                                    const int d1d = 0)
+{
+   MFEM_VERIFY(sdim == 3, "");
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto g = Reshape(g_, D1D, D1D);
+   const auto j = Reshape(j_, D1D, D1D, D1D, 3, 3, NE);
+   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
+            Reshape(y_, D1D, D1D, D1D, VDIM, 3, NE):
+            Reshape(y_, VDIM, 3, D1D, D1D, D1D, NE);
+
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
+
+      MFEM_SHARED real_t uvw[MD1*MD1*MD1];
+      DeviceTensor<3> X(uvw, D1D, D1D, D1D);
+
+      for (int c = 0; c < VDIM; ++c)
+      {
+         kernels::internal::LoadX(e,D1D,c,x,X);
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  real_t v = 0.0;
+                  real_t w = 0.0;
+                  for (int dxyz = 0; dxyz < D1D; ++dxyz)
+                  {
+                     u += X(dxyz, dy, dz) * g(dx,dxyz);
+                     v += X(dx, dxyz, dz) * g(dy,dxyz);
+                     w += X(dx, dy, dxyz) * g(dz,dxyz);
+                  }
+
+                  if (GRAD_PHYS)
+                  {
+                     real_t Jloc[9], Jinv[9];
+                     for (int col = 0; col < 3; col++)
+                     {
+                        for (int row = 0; row < 3; row++)
+                        {
+                           Jloc[row+3*col] = j(dx,dy,dz,row,col,e);
+                        }
+                     }
+                     kernels::CalcInverse<3>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
+                     const real_t V = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
+                     const real_t W = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
+                     u = U; v = V; w = W;
+                  }
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     y(c,0,dx,dy,dz,e) = u;
+                     y(c,1,dx,dy,dz,e) = v;
+                     y(c,2,dx,dy,dz,e) = w;
+                  }
+                  if (Q_LAYOUT == QVectorLayout::byNODES)
+                  {
+                     y(dx,dy,dz,c,0,e) = u;
+                     y(dx,dy,dz,c,1,e) = v;
+                     y(dx,dy,dz,c,2,e) = w;
+                  }
+
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
 } // namespace quadrature_interpolator
 
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
-         int VDIM, int D1D, int Q1D, int NBZ>
+template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
+         int Q1D, int NBZ>
 QuadratureInterpolator::GradKernelType
 QuadratureInterpolator::GradKernels::Kernel()
 {
@@ -389,6 +598,17 @@ QuadratureInterpolator::GradKernels::Kernel()
    else { MFEM_ABORT(""); }
 }
 
+template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
+         int NBZ>
+QuadratureInterpolator::CollocatedGradKernelType
+QuadratureInterpolator::CollocatedGradKernels::Kernel()
+{
+   if (DIM == 1) { return internal::quadrature_interpolator::CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 2) { return internal::quadrature_interpolator::CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, NBZ>; }
+   else if (DIM == 3) { return internal::quadrature_interpolator::CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D>; }
+   else { MFEM_ABORT(""); }
+}
+
 /// @endcond
 
 } // namespace mfem

fem/qinterp/grad_by_nodes.cpp

@@ -23,50 +23,73 @@ template <bool P>
 void InitGradByNodesKernels()
 {
    using k = QuadratureInterpolator::GradKernels;
+   constexpr auto L = QVectorLayout::byNODES;
    // 2D
-   k::Specialization<2,QVectorLayout::byNODES,P,1,3,3>::template Opt<16>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,1,3,4>::template Opt<16>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,1,4,3>::template Opt<16>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,1,4,4>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,3,3>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,3,4>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,4,3>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,4,4>::template Opt<16>::Add();
 
-   k::Specialization<2,QVectorLayout::byNODES,P,2,2,2>::template Opt<16>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,2,3>::template Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,2,4>::template Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,2,5>::template Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,2,6>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,2,2>::template Opt<16>::Add();
+   k::Specialization<2,L,P,2,2,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,2,4>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,2,5>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,2,6>::template Opt<2>::Add();
 
-   k::Specialization<2,QVectorLayout::byNODES,P,2,3,3>::template Opt<2>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,3,4>::template Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,4,3>::template Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,3,6>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,3,3>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,4,3>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,3,6>::template Opt<2>::Add();
 
-   k::Specialization<2,QVectorLayout::byNODES,P,2,4,4>::template Opt<2>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,4,5>::template Opt<2>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,4,6>::template Opt<2>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,P,2,4,7>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,4>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,5>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,7>::template Opt<2>::Add();
 
-   k::Specialization<2,QVectorLayout::byNODES,P,2,5,6>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,5,6>::template Opt<2>::Add();
    // 3D
-   k::Specialization<3,QVectorLayout::byNODES,P,1,2,4>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,1,3,3>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,1,3,4>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,1,3,6>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,1,4,4>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,1,4,8>::template Opt<1>::Add();
+   k::Specialization<3,L,P,1,2,4>::Add();
+   k::Specialization<3,L,P,1,3,3>::Add();
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,3,6>::Add();
+   k::Specialization<3,L,P,1,4,4>::Add();
+   k::Specialization<3,L,P,1,4,8>::Add();
 
-   k::Specialization<3,QVectorLayout::byNODES,P,3,2,3>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,2,4>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,2,5>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,2,6>::template Opt<1>::Add();
+   k::Specialization<3,L,P,3,2,3>::Add();
+   k::Specialization<3,L,P,3,2,4>::Add();
+   k::Specialization<3,L,P,3,2,5>::Add();
+   k::Specialization<3,L,P,3,2,6>::Add();
 
-   k::Specialization<3,QVectorLayout::byNODES,P,3,3,3>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,3,4>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,3,5>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,3,6>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,4,4>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,4,6>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,4,7>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,P,3,4,8>::template Opt<1>::Add();
+   k::Specialization<3,L,P,3,3,3>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,3,5>::Add();
+   k::Specialization<3,L,P,3,3,6>::Add();
+   k::Specialization<3,L,P,3,4,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,4,7>::Add();
+   k::Specialization<3,L,P,3,4,8>::Add();
+
+   using k2 = QuadratureInterpolator::CollocatedGradKernels;
+
+   // 2D
+   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
+   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
+
+   k2::Specialization<3,L,P,1,2>::Add();
+   k2::Specialization<3,L,P,1,3>::Add();
+   k2::Specialization<3,L,P,1,4>::Add();
+
+   k2::Specialization<3,L,P,2,2>::Add();
+   k2::Specialization<3,L,P,2,3>::Add();
+   k2::Specialization<3,L,P,2,4>::Add();
+
+   k2::Specialization<3,L,P,3,2>::Add();
+   k2::Specialization<3,L,P,3,3>::Add();
+   k2::Specialization<3,L,P,3,4>::Add();
 }
 
 template void InitGradByNodesKernels<true>();

fem/qinterp/grad_by_vdim.cpp

@@ -23,22 +23,46 @@ template <bool P>
 void InitGradByVDimKernels()
 {
    using k = QuadratureInterpolator::GradKernels;
+   constexpr auto L = QVectorLayout::byVDIM;
    // 2D
-   k::Specialization<2,QVectorLayout::byVDIM,P,1,3,4>::template Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,P,1,4,6>::template Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,P,1,5,8>::template Opt<2>::Add();
+   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
 
-   k::Specialization<2,QVectorLayout::byVDIM,P,2,3,3>::template Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,P,2,3,4>::template Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,P,2,4,6>::template Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,P,2,5,8>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
    // 3D
-   k::Specialization<3,QVectorLayout::byVDIM,P,1,3,4>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,P,1,4,6>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,P,1,5,8>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,P,3,3,4>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,P,3,4,6>::template Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,P,3,5,8>::template Opt<1>::Add();
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,4,6>::Add();
+   k::Specialization<3,L,P,1,5,8>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,5,8>::Add();
+
+   using k2 = QuadratureInterpolator::CollocatedGradKernels;
+   // 2D
+   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
+
+   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
+   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
+
+   // 3D
+   k2::Specialization<3,L,P,1,2>::Add();
+   k2::Specialization<3,L,P,1,3>::Add();
+   k2::Specialization<3,L,P,1,4>::Add();
+
+   k2::Specialization<3,L,P,2,2>::Add();
+   k2::Specialization<3,L,P,2,3>::Add();
+   k2::Specialization<3,L,P,2,4>::Add();
+
+   k2::Specialization<3,L,P,3,2>::Add();
+   k2::Specialization<3,L,P,3,3>::Add();
+   k2::Specialization<3,L,P,3,4>::Add();
 }
 
 template void InitGradByVDimKernels<true>();

fem/quadinterpolator.cpp

@@ -30,26 +30,27 @@ void InitEvalKernels();
 void InitDetKernels();
 template <bool P> void InitGradByNodesKernels();
 template <bool P> void InitGradByVDimKernels();
-}
-}
-
-QuadratureInterpolator::Kernels QuadratureInterpolator::kernels;
-QuadratureInterpolator::Kernels::Kernels()
+struct Kernels
 {
-   using namespace internal::quadrature_interpolator;
+   Kernels()
+   {
+      using namespace internal::quadrature_interpolator;
 
-   InitEvalByNodesKernels();
-   InitEvalByVDimKernels();
-   // Non-phys grad kernels
-   InitGradByNodesKernels<false>();
-   InitGradByVDimKernels<false>();
-   // Phys grad kernels
-   InitGradByNodesKernels<true>();
-   InitGradByVDimKernels<true>();
-   // Determinants
-   InitDetKernels();
-   // Non-tensor
-   InitEvalKernels();
+      InitEvalByNodesKernels();
+      InitEvalByVDimKernels();
+      // Non-phys grad kernels
+      InitGradByNodesKernels<false>();
+      InitGradByVDimKernels<false>();
+      // Phys grad kernels
+      InitGradByNodesKernels<true>();
+      InitGradByVDimKernels<true>();
+      // Determinants
+      InitDetKernels();
+      // Non-tensor
+      InitEvalKernels();
+   }
+};
+}
 }
 
 QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
@@ -61,6 +62,8 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
    q_layout(QVectorLayout::byNODES),
    use_tensor_products(UsesTensorBasis(fes))
 {
+   static internal::quadrature_interpolator::Kernels kernels;
+
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
    const FiniteElement *fe = fespace->GetFE(0);
@@ -600,34 +603,55 @@ void QuadratureInterpolator::Determinants(const Vector &e_vec,
 
 namespace
 {
+
+using namespace internal::quadrature_interpolator;
+
 using EvalKernel = QuadratureInterpolator::EvalKernelType;
 using TensorEvalKernel = QuadratureInterpolator::TensorEvalKernelType;
 using GradKernel = QuadratureInterpolator::GradKernelType;
+using CollocatedGradKernel = QuadratureInterpolator::CollocatedGradKernelType;
 
 template <QVectorLayout Q_LAYOUT>
 TensorEvalKernel FallbackTensorEvalKernel(int DIM)
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::Values1D<Q_LAYOUT>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::Values2D<Q_LAYOUT>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Values3D<Q_LAYOUT>; }
+   if (DIM == 1) { return Values1D<Q_LAYOUT>; }
+   else if (DIM == 2) { return Values2D<Q_LAYOUT>; }
+   else if (DIM == 3) { return Values3D<Q_LAYOUT>; }
    else { MFEM_ABORT(""); }
 }
 
 template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
 GradKernel GetGradKernel(int DIM)
 {
-   if (DIM == 1) { return internal::quadrature_interpolator::Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::Derivatives2D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Derivatives3D<Q_LAYOUT, GRAD_PHYS>; }
+   if (DIM == 1) { return Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 2) { return Derivatives2D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 3) { return Derivatives3D<Q_LAYOUT, GRAD_PHYS>; }
    else { MFEM_ABORT(""); }
 }
 
+
 template<QVectorLayout Q_LAYOUT>
 GradKernel GetGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetGradKernel<Q_LAYOUT, false>(DIM); }
 }
+
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
+CollocatedGradKernel GetCollocatedGradKernel(int DIM)
+{
+   if (DIM == 1) { return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 2) { return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 3) { return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>; }
+   else { MFEM_ABORT(""); }
+}
+
+template<QVectorLayout Q_LAYOUT>
+CollocatedGradKernel GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
+{
+   if (GRAD_PHYS) { return GetCollocatedGradKernel<Q_LAYOUT, true>(DIM); }
+   else { return GetCollocatedGradKernel<Q_LAYOUT, false>(DIM); }
+}
 } // namespace
 
 template <int DIM, int VDIM, int ND, int NQ>
@@ -673,6 +697,13 @@ GradKernel QuadratureInterpolator::GradKernels::Fallback(
    else { return GetGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
+CollocatedGradKernel QuadratureInterpolator::CollocatedGradKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int)
+{
+   if (Q_LAYOUT == QVectorLayout::byNODES) { return GetCollocatedGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
+   else { return GetCollocatedGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
+}
+
 /// @endcond
 
 namespace internal

fem/quadinterpolator.hpp

@@ -138,6 +138,10 @@ public:
    using GradKernelType = void(*)(const int, const real_t *, const real_t *,
                                   const real_t *, const real_t *, real_t *,
                                   const int, const int, const int, const int);
+   using CollocatedGradKernelType = void(*)(const int, const real_t *,
+                                            const real_t *, const real_t *,
+                                            real_t *, const int, const int,
+                                            const int);
    using DetKernelType = void(*)(const int NE, const real_t *, const real_t *,
                                  const real_t *, real_t *, const int, const int,
                                  Vector *);
@@ -152,8 +156,8 @@ public:
                          (int, QVectorLayout, bool, int, int, int), (int));
    MFEM_REGISTER_KERNELS(DetKernels, DetKernelType, (int, int, int, int));
    MFEM_REGISTER_KERNELS(EvalKernels, EvalKernelType, (int, int, int, int));
-
-   static struct Kernels { Kernels(); } kernels;
+   MFEM_REGISTER_KERNELS(CollocatedGradKernels, CollocatedGradKernelType,
+                         (int, QVectorLayout, bool, int, int), (int));
 };
 
 }

fem/restriction.cpp

@@ -56,7 +56,7 @@ ElementRestriction::ElementRestriction(const FiniteElementSpace &f,
          dynamic_cast<const TensorBasisElement*>(fe);
       const Array<int> &fe_dof_map = el->GetDofMap();
       MFEM_VERIFY(fe_dof_map.Size() > 0, "invalid dof map");
-      dof_map = fe_dof_map.GetData();
+      dof_map = fe_dof_map.HostRead();
    }
    const Table& e2dTable = fes.GetElementToDofTable();
    const int* element_map = e2dTable.GetJ();
@@ -2284,6 +2284,89 @@ void NCL2FaceRestriction::ComputeGatherIndices()
    gather_offsets[0] = 0;
 }
 
+L2InterfaceFaceRestriction::L2InterfaceFaceRestriction(
+   const FiniteElementSpace& fes_,
+   const ElementDofOrdering ordering_,
+   const FaceType type_)
+   : fes(fes_),
+     ordering(ordering_),
+     type(type_),
+     nfaces(fes.GetNFbyType(type)),
+     vdim(fes.GetVDim()),
+     byvdim(fes.GetOrdering() == Ordering::byVDIM),
+     face_dofs(nfaces > 0 ? fes.GetFaceElement(0)->GetDof() : 0),
+     nfdofs(face_dofs*nfaces),
+     ndofs(fes.GetNDofs())
+{
+   height = nfdofs;
+   width = ndofs;
+
+   const Table &face2dof = fes.GetFaceToDofTable();
+
+   const Mesh &mesh = *fes.GetMesh();
+   int face_idx = 0;
+   gather_map.SetSize(nfdofs);
+   for (int f = 0; f < mesh.GetNumFaces(); ++f)
+   {
+      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
+      if (!face.IsOfFaceType(type)) { continue; }
+      for (int i = 0; i < face_dofs; ++i)
+      {
+         gather_map[i + face_idx*face_dofs] = face2dof.GetJ()[i + f*face_dofs];
+      }
+      ++face_idx;
+   }
+}
+
+void L2InterfaceFaceRestriction::Mult(const Vector &x, Vector &y) const
+{
+   const int nd = face_dofs;
+   const int nf = nfaces;
+   const int vd = vdim;
+   const bool t = byvdim;
+   const int *map = gather_map.Read();
+
+   const auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
+   auto d_y = Reshape(y.Write(), nd, vd, nf);
+
+   mfem::forall(nd*nf, [=] MFEM_HOST_DEVICE (int i)
+   {
+      const int j = map[i];
+      for (int c = 0; c < vd; ++c)
+      {
+         d_y(i % nd, c, i / nd) = d_x(t?c:j, t?j:c);
+      }
+   });
+}
+
+void L2InterfaceFaceRestriction::AddMultTranspose(
+   const Vector &x, Vector &y, const real_t a) const
+{
+   const int nd = face_dofs;
+   const int nf = nfaces;
+   const int vd = vdim;
+   const bool t = byvdim;
+   const int *map = gather_map.Read();
+
+   const auto d_x = Reshape(x.Read(), nd, vd, nf);
+   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+
+   mfem::forall(ndofs, [=] MFEM_HOST_DEVICE (int i) { d_y[i] = 0.0; });
+   mfem::forall(nd*nf, [=] MFEM_HOST_DEVICE (int i)
+   {
+      const int j = map[i];
+      for (int c = 0; c < vd; ++c)
+      {
+         d_y(t?c:j, t?j:c) = d_x(i % nd, c, i / nd);
+      }
+   });
+}
+
+const Array<int> &L2InterfaceFaceRestriction::GatherMap() const
+{
+   return gather_map;
+}
+
 Vector GetLVectorFaceNbrData(
    const FiniteElementSpace &fes, const Vector &x, FaceType ftype)
 {

fem/restriction.hpp

@@ -262,6 +262,12 @@ public:
    {
       MFEM_ABORT("Not implemented for this restriction operator.");
    }
+
+   /// @brief Low-level access to the underlying gather map.
+   virtual const Array<int> &GatherMap() const
+   {
+      MFEM_ABORT("Not implemented for this restriction operator.");
+   }
 };
 
 /// @brief Operator that extracts face degrees of freedom for H1, ND, or RT
@@ -1066,6 +1072,62 @@ public:
    void DoubleValuedNonconformingTransposeInterpolationInPlace(Vector& x) const;
 };
 
+/// Operator that extracts face degrees of freedom for L2 interface spaces.
+/** Objects of this type are typically created and owned by FiniteElementSpace
+    objects, see FiniteElementSpace::GetFaceRestriction(). */
+class L2InterfaceFaceRestriction : public FaceRestriction
+{
+protected:
+   const FiniteElementSpace &fes; ///< The finite element space
+   const ElementDofOrdering ordering; ///< Requested ordering
+   const FaceType type; ///< Face type (interior or boundary)
+   const int nfaces; ///< Number of faces of the requested type
+   const int vdim; ///< vdim of the space
+   const bool byvdim; ///< DOF ordering (by nodes or by vdim)
+   const int face_dofs; ///< Number of dofs on each face
+   const int nfdofs; ///< Total number of dofs on the faces (E-vector size)
+   const int ndofs; ///< Number of dofs in the space (L-vector size)
+   Array<int> gather_map; ///< Gather map
+
+public:
+   /** @brief Constructs an L2InterfaceFaceRestriction.
+
+       @param[in] fes_       The FiniteElementSpace on which this operates
+       @param[in] ordering_  Request a specific face dof ordering
+       @param[in] type_      Request internal or boundary faces dofs */
+   L2InterfaceFaceRestriction(const FiniteElementSpace& fes_,
+                              const ElementDofOrdering ordering_,
+                              const FaceType type_);
+
+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+       face E-Vector.
+
+       @param[in]  x The L-vector degrees of freedom.
+       @param[out] y The face E-Vector degrees of freedom with size (face_dofs,
+                     vdim, nf), where nf is the number of interior or boundary
+                     faces requested by @a type in the constructor. The
+                     face_dofs are ordered according to the given
+                     ElementDofOrdering. */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   using FaceRestriction::AddMultTranspose;
+
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in]     x The face E-Vector degrees of freedom with size
+                        (face_dofs, vdim, nf), where nf is the number of
+                        interior or boundary faces requested by @a type in the
+                        constructor. The face_dofs should be ordered according
+                        to the given ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom.
+       @param[in]     a Scalar coefficient for addition. */
+   void AddMultTranspose(const Vector &x, Vector &y,
+                         const real_t a = 1.0) const override;
+
+   const Array<int> &GatherMap() const override;
+};
+
 /** @brief Convert a dof face index from Native ordering to lexicographic
     ordering for quads and hexes.
 

fem/transfer.hpp

@@ -40,6 +40,10 @@ protected:
    OperatorHandle fw_t_oper; ///< Forward true-dof operator
    OperatorHandle bw_t_oper; ///< Backward true-dof operator
 
+   bool use_ea;
+
+   MemoryType d_mt;
+
 #ifdef MFEM_USE_MPI
    bool parallel;
 #endif
@@ -59,14 +63,23 @@ protected:
 
 public:
    /** Construct a transfer algorithm between the domain, @a dom_fes_, and
-       range, @a ran_fes_, FE spaces. */
-   GridTransfer(FiniteElementSpace &dom_fes_, FiniteElementSpace &ran_fes_);
+       range, @a ran_fes_, FE spaces, d_mt_ will specify memory space for
+       large data structures */
+   GridTransfer(FiniteElementSpace &dom_fes_,
+                FiniteElementSpace &ran_fes_);
 
    /// Virtual destructor
    virtual ~GridTransfer() { }
 
+   /** Uses device friendly element assembly versions for L2Projection
+       transfers, L2, H1 FEM spaces currently supported */
+   void UseEA(bool use_ea_) { use_ea = use_ea_;}
+
+   /** Set memory type for large data structures */
+   void SetMemType(MemoryType d_mt_) {d_mt = d_mt_;}
+
    /** @brief Set the desired Operator::Type for the construction of all
-       operators defined by the underlying transfer algorithm. */
+        operators defined by the underlying transfer algorithm. */
    /** The default value is Operator::ANY_TYPE which typically corresponds to a
        matrix-free operator representation. Note that derived classes are not
        required to support this setting and can ignore it. */
@@ -169,7 +182,8 @@ public:
     smaller than the number of coarse dofs. */
 class L2ProjectionGridTransfer : public GridTransfer
 {
-protected:
+   // Must be public due to host device lambdas
+public:
    /** Abstract class representing projection operator between a high-order
        finite element space on a coarse mesh, and a low-order finite element
        space on a refined mesh (LOR). We assume that the low-order space,
@@ -194,10 +208,13 @@ protected:
       const FiniteElementSpace& fes_ho;
       const FiniteElementSpace& fes_lor;
 
+      MemoryType d_mt;
+      Array<int> offsets;
       Table ho2lor;
 
       L2Projection(const FiniteElementSpace& fes_ho_,
-                   const FiniteElementSpace& fes_lor_);
+                   const FiniteElementSpace& fes_lor_,
+                   MemoryType d_mt_ = Device::GetHostMemoryType());
 
       void BuildHo2Lor(int nel_ho, int nel_lor,
                        const CoarseFineTransformations& cf_tr);
@@ -207,6 +224,50 @@ protected:
                          ElementTransformation* tr_lor,
                          IntegrationPointTransformation& ip_tr,
                          DenseMatrix& M_mixed_el) const;
+
+      void ElemMixedMass(Geometry::Type geom, const FiniteElement& fe_ho,
+                         const FiniteElement& fe_lor,
+                         ElementTransformation* el_tr,
+                         IntegrationPointTransformation& ip_tr,
+                         DenseMatrix& B_L, DenseMatrix& B_H) const;
+   public:
+      /* Returns the Mixed Mass M_LH via device element assembly by building the
+      basis functions and data at the quadrature points. */
+      void MixedMassEA(const FiniteElementSpace& fes_ho_,
+                       const FiniteElementSpace& fes_lor_,
+                       Vector &M_LH,
+                       MemoryType d_mt_ = Device::GetHostMemoryType());
+   };
+
+   // Class below must be public as we now have device code
+public:
+   class H1SpaceMixedMassOperator : public Operator
+   {
+   protected:
+      const FiniteElementSpace* fes_ho;
+      const FiniteElementSpace* fes_lor;
+      Table* ho2lor;
+      Vector* M_LH_ea;
+   public:
+      H1SpaceMixedMassOperator(const FiniteElementSpace* fes_ho_,
+                               const FiniteElementSpace* fes_lor_,
+                               Table* ho2lor_, Vector* M_LH_ea_);
+      void Mult(const Vector& x, Vector& y) const;
+      void MultTranspose(const Vector& x, Vector& y) const;
+   };
+
+   class H1SpaceLumpedMassOperator : public Operator
+   {
+   protected:
+      const FiniteElementSpace* fes_ho;
+      const FiniteElementSpace* fes_lor;
+      Vector* ML_inv; // inverse of lumped M_L
+   public:
+      H1SpaceLumpedMassOperator(const FiniteElementSpace* fes_ho_,
+                                const FiniteElementSpace* fes_lor_,
+                                Vector& ML_inv_);
+      void Mult(const Vector& x, Vector& y) const;
+      void MultTranspose(const Vector& x, Vector& y) const;
    };
 
    /** Class for projection operator between a L2 high-order finite element
@@ -214,17 +275,24 @@ protected:
        refined mesh (LOR). */
    class L2ProjectionL2Space : public L2Projection
    {
-      // The restriction and prolongation operators are represented as dense
-      // elementwise matrices (of potentially different sizes, because of mixed
-      // meshes or p-refinement). The matrix entries are stored in the R and P
-      // arrays. The entries of the i'th high-order element are stored at the
-      // index given by offsets[i].
+      /// The restriction and prolongation operators are represented as dense
+      /// elementwise matrices (of potentially different sizes, because of mixed
+      /// meshes or p-refinement). The matrix entries are stored in the R and P
+      /// arrays. The entries of the i'th high-order element are stored at the
+      /// index given by offsets[i].
       mutable Array<real_t> R, P;
-      Array<int> offsets;
+
+      const bool use_ea;
 
    public:
       L2ProjectionL2Space(const FiniteElementSpace& fes_ho_,
-                          const FiniteElementSpace& fes_lor_);
+                          const FiniteElementSpace& fes_lor_,
+                          const bool use_ea_,
+                          MemoryType d_mt_ = Device::GetHostMemoryType());
+
+      /*Same as above but assembles and stores R_ea, P_ea */
+      void EAL2ProjectionL2Space();
+
       /// Maps <tt>x</tt>, primal field coefficients defined on a coarse mesh
       /// with a higher order L2 finite element space, to <tt>y</tt>, primal
       /// field coefficients defined on a refined mesh with a low order L2
@@ -232,6 +300,10 @@ protected:
       /// the coarse mesh. Coefficients are computed through minimization of L2
       /// error between the fields.
       void Mult(const Vector& x, Vector& y) const override;
+
+      /// Perform mult on the device (same as above)
+      void EAMult(const Vector& x, Vector& y) const;
+
       /// Maps <tt>x</tt>, dual field coefficients defined on a refined mesh
       /// with a low order L2 finite element space, to <tt>y</tt>, dual field
       /// coefficients defined on a coarse mesh with a higher order L2 finite
@@ -240,6 +312,9 @@ protected:
       /// error between the primal fields. Note, if the <tt>x</tt>-coefficients
       /// come from ProlongateTranspose, then mass is conserved.
       void MultTranspose(const Vector& x, Vector& y) const override;
+
+      void EAMultTranspose(const Vector& x, Vector& y) const;
+
       /// Maps <tt>x</tt>, primal field coefficients defined on a refined mesh
       /// with a low order L2 finite element space, to <tt>y</tt>, primal field
       /// coefficients defined on a coarse mesh with a higher order L2 finite
@@ -248,6 +323,9 @@ protected:
       /// left-inverse prolongation operation. This functionality is also
       /// provided as an Operator by L2Prolongation.
       void Prolongate(const Vector& x, Vector& y) const override;
+
+      void EAProlongate(const Vector& x, Vector& y) const;
+
       /// Maps <tt>x</tt>, dual field coefficients defined on a coarse mesh with
       /// a higher order L2 finite element space, to <tt>y</tt>, dual field
       /// coefficients defined on a refined mesh with a low order L2 finite
@@ -256,21 +334,46 @@ protected:
       /// conservative left-inverse prolongation operation. This functionality
       /// is also provided as an Operator by L2Prolongation.
       void ProlongateTranspose(const Vector& x, Vector& y) const override;
+
+      void EAProlongateTranspose(const Vector& x, Vector& y) const;
+
       void SetRelTol(real_t p_rtol_) override { } ///< No-op.
       void SetAbsTol(real_t p_atol_) override { } ///< No-op.
    };
 
+protected:
+
+   /// Class below must be public as we now have device code
+public:
+
    /** Projection operator between a H1 high-order finite element space on a
        coarse mesh, and a H1 low-order finite element space on a refined mesh
        (LOR). */
    class L2ProjectionH1Space : public L2Projection
    {
+      const bool use_ea;
+
    public:
       L2ProjectionH1Space(const FiniteElementSpace &fes_ho_,
-                          const FiniteElementSpace &fes_lor_);
+                          const FiniteElementSpace &fes_lor_,
+                          const bool use_ea_,
+                          MemoryType d_mt_ = Device::GetHostMemoryType());
 #ifdef MFEM_USE_MPI
       L2ProjectionH1Space(const ParFiniteElementSpace &pfes_ho_,
-                          const ParFiniteElementSpace &pfes_lor_);
+                          const ParFiniteElementSpace &pfes_lor_,
+                          const bool use_ea_,
+                          MemoryType d_mt_ = Device::GetHostMemoryType());
+#endif
+      /// Same as above but assembles action of R through 4 parts:
+      ///   ( )  inv( lumped(M_L) ), which is a diagonal matrix (essentially a vector)
+      ///   ( )  ElementRestrictionOperator for LOR space
+      ///   ( )  mixed mass matrix M_{LH}
+      ///   ( )  ElementRestrictionOperator for HO space
+      void EAL2ProjectionH1Space();
+
+#ifdef MFEM_USE_MPI
+      void EAL2ProjectionH1Space(const ParFiniteElementSpace &pfes_ho_,
+                                 const ParFiniteElementSpace &pfes_lor_);
 #endif
       /// Maps <tt>x</tt>, primal field coefficients defined on a coarse mesh
       /// with a higher order H1 finite element space, to <tt>y</tt>, primal
@@ -279,6 +382,7 @@ protected:
       /// the coarse mesh. Coefficients are computed through minimization of L2
       /// error between the fields.
       void Mult(const Vector& x, Vector& y) const override;
+
       /// Maps <tt>x</tt>, dual field coefficients defined on a refined mesh
       /// with a low order H1 finite element space, to <tt>y</tt>, dual field
       /// coefficients defined on a coarse mesh with a higher order H1 finite
@@ -287,6 +391,7 @@ protected:
       /// error between the primal fields. Note, if the <tt>x</tt>-coefficients
       /// come from ProlongateTranspose, then mass is conserved.
       void MultTranspose(const Vector& x, Vector& y) const override;
+
       /// Maps <tt>x</tt>, primal field coefficients defined on a refined mesh
       /// with a low order H1 finite element space, to <tt>y</tt>, primal field
       /// coefficients defined on a coarse mesh with a higher order H1 finite
@@ -295,6 +400,7 @@ protected:
       /// left-inverse prolongation operation. This functionality is also
       /// provided as an Operator by L2Prolongation.
       void Prolongate(const Vector& x, Vector& y) const override;
+
       /// Maps <tt>x</tt>, dual field coefficients defined on a coarse mesh with
       /// a higher order H1 finite element space, to <tt>y</tt>, dual field
       /// coefficients defined on a refined mesh with a low order H1 finite
@@ -303,14 +409,22 @@ protected:
       /// conservative left-inverse prolongation operation. This functionality
       /// is also provided as an Operator by L2Prolongation.
       void ProlongateTranspose(const Vector& x, Vector& y) const override;
+
+      /// Returns the inverse of an on-rank lumped mass matrix
+      void LumpedMassInverse(Vector& ML_inv) const;
+
       void SetRelTol(real_t p_rtol_) override;
       void SetAbsTol(real_t p_atol_) override;
+
    protected:
       /// Sets up the PCG solver (sets parameters, operator, and preconditioner)
       void SetupPCG();
-      /// Computes on-rank R and M_LH matrices.
+
+      /// @brief Computes on-rank R and M_LH matrices. If true, computes mixed mass and/or
+      /// inverse lumped mass matrix error when compared to device implementation.
       std::pair<std::unique_ptr<SparseMatrix>,
           std::unique_ptr<SparseMatrix>> ComputeSparseRAndM_LH();
+
       /// @brief Recovers vector of tdofs given a vector of dofs and a finite
       /// element space
       void GetTDofs(const FiniteElementSpace& fes, const Vector& x, Vector& X) const;
@@ -333,10 +447,8 @@ protected:
       void TDofsListByVDim(const FiniteElementSpace& fes,
                            int vdim,
                            Array<int>& vdofs_list) const;
-      /// Returns the inverse of an on-rank lumped mass matrix
-      void LumpedMassInverse(Vector& ML_inv) const;
+
       /// @brief Computes sparsity pattern and initializes R matrix.
-      ///
       /// Based on BilinearForm::AllocMat(), except maps between coarse HO
       /// elements and refined LOR elements.
       std::unique_ptr<SparseMatrix> AllocR();
@@ -346,10 +458,34 @@ protected:
       // The restriction operator is represented as an Operator R. The
       // prolongation operator is a dense matrix computed as the inverse of (R^T
       // M_L R), and hence, is not stored.
+      // If element assembly is enabled
       std::unique_ptr<Operator> R;
       // Used to compute P = (RT*M_LH)^(-1) M_LH^T
       std::unique_ptr<Operator> M_LH;
+      // Inverted operator in P = (RT*M_LH)^(-1) M_LH^T. Used to compute P via PCG.
       std::unique_ptr<Operator> RTxM_LH;
+      // Lumped M_L inverse operator built via EA. Wrapped with restriction maps
+      // to multiply with scalar TDof LOR vectors.
+      std::unique_ptr<Operator> ML_inv_vea;
+      // LDof Mixed mass operator built via EA. Wrapped with restrition maps to send
+      // scalar LDof HO vectors to LDof LOR vectors.
+      Operator *M_LH_local_op;
+
+      // Scalar finite element spaces for stored Tdof-to-and-from-LDof maps.
+      FiniteElementSpace* fes_ho_scalar;
+      FiniteElementSpace* fes_lor_scalar;
+      // Element Assembled mixed mass
+      Vector M_LH_ea;
+      // Element Assembled lumped M_L inverse built via EA. Stores diagonal as a Ldof vector.
+      Vector ML_inv_ea;
+
+#ifdef MFEM_USE_MPI
+      ParFiniteElementSpace* pfes_ho_scalar;
+      ParFiniteElementSpace* pfes_lor_scalar;
+      Vector RML_inv;
+#endif
+
+      friend class L2ProjectionL2Space;
    };
 
    /** Mass-conservative prolongation operator going in the opposite direction
@@ -379,7 +515,8 @@ protected:
 public:
    L2ProjectionGridTransfer(FiniteElementSpace &coarse_fes_,
                             FiniteElementSpace &fine_fes_,
-                            bool force_l2_space_ = false)
+                            bool force_l2_space_ = false,
+                            MemoryType d_mt_ = Device::GetHostMemoryType()) //move to method
       : GridTransfer(coarse_fes_, fine_fes_),
         F(NULL), B(NULL), force_l2_space(force_l2_space_)
    { }

general/array.hpp

@@ -52,10 +52,7 @@ protected:
 
    inline void GrowSize(int minsize);
 
-   static inline void TypeAssert()
-   {
-      static_assert(std::is_trivial<T>::value, "type T must be trivial");
-   }
+   static_assert(std::is_trivial<T>::value, "type T must be trivial");
 
 public:
    friend void Swap<T>(Array<T> &, Array<T> &);
@@ -95,11 +92,26 @@ public:
    template <typename CT, int N>
    explicit inline Array(const CT (&values)[N]);
 
+   /**
+    * @brief Construct a new Array object from an initializer list.
+    *
+    * @param init_list List of entities to construct from.
+    */
+   Array(const std::initializer_list<T> &init_list)
+      : Array(static_cast<int>(init_list.size()))
+   {
+      auto * it = GetData();
+      for (auto value : init_list)
+      {
+         *it++ = value;
+      }
+   }
+
    /// Move constructor ("steals" data from 'src')
    inline Array(Array<T> &&src) { Swap(src, *this); }
 
    /// Destructor
-   inline ~Array() { TypeAssert(); data.Delete(); }
+   inline ~Array() { data.Delete(); }
 
    /// Assignment operator: deep copy from 'src'.
    Array<T> &operator=(const Array<T> &src) { src.Copy(*this); return *this; }
@@ -204,6 +216,8 @@ public:
    /// Delete the whole array.
    inline void DeleteAll();
 
+   /// Reduces the capacity of the array to exactly match the current size.
+   inline void ShrinkToFit();
 
    ///  Create a copy of the internal array to the provided @a copy.
    inline void Copy(Array &copy) const;
@@ -221,6 +235,18 @@ public:
    /// Make this Array a reference to 'master'.
    inline void MakeRef(const Array &master);
 
+   /**
+    * @brief Permute the array using the provided indices. Sorts the indices
+    * variable in the process, thereby destroying the permutation. The rvalue
+    * reference is to be used when this destruction is allowed, whilst the const
+    * reference preserves at the cost of duplication.
+    *
+    * @param indices The indices of the ordering. data[i] = data[indices[i]].
+    */
+   template <typename I>
+   inline void Permute(I &&indices);
+   template <typename I>
+   inline void Permute(const I &indices) { Permute(I(indices)); }
 
    /// Copy sub array starting from @a offset out to the provided @a sa.
    inline void GetSubArray(int offset, int sa_size, Array<T> &sa) const;
@@ -275,6 +301,9 @@ public:
    /// Return 1 if the array is sorted from lowest to highest.  Otherwise return 0.
    int IsSorted() const;
 
+   /// Does the Array have Size zero.
+   bool IsEmpty() const { return Size() == 0; }
+
    /// Fill the entries of the array with the cumulative sum of the entries.
    void PartialSum();
 
@@ -492,6 +521,8 @@ public:
    BlockArray(int block_size = 16*1024);
    BlockArray(const BlockArray<T> &other); // deep copy
    BlockArray& operator=(const BlockArray&) = delete; // not supported
+   BlockArray(BlockArray<T> &&other) = default;
+   BlockArray& operator=(BlockArray<T> &&other) = default;
    ~BlockArray() { Destroy(); }
 
    /// Allocate and construct a new item in the array, return its index.
@@ -613,6 +644,8 @@ public:
 
    iterator begin() { return size ? iterator(this) : iterator(true); }
    iterator end() { return iterator(); }
+   const_iterator begin() const { return cbegin(); }
+   const_iterator end() const { return cend(); }
 
    const_iterator cbegin() const
    { return size ? const_iterator(this) : const_iterator(true); }
@@ -685,6 +718,35 @@ inline void Array<T>::GrowSize(int minsize)
    data = p;
 }
 
+template <typename T>
+inline void Array<T>::ShrinkToFit()
+{
+   if (Capacity() == size) { return; }
+   Memory<T> p(size, data.GetMemoryType());
+   p.CopyFrom(data, size);
+   p.UseDevice(data.UseDevice());
+   data.Delete();
+   data = p;
+}
+
+template <typename T>
+template <typename I>
+inline void Array<T>::Permute(I &&indices)
+{
+   for (int i = 0; i < size; i++)
+   {
+      auto current = i;
+      while (i != indices[current])
+      {
+         auto next = indices[current];
+         std::swap(data[current], data[next]);
+         indices[current] = current;
+         current = next;
+      }
+      indices[current] = current;
+   }
+}
+
 template <typename T> template <typename CT>
 inline Array<T> &Array<T>::operator=(const Array<CT> &src)
 {

general/forall.hpp

@@ -838,6 +838,16 @@ inline void hypre_forall(int N, lambda &&body)
 #endif
 }
 
+// Return the most general MemoryClass that can be used with mfem::hypre_forall
+// kernels. The returned MemoryClass is the same as the one returned by
+// GerHypreMemoryClass() except when hypre is configured to use UVM, in which
+// case this function returns MemoryClass::HOST or MemoryClass::DEVICE depending
+// on the result of HypreUsingGPU().
+inline MemoryClass GetHypreForallMemoryClass()
+{
+   return HypreUsingGPU() ? MemoryClass::DEVICE : MemoryClass::HOST;
+}
+
 #endif // MFEM_USE_MPI
 
 } // namespace mfem

general/mem_manager.cpp

@@ -408,8 +408,26 @@ class UvmHostMemorySpace : public HostMemorySpace
 {
 public:
    UvmHostMemorySpace(): HostMemorySpace() { }
-   void Alloc(void **ptr, size_t bytes) override { CuMallocManaged(ptr, bytes == 0 ? 8 : bytes); }
-   void Dealloc(void *ptr) override { CuMemFree(ptr); }
+
+   void Alloc(void **ptr, size_t bytes) override
+   {
+#ifdef MFEM_USE_CUDA
+      CuMallocManaged(ptr, bytes == 0 ? 8 : bytes);
+#endif
+#ifdef MFEM_USE_HIP
+      HipMallocManaged(ptr, bytes == 0 ? 8 : bytes);
+#endif
+   }
+
+   void Dealloc(void *ptr) override
+   {
+#ifdef MFEM_USE_CUDA
+      CuMemFree(ptr);
+#endif
+#ifdef MFEM_USE_HIP
+      HipMemFree(ptr);
+#endif
+   }
 };
 
 /// The 'No' device memory space
@@ -504,6 +522,25 @@ public:
    }
 };
 
+class UvmHipMemorySpace : public DeviceMemorySpace
+{
+public:
+   void Alloc(Memory &base) { base.d_ptr = base.h_ptr; }
+   void Dealloc(Memory&) { }
+   void *HtoD(void *dst, const void *src, size_t bytes)
+   {
+      if (dst == src) { MFEM_STREAM_SYNC; return dst; }
+      return HipMemcpyHtoD(dst, src, bytes);
+   }
+   void *DtoD(void* dst, const void* src, size_t bytes)
+   { return HipMemcpyDtoD(dst, src, bytes); }
+   void *DtoH(void *dst, const void *src, size_t bytes)
+   {
+      if (dst == src) { MFEM_STREAM_SYNC; return dst; }
+      return HipMemcpyDtoH(dst, src, bytes);
+   }
+};
+
 /// The MMU device memory space
 class MmuDeviceMemorySpace : public DeviceMemorySpace
 {
@@ -661,7 +698,15 @@ public:
 
       // Filling the device memory backends, shifting with the device size
       constexpr int shift = DeviceMemoryType;
+#if defined(MFEM_USE_CUDA)
       device[static_cast<int>(MT::MANAGED)-shift] = new UvmCudaMemorySpace();
+#elif defined(MFEM_USE_HIP)
+      device[static_cast<int>(MT::MANAGED)-shift] = new UvmHipMemorySpace();
+#else
+      // this re-creates the original behavior, but should this be nullptr instead?
+      device[static_cast<int>(MT::MANAGED)-shift] = new UvmCudaMemorySpace();
+#endif
+
       // All other devices controllers are delayed
       device[static_cast<int>(MemoryType::DEVICE)-shift] = nullptr;
       device[static_cast<int>(MT::DEVICE_DEBUG)-shift] = nullptr;
@@ -1193,8 +1238,9 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
       {
          if (dst_h_ptr != src_d_ptr && bytes != 0)
          {
-            internal::Memory &src_d_base = maps->memories.at(src_h_ptr);
-            MemoryType src_d_mt = src_d_base.d_mt;
+            MemoryType src_d_mt = (src_flags & Mem::ALIAS) ?
+                                  maps->aliases.at(src_h_ptr).mem->d_mt :
+                                  maps->memories.at(src_h_ptr).d_mt;
             ctrl->Device(src_d_mt)->DtoH(dst_h_ptr, src_d_ptr, bytes);
          }
       }
@@ -1254,9 +1300,10 @@ void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
       const void *src_d_ptr = (src_flags & Mem::ALIAS) ?
                               mm.GetAliasDevicePtr(src_h_ptr, bytes, false) :
                               mm.GetDevicePtr(src_h_ptr, bytes, false);
-      const internal::Memory &base = maps->memories.at(dest_h_ptr);
-      const MemoryType d_mt = base.d_mt;
-      ctrl->Device(d_mt)->DtoH(dest_h_ptr, src_d_ptr, bytes);
+      MemoryType src_d_mt = (src_flags & Mem::ALIAS) ?
+                            maps->aliases.at(src_h_ptr).mem->d_mt :
+                            maps->memories.at(src_h_ptr).d_mt;
+      ctrl->Device(src_d_mt)->DtoH(dest_h_ptr, src_d_ptr, bytes);
    }
 }
 
@@ -1283,9 +1330,10 @@ void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
       void *dest_d_ptr = (dest_flags & Mem::ALIAS) ?
                          mm.GetAliasDevicePtr(dest_h_ptr, bytes, false) :
                          mm.GetDevicePtr(dest_h_ptr, bytes, false);
-      const internal::Memory &base = maps->memories.at(dest_h_ptr);
-      const MemoryType d_mt = base.d_mt;
-      ctrl->Device(d_mt)->HtoD(dest_d_ptr, src_h_ptr, bytes);
+      MemoryType dest_d_mt = (dest_flags & Mem::ALIAS) ?
+                             maps->aliases.at(dest_h_ptr).mem->d_mt :
+                             maps->memories.at(dest_h_ptr).d_mt;
+      ctrl->Device(dest_d_mt)->HtoD(dest_d_ptr, src_h_ptr, bytes);
    }
    dest_flags = dest_flags &
                 ~(dest_on_host ? Mem::VALID_DEVICE : Mem::VALID_HOST);

general/sets.cpp

@@ -15,80 +15,45 @@
 namespace mfem
 {
 
-IntegerSet::IntegerSet(IntegerSet &s)
-   : me(s.me.Size())
+int IntegerSet::PickRandomElement() const
 {
-   for (int i = 0; i < me.Size(); i++)
-   {
-      me[i] = s.me[i];
-   }
-}
-
-IntegerSet& IntegerSet::operator=(const IntegerSet &s)
-{
-   me.SetSize(s.me.Size());
-   for (int i = 0; i < me.Size(); i++)
-   {
-      me[i] = s.me[i];
-   }
-   return *this;
-}
-
-int IntegerSet::operator== (IntegerSet &s)
-{
-   if (me.Size() != s.me.Size())
-   {
-      return 0;
-   }
-
-   for (int i = 0; i < me.Size(); i++)
-      if (me[i] != s.me[i])
-      {
-         return 0;
-      }
-
-   return 1;
-}
-
-int IntegerSet::PickRandomElement()
-{
-   int i, size = me.Size();
+   int i, size = Size();
    unsigned int seed = 0;
 
    for (i = 0; i < size; i++)
    {
-      seed += me[i];
+      seed += data[i];
    }
 
    srand(seed);
 
-   return me[rand()/(RAND_MAX/size)];
+   return data[rand()/(RAND_MAX/size)];
 }
 
 void IntegerSet::Recreate(const int n, const int *p)
 {
    int i, j;
 
-   me.SetSize(n);
+   SetSize(n);
 
    for (i = 0; i < n; i++)
    {
-      me[i] = p[i];
+      data[i] = p[i];
    }
 
-   me.Sort();
+   Sort();
 
    for (j = 0, i = 1; i < n; i++)
-      if (me[i] != me[j])
+      if (data[i] != data[j])
       {
-         me[++j] = me[i];
+         data[++j] = data[i];
       }
 
-   me.SetSize(j+1);
+   SetSize(j+1);
 }
 
 
-int ListOfIntegerSets::Insert(IntegerSet &s)
+int ListOfIntegerSets::Insert(const IntegerSet &s)
 {
    for (int i = 0; i < TheList.Size(); i++)
       if (*TheList[i] == s)
@@ -101,7 +66,7 @@ int ListOfIntegerSets::Insert(IntegerSet &s)
    return TheList.Size()-1;
 }
 
-int ListOfIntegerSets::Lookup(IntegerSet &s)
+int ListOfIntegerSets::Lookup(const IntegerSet &s) const
 {
    for (int i = 0; i < TheList.Size(); i++)
       if (*TheList[i] == s)
@@ -113,7 +78,7 @@ int ListOfIntegerSets::Lookup(IntegerSet &s)
    return -1;
 }
 
-void ListOfIntegerSets::AsTable(Table & t)
+void ListOfIntegerSets::AsTable(Table & t) const
 {
    int i;
 

general/sets.hpp

@@ -20,38 +20,26 @@ namespace mfem
 {
 
 /// A set of integers
-class IntegerSet
+class IntegerSet : public Array<int>
 {
-private:
-   Array<int> me;
-
 public:
-   /// Create an empty set.
-   IntegerSet() { }
-
-   /// Create a copy of set 's'.
-   IntegerSet(IntegerSet &s);
+   using Array<int>::Array; ///< Inherit all Array constructors.
+   // MSVC fails to recognize that rule of zero applies after using base class
+   // constructors.
+   IntegerSet() = default; ///< Default construct and empty set.
+   IntegerSet(const IntegerSet &) = default; ///< Copy constructor.
+   IntegerSet(IntegerSet &&) = default; ///< Move constructor.
+   IntegerSet& operator=(const IntegerSet &) = default; ///< Copy assignment.
+   IntegerSet& operator=(IntegerSet &&) = default; ///< Move assignment.
 
    /// Create an integer set from C-array 'p' of 'n' integers.
    IntegerSet(const int n, const int *p) { Recreate(n, p); }
 
-   /// Return the size of the set.
-   int Size() { return me.Size(); }
-
-   /// Return a reference to the sorted array of all the set entries.
-   operator Array<int>& () { return me; }
-
    /// Return the value of the lowest element of the set.
-   int PickElement() { return me[0]; }
+   int PickElement() const { return data[0]; }
 
    /// Return the value of a random element of the set.
-   int PickRandomElement();
-
-   /// Create a copy of set 's'.
-   IntegerSet& operator=(const IntegerSet &s);
-
-   /// Return 1 if the sets are equal and 0 otherwise.
-   int operator==(IntegerSet &s);
+   int PickRandomElement() const;
 
    /** @brief Create an integer set from C-array 'p' of 'n' integers.
        Overwrites any existing set data. */
@@ -67,25 +55,25 @@ private:
 public:
 
    /// Return the number of integer sets in the list.
-   int Size() { return TheList.Size(); }
+   int Size() const { return TheList.Size(); }
 
    /// Return the value of the first element of the ith set.
-   int PickElementInSet(int i) { return TheList[i]->PickElement(); }
+   int PickElementInSet(int i) const { return TheList[i]->PickElement(); }
 
    /// Return a random value from the ith set in the list.
-   int PickRandomElementInSet(int i) { return TheList[i]->PickRandomElement(); }
+   int PickRandomElementInSet(int i) const { return TheList[i]->PickRandomElement(); }
 
    /** @brief Check to see if set 's' is in the list. If not append it to the
        end of the list. Returns the index of the list where set 's' can be
        found. */
-   int Insert(IntegerSet &s);
+   int Insert(const IntegerSet &s);
 
    /** Return the index of the list where set 's' can be found. Returns -1 if
        not found. */
-   int Lookup(IntegerSet &s);
+   int Lookup(const IntegerSet &s) const;
 
    /// Write the list of sets into table 't'.
-   void AsTable(Table &t);
+   void AsTable(Table &t) const;
 
    ~ListOfIntegerSets();
 };

linalg/batched/batched.cpp

@@ -52,9 +52,9 @@ BatchedLinAlg &BatchedLinAlg::Instance()
 }
 
 void BatchedLinAlg::AddMult(const DenseTensor &A, const Vector &x, Vector &y,
-                            real_t alpha, real_t beta)
+                            real_t alpha, real_t beta, Op op)
 {
-   Get(Instance().active_backend).AddMult(A, x, y, alpha, beta);
+   Get(Instance().active_backend).AddMult(A, x, y, alpha, beta, op);
 }
 
 void BatchedLinAlg::Mult(const DenseTensor &A, const Vector &x, Vector &y)
@@ -62,6 +62,12 @@ void BatchedLinAlg::Mult(const DenseTensor &A, const Vector &x, Vector &y)
    Get(Instance().active_backend).Mult(A, x, y);
 }
 
+void BatchedLinAlg::MultTranspose(const DenseTensor &A, const Vector &x,
+                                  Vector &y)
+{
+   Get(Instance().active_backend).MultTranspose(A, x, y);
+}
+
 void BatchedLinAlg::Invert(DenseTensor &A)
 {
    Get(Instance().active_backend).Invert(A);
@@ -107,4 +113,10 @@ void BatchedLinAlgBase::Mult(const DenseTensor &A, const Vector &x,
    AddMult(A, x, y, 1.0, 0.0);
 }
 
+void BatchedLinAlgBase::MultTranspose(const DenseTensor &A, const Vector &x,
+                                      Vector &y) const
+{
+   AddMult(A, x, y, 1.0, 0.0, Op::T);
+}
+
 }

linalg/batched/batched.hpp

@@ -48,6 +48,14 @@ public:
       /// Counter for the number of backends.
       NUM_BACKENDS
    };
+
+   /// Operation type (transposed or not transposed)
+   enum Op
+   {
+      N, ///< Not transposed.
+      T  ///< Transposed.
+   };
+
 private:
    /// All available backends. Unavailble backends will be nullptr.
    std::array<std::unique_ptr<class BatchedLinAlgBase>,
@@ -58,15 +66,19 @@ private:
    /// Return the singleton instance.
    static BatchedLinAlg &Instance();
 public:
-   /// @brief Computes $y = \alpha A x + \beta y$.
+   /// @brief Computes $y = \alpha A^{op} x + \beta y$.
    ///
+   /// $A^{op}$ is either $A$ or $A^T$ depending on the value of @a op.
    /// $A$ is a block diagonal matrix, represented by the DenseTensor @a A with
-   /// shape (m, n, n_mat). $x$ has shape (n, k, n_mat), and $y$ has shape
-   /// (m, k, n_mat).
+   /// shape (m, n, n_mat). $x$ has shape (tr?m:n, k, n_mat), and $y$ has shape
+   /// (tr?n:m, k, n_mat), where 'tr' is true in the transposed case.
    static void AddMult(const DenseTensor &A, const Vector &x, Vector &y,
-                       real_t alpha = 1.0, real_t beta = 1.0);
-   /// Computes $y = A x$ (e.g. by calling @ref AddMult "AddMult(A,x,y,1,0)").
+                       real_t alpha = 1.0, real_t beta = 1.0,
+                       Op op = Op::N);
+   /// Computes $y = A x$ (e.g. by calling @ref AddMult "AddMult(A,x,y,1,0,Op::N)").
    static void Mult(const DenseTensor &A, const Vector &x, Vector &y);
+   /// Computes $y = A^T x$ (e.g. by calling @ref AddMult "AddMult(A,x,y,1,0,Op::T)").
+   static void MultTranspose(const DenseTensor &A, const Vector &x, Vector &y);
    /// @brief Replaces the block diagonal matrix $A$ with its inverse $A^{-1}$.
    ///
    /// $A$ is represented by the DenseTensor @a A with shape (m, m, n_mat).
@@ -109,11 +121,16 @@ public:
 class BatchedLinAlgBase
 {
 public:
+   using Op = BatchedLinAlg::Op;
    /// See BatchedLinAlg::AddMult.
    virtual void AddMult(const DenseTensor &A, const Vector &x, Vector &y,
-                        real_t alpha = 1.0, real_t beta = 1.0) const = 0;
+                        real_t alpha = 1.0, real_t beta = 1.0,
+                        Op op = Op::N) const = 0;
    /// See BatchedLinAlg::Mult.
    virtual void Mult(const DenseTensor &A, const Vector &x, Vector &y) const;
+   /// See BatchedLinAlg::MultTranspose.
+   virtual void MultTranspose(const DenseTensor &A, const Vector &x,
+                              Vector &y) const;
    /// See BatchedLinAlg::Invert.
    virtual void Invert(DenseTensor &A) const = 0;
    /// See BatchedLinAlg::LUFactor.

linalg/batched/gpu_blas.cpp

@@ -82,23 +82,27 @@ void GPUBlas::DisableAtomics()
 }
 
 void GPUBlasBatchedLinAlg::AddMult(const DenseTensor &A, const Vector &x,
-                                   Vector &y, real_t alpha, real_t beta) const
+                                   Vector &y, real_t alpha, real_t beta,
+                                   Op op) const
 {
-   const int m = A.SizeI();
-   const int n = A.SizeJ();
+   const bool tr = (op == Op::T);
+
+   const int m = tr ? A.SizeJ() : A.SizeI();
+   const int n = tr ? A.SizeI() : A.SizeJ();
    const int n_mat = A.SizeK();
    const int k = x.Size() / n / n_mat;
 
-   auto d_A = mfem::Reshape(A.Read(), m, n, n_mat);
-   auto d_x = mfem::Reshape(x.Read(), n, k, n_mat);
-   auto d_y = mfem::Reshape(beta == 0.0 ? y.Write() : y.ReadWrite(), m, k, n_mat);
+   auto d_A = A.Read();
+   auto d_x = x.Read(); // Shape: (n, k, n_mat)
+   auto d_y = beta == 0.0 ? y.Write() : y.ReadWrite(); // Shape (m, k, n_mat)
 
-   const auto op = MFEM_CU_or_HIP(BLAS_OP_N);
+   const auto op_A = tr ? MFEM_CU_or_HIP(BLAS_OP_T) : MFEM_CU_or_HIP(BLAS_OP_N);
+   const auto op_B = MFEM_CU_or_HIP(BLAS_OP_N);
 
    const blasStatus_t status = MFEM_GPUBLAS_PREFIX(gemmStridedBatched)(
-                                  GPUBlas::Handle(), op, op, m, k, n, &alpha,
-                                  d_A, m, m*n, d_x, n, n*k, &beta, d_y, m, m*k,
-                                  n_mat);
+                                  GPUBlas::Handle(), op_A, op_B, m, k, n,
+                                  &alpha, d_A, m, m*n, d_x, n, n*k, &beta, d_y,
+                                  m, m*k, n_mat);
    MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
 }
 

linalg/batched/gpu_blas.hpp

@@ -57,7 +57,8 @@ class GPUBlasBatchedLinAlg : public BatchedLinAlgBase
 {
 public:
    void AddMult(const DenseTensor &A, const Vector &x, Vector &y,
-                real_t alpha = 1.0, real_t beta = 1.0) const override;
+                real_t alpha = 1.0, real_t beta = 1.0,
+                Op op = Op::N) const override;
    void Invert(DenseTensor &A) const override;
    void LUFactor(DenseTensor &A, Array<int> &P) const override;
    void LUSolve(const DenseTensor &LU, const Array<int> &P,

linalg/batched/magma.cpp

@@ -54,19 +54,24 @@ magma_queue_t Magma::Queue()
 }
 
 void MagmaBatchedLinAlg::AddMult(const DenseTensor &A, const Vector &x,
-                                 Vector &y, real_t alpha, real_t beta) const
+                                 Vector &y, real_t alpha, real_t beta,
+                                 Op op) const
 {
-   const int m = A.SizeI();
-   const int n = A.SizeJ();
+   const bool tr = (op == Op::T);
+
+   const int m = tr ? A.SizeJ() : A.SizeI();
+   const int n = tr ? A.SizeI() : A.SizeJ();
    const int n_mat = A.SizeK();
    const int k = x.Size() / n / n_mat;
 
-   auto d_A = mfem::Reshape(A.Read(), m, n, n_mat);
-   auto d_x = mfem::Reshape(x.Read(), n, k, n_mat);
-   auto d_y = mfem::Reshape(beta == 0.0 ? y.Write() : y.ReadWrite(), m, k, n_mat);
+   auto d_A = A.Read();
+   auto d_x = x.Read(); // Shape (n, k, n_mat);
+   auto d_y = beta == 0.0 ? y.Write() : y.ReadWrite(); // Shape (m, k, n_mat);
+
+   magma_trans_t magma_op = tr ? MagmaNoTrans : MagmaTrans;
 
    MFEM_MAGMABLAS_PREFIX(gemm_batched_strided)(
-      MagmaNoTrans, MagmaNoTrans, m, k, n, alpha, d_A, m, m*n, d_x, n, n*k,
+      magma_op, MagmaNoTrans, m, k, n, alpha, d_A, m, m*n, d_x, n, n*k,
       beta, d_y, m, m*k, n_mat, Magma::Queue());
 }
 

linalg/batched/magma.hpp

@@ -25,7 +25,8 @@ class MagmaBatchedLinAlg : public BatchedLinAlgBase
 {
 public:
    void AddMult(const DenseTensor &A, const Vector &x, Vector &y,
-                real_t alpha = 1.0, real_t beta = 1.0) const override;
+                real_t alpha = 1.0, real_t beta = 1.0,
+                Op op = Op::N) const override;
    void Invert(DenseTensor &A) const override;
    void LUFactor(DenseTensor &A, Array<int> &P) const override;
    void LUSolve(const DenseTensor &A, const Array<int> &P,

linalg/batched/native.cpp

@@ -18,22 +18,37 @@ namespace mfem
 {
 
 void NativeBatchedLinAlg::AddMult(const DenseTensor &A, const Vector &x,
-                                  Vector &y, real_t alpha, real_t beta) const
+                                  Vector &y, real_t alpha, real_t beta,
+                                  Op op) const
 {
+   const bool tr = (op == Op::T);
+
    const int m = A.SizeI();
    const int n = A.SizeJ();
    const int n_mat = A.SizeK();
-   const int k = x.Size() / n / n_mat;
+   const int k = x.Size() / (tr ? m : n) / n_mat;
 
-   auto d_A = mfem::Reshape(A.Read(), m, n, n_mat);
-   auto d_x = mfem::Reshape(x.Read(), n, k, n_mat);
-   auto d_y = mfem::Reshape(beta == 0.0 ? y.Write() : y.ReadWrite(), m, k, n_mat);
+   auto d_A = Reshape(A.Read(), m, n, n_mat);
+   auto d_x = Reshape(x.Read(), (tr ? m : n), k, n_mat);
+   auto d_y = Reshape(beta == 0.0 ? y.Write() : y.ReadWrite(),
+                      (tr ? n : m), k, n_mat);
 
-   mfem::forall(n_mat, [=] MFEM_HOST_DEVICE (int i)
+   if (tr)
    {
-      kernels::AddMult(m, k, n, &d_A(0,0,i), &d_x(0,0,i), &d_y(0,0,i),
-                       alpha, beta);
-   });
+      mfem::forall(n_mat, [=] MFEM_HOST_DEVICE (int i)
+      {
+         kernels::AddMultAtB(m, n, k, &d_A(0,0,i), &d_x(0,0,i), &d_y(0,0,i),
+                             alpha, beta);
+      });
+   }
+   else
+   {
+      mfem::forall(n_mat, [=] MFEM_HOST_DEVICE (int i)
+      {
+         kernels::AddMult(m, k, n, &d_A(0,0,i), &d_x(0,0,i), &d_y(0,0,i),
+                          alpha, beta);
+      });
+   }
 
    // Alternative approach, threading also over the second index. Which one is
    // better?
@@ -48,70 +63,103 @@ void NativeBatchedLinAlg::AddMult(const DenseTensor &A, const Vector &x,
 
 void NativeBatchedLinAlg::Invert(DenseTensor &A) const
 {
-   MFEM_ABORT("");
+   const int m = A.SizeI();
+   const int NE = A.SizeK();
+   DenseTensor LU = A;
+   Array<int> P(m*NE);
+
+   LUFactor(LU, P);
+
+   auto data_all = Reshape(LU.Read(), m, m, NE);
+   auto piv_all  = Reshape(P.Read(), m, NE);
+   auto inv_all  = Reshape(A.Write(), m, m, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      // A^{-1} = U^{-1} L^{-1} P
+      // X <- U^{-1} (set only the upper triangular part of X)
+      real_t *X          = &inv_all(0, 0, e);
+      real_t *x          = X;
+      const real_t *data = &data_all(0, 0, e);
+      const int *ipiv    = &piv_all(0, e);
+
+      for (int k = 0; k < m; k++)
+      {
+         const real_t minus_x_k = -(x[k] = 1.0 / data[k + k * m]);
+         for (int i = 0; i < k; i++)
+         {
+            x[i] = data[i + k * m] * minus_x_k;
+         }
+         for (int j = k - 1; j >= 0; j--)
+         {
+            const real_t x_j = (x[j] /= data[j + j * m]);
+            for (int i = 0; i < j; i++)
+            {
+               x[i] -= data[i + j * m] * x_j;
+            }
+         }
+         x += m;
+      }
+
+      // X <- X L^{-1} (use input only from the upper triangular part of X)
+      {
+         int k = m - 1;
+         for (int j = 0; j < k; j++)
+         {
+            const real_t minus_L_kj = -data[k + j * m];
+            for (int i = 0; i <= j; i++)
+            {
+               X[i + j * m] += X[i + k * m] * minus_L_kj;
+            }
+            for (int i = j + 1; i < m; i++)
+            {
+               X[i + j * m] = X[i + k * m] * minus_L_kj;
+            }
+         }
+      }
+      for (int k = m - 2; k >= 0; k--)
+      {
+         for (int j = 0; j < k; j++)
+         {
+            const real_t L_kj = data[k + j * m];
+            for (int i = 0; i < m; i++)
+            {
+               X[i + j * m] -= X[i + k * m] * L_kj;
+            }
+         }
+      }
+
+      // X <- X P
+      for (int k = m - 1; k >= 0; k--)
+      {
+         const int piv_k = ipiv[k];
+         if (k != piv_k)
+         {
+            for (int i = 0; i < m; i++)
+            {
+               kernels::internal::Swap(X[i + k * m], X[i + piv_k * m]);
+            }
+         }
+      }
+   });
 }
 
 void NativeBatchedLinAlg::LUFactor(DenseTensor &A, Array<int> &P) const
 {
-   constexpr real_t tol = 0.0; // Make this user-adjustable?
    const int m = A.SizeI();
    const int NE = A.SizeK();
    P.SetSize(m*NE);
 
-   auto data_all = mfem::Reshape(A.ReadWrite(), m, m, NE);
-   auto ipiv_all = mfem::Reshape(P.Write(), m, NE);
+   auto data_all = Reshape(A.ReadWrite(), m, m, NE);
+   auto ipiv_all = Reshape(P.Write(), m, NE);
    Array<bool> pivot_flag(1);
    pivot_flag[0] = true;
    bool *d_pivot_flag = pivot_flag.ReadWrite();
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      for (int i = 0; i < m; i++)
-      {
-         // pivoting
-         {
-            int piv = i;
-            real_t a = fabs(data_all(piv,i,e));
-            for (int j = i+1; j < m; j++)
-            {
-               const real_t b = fabs(data_all(j,i,e));
-               if (b > a)
-               {
-                  a = b;
-                  piv = j;
-               }
-            }
-            ipiv_all(i,e) = piv;
-            if (piv != i)
-            {
-               // swap rows i and piv in both L and U parts
-               for (int j = 0; j < m; j++)
-               {
-                  mfem::kernels::internal::Swap<real_t>(data_all(i,j,e), data_all(piv,j,e));
-               }
-            }
-         } // pivot end
-
-         if (abs(data_all(i,i,e)) <= tol)
-         {
-            d_pivot_flag[0] = false;
-         }
-
-         const real_t a_ii_inv = 1.0 / data_all(i,i,e);
-         for (int j = i+1; j < m; j++)
-         {
-            data_all(j,i,e) *= a_ii_inv;
-         }
-
-         for (int k = i+1; k < m; k++)
-         {
-            const real_t a_ik = data_all(i,k,e);
-            for (int j = i+1; j < m; j++)
-            {
-               data_all(j,k,e) -= a_ik * data_all(j,i,e);
-            }
-         }
-      }
+      const bool flag = kernels::LUFactor(&data_all(0,0,e), m, &ipiv_all(0,e));
+      if (!flag) { d_pivot_flag[0] = false; }
    });
 
    MFEM_VERIFY(pivot_flag.HostRead()[0], "Batch LU factorization failed");
@@ -124,9 +172,9 @@ void NativeBatchedLinAlg::LUSolve(const DenseTensor &LU, const Array<int> &P,
    const int n_mat = LU.SizeK();
    const int n_rhs = x.Size() / m / n_mat;
 
-   auto d_LU = mfem::Reshape(LU.Read(), m, m, n_mat);
-   auto d_P = mfem::Reshape(P.Read(), m, n_mat);
-   auto d_x = mfem::Reshape(x.Write(), m, n_rhs, n_mat);
+   auto d_LU = Reshape(LU.Read(), m, m, n_mat);
+   auto d_P = Reshape(P.Read(), m, n_mat);
+   auto d_x = Reshape(x.Write(), m, n_rhs, n_mat);
 
    mfem::forall(n_mat * n_rhs, [=] MFEM_HOST_DEVICE (int idx)
    {

linalg/batched/native.hpp

@@ -21,7 +21,7 @@ class NativeBatchedLinAlg : public BatchedLinAlgBase
 {
 public:
    void AddMult(const DenseTensor &A, const Vector &x, Vector &y,
-                real_t alpha, real_t beta) const override;
+                real_t alpha, real_t beta, Op op) const override;
    void Invert(DenseTensor &A) const override;
    void LUFactor(DenseTensor &A, Array<int> &P) const override;
    void LUSolve(const DenseTensor &LU, const Array<int> &P,

linalg/densemat.cpp

@@ -3487,20 +3487,7 @@ void LUFactors::LSolve(int m, int n, real_t *X) const
    real_t *x = X;
    for (int k = 0; k < n; k++)
    {
-      // X <- P X
-      for (int i = 0; i < m; i++)
-      {
-         mfem::Swap<real_t>(x[i], x[ipiv[i]-ipiv_base]);
-      }
-      // X <- L^{-1} X
-      for (int j = 0; j < m; j++)
-      {
-         const real_t x_j = x[j];
-         for (int i = j+1; i < m; i++)
-         {
-            x[i] -= data[i+j*m] * x_j;
-         }
-      }
+      kernels::LSolve(data, m, ipiv, x);
       x += m;
    }
 }
@@ -3508,17 +3495,9 @@ void LUFactors::LSolve(int m, int n, real_t *X) const
 void LUFactors::USolve(int m, int n, real_t *X) const
 {
    real_t *x = X;
-   // X <- U^{-1} X
    for (int k = 0; k < n; k++)
    {
-      for (int j = m-1; j >= 0; j--)
-      {
-         const real_t x_j = ( x[j] /= data[j+j*m] );
-         for (int i = 0; i < j; i++)
-         {
-            x[i] -= data[i+j*m] * x_j;
-         }
-      }
+      kernels::USolve(data, m, x);
       x += m;
    }
 }
@@ -3661,44 +3640,13 @@ void LUFactors::GetInverseMatrix(int m, real_t *X) const
 void LUFactors::SubMult(int m, int n, int r, const real_t *A21,
                         const real_t *X1, real_t *X2)
 {
-   // X2 <- X2 - A21 X1
-   for (int k = 0; k < r; k++)
-   {
-      for (int j = 0; j < m; j++)
-      {
-         const real_t x1_jk = X1[j+k*m];
-         for (int i = 0; i < n; i++)
-         {
-            X2[i+k*n] -= A21[i+j*n] * x1_jk;
-         }
-      }
-   }
+   kernels::SubMult(m, n, r, A21, X1, X2);
 }
 
 void LUFactors::BlockFactor(
    int m, int n, real_t *A12, real_t *A21, real_t *A22) const
 {
-   // A12 <- L^{-1} P A12
-   LSolve(m, n, A12);
-   // A21 <- A21 U^{-1}
-   for (int j = 0; j < m; j++)
-   {
-      const real_t u_jj_inv = 1.0/data[j+j*m];
-      for (int i = 0; i < n; i++)
-      {
-         A21[i+j*n] *= u_jj_inv;
-      }
-      for (int k = j+1; k < m; k++)
-      {
-         const real_t u_jk = data[j+k*m];
-         for (int i = 0; i < n; i++)
-         {
-            A21[i+k*n] -= A21[i+j*n] * u_jk;
-         }
-      }
-   }
-   // A22 <- A22 - A21 A12
-   SubMult(m, n, n, A21, A12, A22);
+   kernels::BlockFactor(data, m, ipiv, n, A12, A21, A22);
 }
 
 void LUFactors::BlockForwSolve(int m, int n, int r, const real_t *L21,

linalg/densemat.hpp

@@ -1174,6 +1174,31 @@ public:
       tdata.Wrap(ext_data, i*j*k, false);
    }
 
+   /// @brief Reset the DenseTensor to use the given external Memory @a mem and
+   /// dimensions @a i, @a j, and @a k.
+   ///
+   /// If @a own_mem is false, the DenseTensor will not own any of the pointers
+   /// of @a mem.
+   ///
+   /// Note that when @a own_mem is true, the @a mem object can be destroyed
+   /// immediately by the caller but `mem.Delete()` should NOT be called since
+   /// the DenseTensor object takes ownership of all pointers owned by @a mem.
+   void NewMemoryAndSize(const Memory<real_t> &mem, int i, int j, int k,
+                         bool own_mem)
+   {
+      tdata.Delete();
+      Mk.UseExternalData(NULL, i, j);
+      nk = k;
+      if (own_mem)
+      {
+         tdata = mem;
+      }
+      else
+      {
+         tdata.MakeAlias(mem, 0, i*j*k);
+      }
+   }
+
    /// Sets the tensor elements equal to constant c
    DenseTensor &operator=(real_t c);
 
@@ -1295,7 +1320,6 @@ void BatchLUFactor(DenseTensor &Mlu, Array<int> &P, const real_t TOL = 0.0);
     dimension m x n. */
 void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X);
 
-
 // Inline methods
 
 inline real_t &DenseMatrix::operator()(int i, int j)

linalg/handle.cpp

@@ -17,8 +17,8 @@
 
 // Make sure that hypre and PETSc use the same size indices.
 #if defined(MFEM_USE_MPI) && defined(MFEM_USE_PETSC)
-#if (defined(HYPRE_BIGINT) && !defined(PETSC_USE_64BIT_INDICES)) || \
-    (!defined(HYPRE_BIGINT) && defined(PETSC_USE_64BIT_INDICES))
+#if ((defined(HYPRE_BIGINT) || defined(HYPRE_MIXEDINT)) && !defined(PETSC_USE_64BIT_INDICES)) || \
+    (!defined(HYPRE_BIGINT) && !defined(HYPRE_MIXEDINT) && defined(PETSC_USE_64BIT_INDICES))
 #error HYPRE and PETSC do not use the same size integers!
 #endif
 #endif

linalg/hypre.cpp

@@ -211,6 +211,24 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
    own_ParVector = 1;
 }
 
+HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
+                               Vector &base, int offset, HYPRE_BigInt *col)
+   : HypreParVector(comm, glob_size, nullptr, col, false)
+{
+   MFEM_ASSERT(CanShallowCopy(base.GetMemory(), GetHypreMemoryClass()),
+               "the MemoryTypes of 'base' are incompatible with Hypre!");
+   MFEM_ASSERT(offset + size <= base.Size(),
+               "the size of 'base' is too small!");
+
+   data.Delete();
+   data.MakeAlias(base.GetMemory(), offset, size);
+   hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
+   hypre_VectorData(x_loc) = data.ReadWrite(GetHypreMemoryClass(), size);
+#ifdef HYPRE_USING_GPU
+   hypre_VectorMemoryLocation(x_loc) = GetHypreMemoryLocation();
+#endif
+}
+
 // Call the move constructor on the "compatible" temp vector
 HypreParVector::HypreParVector(const HypreParVector &y) : HypreParVector(
       y.CreateCompatibleVector())
@@ -1580,14 +1598,12 @@ void HypreParMatrix::GetDiag(Vector &diag) const
 {
    const int size = Height();
    diag.SetSize(size);
-   auto hypre_ml = GetHypreMemoryLocation();
    // Avoid using GetHypreMemoryClass() since it may be MemoryClass::MANAGED and
    // that may not play well with the memory types used by 'diag'.
-   MemoryClass hypre_mc = (hypre_ml == HYPRE_MEMORY_HOST) ?
-                          MemoryClass::HOST : MemoryClass::DEVICE;
+   MemoryClass hypre_mc = GetHypreForallMemoryClass();
    real_t *diag_hd = diag.GetMemory().Write(hypre_mc, size);
 #if MFEM_HYPRE_VERSION >= 21800
-   MFEM_VERIFY(A->diag->memory_location == hypre_ml,
+   MFEM_VERIFY(A->diag->memory_location == GetHypreMemoryLocation(),
                "unexpected HypreParMatrix memory location!");
 #endif
    const HYPRE_Int *A_diag_i = A->diag->i;
@@ -2494,7 +2510,7 @@ void HypreParMatrix::EliminateBC(const Array<int> &ess_dofs,
 
    const int n_ess_dofs = ess_dofs.Size();
    const auto ess_dofs_d = ess_dofs.GetMemory().Read(
-                              GetHypreMemoryClass(), n_ess_dofs);
+                              GetHypreForallMemoryClass(), n_ess_dofs);
 
    // Start communication to figure out which columns need to be eliminated in
    // the off-diagonal block
@@ -2777,6 +2793,33 @@ void HypreParMatrix::PrintHash(std::ostream &os) const
    os << "col map offd hash : " << hf.GetHash() << '\n';
 }
 
+real_t HypreParMatrix::FNorm() const
+{
+   real_t norm_fro = 0.0;
+   if (A != NULL)
+#if MFEM_HYPRE_VERSION >= 21900
+   {
+      const int ierr = hypre_ParCSRMatrixNormFro(A, &norm_fro);
+      MFEM_VERIFY(ierr == 0, "");
+   }
+#else
+   {
+      // HYPRE_USING_GPU is not defined for
+      // MFEM_HYPRE_VERSION < 22100 and so here it is
+      // guaranteed that the matrix is in "host" memory
+      Vector Avec_diag(A->diag->data, A->diag->num_nonzeros);
+      real_t normsqr_fro = InnerProduct(Avec_diag, Avec_diag);
+      Vector Avec_offd(A->offd->data, A->offd->num_nonzeros);
+      normsqr_fro += InnerProduct(Avec_offd, Avec_offd);
+      MPI_Allreduce(MPI_IN_PLACE, &normsqr_fro, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, hypre_ParCSRMatrixComm(A));
+      norm_fro = sqrt(normsqr_fro);
+   }
+#endif
+   return norm_fro;
+}
+
+
 inline void delete_hypre_ParCSRMatrixColMapOffd(hypre_ParCSRMatrix *A)
 {
    HYPRE_BigInt  *A_col_map_offd = hypre_ParCSRMatrixColMapOffd(A);

linalg/hypre.hpp

@@ -247,6 +247,12 @@ public:
        allocated in the memory location HYPRE_MEMORY_DEVICE. */
    HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size, real_t *data_,
                   HYPRE_BigInt *col, bool is_device_ptr = false);
+   /** @brief Creates a vector that uses the data of the Vector @a base,
+       starting at the given @a offset. */
+   /** The @a base Vector must have memory types compatible with the MemoryClass
+       returned by GetHypreMemoryClass(). */
+   HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size, Vector &base,
+                  int offset, HYPRE_BigInt *col);
    /// Creates a deep copy of @a y
    HypreParVector(const HypreParVector &y);
    /// Move constructor for HypreParVector. "Steals" data from its argument.
@@ -312,7 +318,8 @@ public:
    /// Sets the data of the Vector and the hypre_ParVector to @a data_.
    /** Must be used only for HypreParVector%s that do not own the data,
        e.g. created with the constructor:
-       HypreParVector(MPI_Comm, HYPRE_BigInt, double *, HYPRE_BigInt *). */
+       HypreParVector(MPI_Comm, HYPRE_BigInt, real_t *, HYPRE_BigInt *, bool).
+   */
    void SetData(real_t *data_);
 
    /** @brief Prepare the HypreParVector for read access in hypre's device
@@ -332,7 +339,7 @@ public:
        HYPRE_MEMORY_DEVICE. */
    /** This method must be used with HypreParVector%s that do not own the data,
        e.g. created with the constructor:
-       HypreParVector(MPI_Comm, HYPRE_BigInt, double *, HYPRE_BigInt *).
+       HypreParVector(MPI_Comm, HYPRE_BigInt, real_t *, HYPRE_BigInt *, bool).
 
        The Memory @a mem must be accessible with the hypre MemoryClass defined
        by GetHypreMemoryClass(). */
@@ -343,7 +350,7 @@ public:
        space, HYPRE_MEMORY_DEVICE. */
    /** This method must be used with HypreParVector%s that do not own the data,
        e.g. created with the constructor:
-       HypreParVector(MPI_Comm, HYPRE_BigInt, double *, HYPRE_BigInt *).
+       HypreParVector(MPI_Comm, HYPRE_BigInt, real_t *, HYPRE_BigInt *, bool).
 
        The Memory @a mem must be accessible with the hypre MemoryClass defined
        by GetHypreMemoryClass(). */
@@ -354,7 +361,7 @@ public:
        HYPRE_MEMORY_DEVICE. */
    /** This method must be used with HypreParVector%s that do not own the data,
        e.g. created with the constructor:
-       HypreParVector(MPI_Comm, HYPRE_BigInt, double *, HYPRE_BigInt *).
+       HypreParVector(MPI_Comm, HYPRE_BigInt, real_t *, HYPRE_BigInt *, bool).
 
        The Memory @a mem must be accessible with the hypre MemoryClass defined
        by GetHypreMemoryClass(). */
@@ -393,7 +400,7 @@ private:
    /// Auxiliary vectors for typecasting
    mutable HypreParVector *X, *Y;
    /** @brief Auxiliary buffers for the case when the input or output arrays in
-       methods like Mult(double, const Vector &, double, Vector &) need to be
+       methods like Mult(real_t, const Vector &, real_t, Vector &) need to be
        deep copied in order to be used by hypre. */
    mutable Memory<real_t> auxX, auxY;
 
@@ -938,6 +945,10 @@ public:
        without the need to save the whole matrix. */
    void PrintHash(std::ostream &out) const;
 
+   /// @brief Return the Frobenius norm of the matrix (or 0 if the underlying
+   /// hypre matrix is NULL)
+   real_t FNorm() const;
+
    /// Calls hypre's destroy function
    virtual ~HypreParMatrix() { Destroy(); }
 

linalg/kernels.hpp

@@ -402,6 +402,43 @@ void MultABt(const int Aheight, const int Awidth, const int Bheight,
    }
 }
 
+/** @brief Compute C = alpha*At*B + beta*C.
+
+    Multiply the transpose of a matrix of size @a Aheight x @a Awidth and data
+    @a Adata with a matrix of size @a Aheight x @a Bwidth and data @a Bdata. */
+template<typename TA, typename TB, typename TC>
+MFEM_HOST_DEVICE inline
+void AddMultAtB(const int Aheight, const int Awidth, const int Bwidth,
+                const TA *Adata, const TB *Bdata, TC *Cdata, const TB alpha,
+                const TA beta)
+{
+   const int aw_x_bw = Awidth * Bwidth;
+
+   if (beta == 0.0)
+   {
+      for (int i = 0; i < aw_x_bw; i++) { Cdata[i] = 0.0; }
+   }
+   else if (beta != 1.0)
+   {
+      for (int i = 0; i < aw_x_bw; i++) { Cdata[i] *= beta; }
+   }
+
+   TC *c = Cdata;
+   for (int i = 0; i < Bwidth; ++i)
+   {
+      for (int j = 0; j < Awidth; ++j)
+      {
+         TC val = 0.0;
+         for (int k = 0; k < Aheight; ++k)
+         {
+            val += alpha * Adata[j * Aheight + k] * Bdata[i * Aheight + k];
+         }
+         *c += val;
+         c++;
+      }
+   }
+}
+
 /** @brief Multiply the transpose of a matrix of size @a Aheight x @a Awidth
     and data @a Adata with a matrix of size @a Aheight x @a Bwidth and data @a
     Bdata: At * B. Return the result in a matrix with data @a AtBdata. */
@@ -409,6 +446,17 @@ template<typename TA, typename TB, typename TC>
 MFEM_HOST_DEVICE inline
 void MultAtB(const int Aheight, const int Awidth, const int Bwidth,
              const TA *Adata, const TB *Bdata, TC *AtBdata)
+{
+   AddMultAtB(Aheight, Awidth, Bwidth, Adata, Bdata, AtBdata, TB(1.0), TA(0.0));
+}
+
+/** @brief Multiply the transpose of a matrix of size @a Aheight x @a Awidth
+    and data @a Adata with a matrix of size @a Aheight x @a Bwidth and data @a
+    Bdata: At * B. Add the result to the matrix with data @a AtBdata. */
+template<typename TA, typename TB, typename TC>
+MFEM_HOST_DEVICE inline
+void AddMultAtB(const int Aheight, const int Awidth, const int Bwidth,
+                const TA *Adata, const TB *Bdata, TC *AtBdata)
 {
    TC *c = AtBdata;
    for (int i = 0; i < Bwidth; ++i)
@@ -420,7 +468,7 @@ void MultAtB(const int Aheight, const int Awidth, const int Bwidth,
          {
             val += Adata[j * Aheight + k] * Bdata[i * Aheight + k];
          }
-         *c = val;
+         *c += val;
          c++;
       }
    }
@@ -1641,24 +1689,21 @@ have_aa:
    return sqrt(fabs(aa))*mult; // take abs before we sort?
 }
 
-
-/// Assuming L.U = P.A for a factored matrix (m x m),
-//  compute x <- A x
+/// @brief Assuming L.U = P.A factored matrix of size (m x m), compute
+/// X <- L^{-1} P X, for a vector X of length m.
 //
 // @param [in] data LU factorization of A
 // @param [in] m square matrix height
-// @param [in] ipiv array storing pivot information
+// @param [in] ipiv array storing pivots
 // @param [in, out] x vector storing right-hand side and then solution
 MFEM_HOST_DEVICE
-inline void LUSolve(const real_t *data, const int m, const int *ipiv,
-                    real_t *x)
+inline void LSolve(const real_t *data, const int m, const int *ipiv, real_t *x)
 {
    // X <- P X
    for (int i = 0; i < m; i++)
    {
       internal::Swap<real_t>(x[i], x[ipiv[i]]);
    }
-
    // X <- L^{-1} X
    for (int j = 0; j < m; j++)
    {
@@ -1668,8 +1713,17 @@ inline void LUSolve(const real_t *data, const int m, const int *ipiv,
          x[i] -= data[i + j * m] * x_j;
       }
    }
+}
 
-   // X <- U^{-1} X
+/// @brief Assuming L.U = P.A factored matrix of size (m x m), compute
+/// X <- U^{-1} X, for a vector X of length m.
+//
+// @param [in] data LU factorization of A
+// @param [in] m square matrix height
+// @param [in, out] x vector storing right-hand side and then solution
+MFEM_HOST_DEVICE
+inline void USolve(const real_t *data, const int m, real_t *x)
+{
    for (int j = m - 1; j >= 0; j--)
    {
       const real_t x_j = (x[j] /= data[j + j * m]);
@@ -1680,6 +1734,148 @@ inline void LUSolve(const real_t *data, const int m, const int *ipiv,
    }
 }
 
+/// @brief Assuming L.U = P.A for a factored matrix (m x m),
+//  compute x <- A x
+//
+// @param [in] data LU factorization of A
+// @param [in] m square matrix height
+// @param [in] ipiv array storing pivot information
+// @param [in, out] x vector storing right-hand side and then solution
+MFEM_HOST_DEVICE
+inline void LUSolve(const real_t *data, const int m, const int *ipiv, real_t *x)
+{
+   LSolve(data, m, ipiv, x);
+   USolve(data, m, x);
+}
+
+
+/// @brief Given an (n x m) matrix A21, compute X2 <- X2 - A21 X1, for matrices
+/// X1, and X2 of size (m x r) and (n x r), respectively.
+MFEM_HOST_DEVICE
+inline void SubMult(const int m, const int n, const int r, const real_t *A21,
+                    const real_t *X1, real_t *X2)
+{
+   // X2 <- X2 - A21 X1
+   for (int k = 0; k < r; k++)
+   {
+      for (int j = 0; j < m; j++)
+      {
+         const real_t x1_jk = X1[j+k*m];
+         for (int i = 0; i < n; i++)
+         {
+            X2[i+k*n] -= A21[i+j*n] * x1_jk;
+         }
+      }
+   }
+}
+
+/// Assuming P.A = L.U factored data of size (m x m), compute the 2x2 block
+/// decomposition:
+///      | P 0 | |  A  A12 | = |  L  0 | | U U12 |
+///      | 0 I | | A21 A22 |   | L21 I | | 0 S22 |
+///   where A12, A21, and A22 are matrices of size (m x n), (n x m), and
+///   (n x n), respectively. The blocks are overwritten as follows:
+///      A12 <- U12 = L^{-1} P A12
+///      A21 <- L21 = A21 U^{-1}
+///      A22 <- S22 = A22 - L21 U12.
+///   The block S22 is the Schur complement.
+MFEM_HOST_DEVICE
+inline void BlockFactor(const real_t *data, int m, const int *ipiv,
+                        int n, real_t *A12, real_t *A21, real_t *A22)
+{
+   // A12 <- L^{-1} P A12
+   for (int i = 0; i < n; ++i)
+   {
+      LSolve(data, m, ipiv, A12 + i*m);
+   }
+   // A21 <- A21 U^{-1}
+   for (int j = 0; j < m; j++)
+   {
+      const real_t u_jj_inv = 1.0/data[j+j*m];
+      for (int i = 0; i < n; i++)
+      {
+         A21[i+j*n] *= u_jj_inv;
+      }
+      for (int k = j+1; k < m; k++)
+      {
+         const real_t u_jk = data[j+k*m];
+         for (int i = 0; i < n; i++)
+         {
+            A21[i+k*n] -= A21[i+j*n] * u_jk;
+         }
+      }
+   }
+   // A22 <- A22 - A21 A12
+   SubMult(m, n, n, A21, A12, A22);
+}
+
+/// @brief Compute the LU factorization of the m x m matrix @a A.
+///
+/// Factorize the matrix of size (m x m) overwriting it with the LU factors. The
+/// factorization is such that L.U = P.A, where A is the original matrix and P
+/// is a permutation matrix represented by ipiv.
+///
+/// @param [in, out] A matrix
+/// @param [in] m size of the square matrix
+/// @param [out] ipiv array of pivots (length m)
+/// @param [in] tol optional fuzzy comparison tolerance. Defaults to 0.0.
+///
+/// @return true if the factorization succeeds, false otherwise (zero pivot).
+MFEM_HOST_DEVICE
+inline bool LUFactor(real_t *A, const int m, int *ipiv, const real_t tol=0.0)
+{
+   bool pivot_flag = true;
+
+   for (int i = 0; i < m; i++)
+   {
+      // pivoting
+      {
+         int piv = i;
+         real_t a = fabs(A[piv + m*i]);
+         for (int j = i+1; j < m; j++)
+         {
+            const real_t b = fabs(A[j + m*i]);
+            if (b > a)
+            {
+               a = b;
+               piv = j;
+            }
+         }
+         ipiv[i] = piv;
+         if (piv != i)
+         {
+            // swap rows i and piv in both L and U parts
+            for (int j = 0; j < m; j++)
+            {
+               internal::Swap<real_t>(A[i + m*j], A[piv + m*j]);
+            }
+         }
+      } // pivot end
+
+      if (abs(A[i + m*i]) <= tol)
+      {
+         pivot_flag = false;
+      }
+
+      const real_t a_ii_inv = 1.0 / A[i + m*i];
+      for (int j = i+1; j < m; j++)
+      {
+         A[j + m*i] *= a_ii_inv;
+      }
+
+      for (int k = i+1; k < m; k++)
+      {
+         const real_t a_ik = A[i + m*k];
+         for (int j = i+1; j < m; j++)
+         {
+            A[j + m*k] -= a_ik * A[j + m*i];
+         }
+      }
+   }
+
+   return pivot_flag;
+}
+
 } // namespace kernels
 
 } // namespace mfem

linalg/ode.cpp

@@ -9,12 +9,155 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#include "../general/communication.hpp"
 #include "operator.hpp"
 #include "ode.hpp"
 
 namespace mfem
 {
 
+std::string ODESolver::ExplicitTypes =
+   "\n\tExplicit solver: \n\t"
+   "        RK      :  1 - Forward Euler, 2 - RK2(0.5), 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+   "        AB      : 11 - AB1, 12 - AB2, 13 - AB3, 14 - AB4, 15 - AB5\n";
+
+std::string ODESolver::ImplicitTypes  =
+   "\n\tImplicit solver: \n\t"
+   "        (L-Stab): 21 - Backward Euler, 22 - SDIRK23(2), 23 - SDIRK33,\n\t"
+   "        (A-Stab): 32 - Implicit Midpoint, 33 - SDIRK23, 34 - SDIRK34,\n\t"
+   "        GA      : 40 -- 50  - Generalized-alpha,\n\t"
+   "        AM      : 51 - AM1, 52 - AM2, 53 - AM3, 54 - AM4\n";
+
+std::string ODESolver::Types = ODESolver::ExplicitTypes +
+                               ODESolver::ImplicitTypes;
+
+std::unique_ptr<ODESolver> ODESolver::Select(int ode_solver_type)
+{
+   if (ode_solver_type < 20)
+   {
+      return SelectExplicit(ode_solver_type);
+   }
+   else
+   {
+      return SelectImplicit(ode_solver_type);
+   }
+}
+
+std::unique_ptr<ODESolver> ODESolver::SelectExplicit(int ode_solver_type)
+{
+   using ode_ptr = std::unique_ptr<ODESolver>;
+   switch (ode_solver_type)
+   {
+      // Explicit RK methods
+      case 1: return ode_ptr(new ForwardEulerSolver);
+      case 2: return ode_ptr(new RK2Solver(0.5)); // midpoint method
+      case 3: return ode_ptr(new RK3SSPSolver);
+      case 4: return ode_ptr(new RK4Solver);
+      case 6: return ode_ptr(new RK6Solver);
+
+      // Explicit AB methods
+      case 11: return ode_ptr(new AB1Solver);
+      case 12: return ode_ptr(new AB2Solver);
+      case 13: return ode_ptr(new AB3Solver);
+      case 14: return ode_ptr(new AB4Solver);
+      case 15: return ode_ptr(new AB5Solver);
+
+      default:
+         MFEM_ABORT("Unknown ODE solver type: " << ode_solver_type);
+   }
+}
+
+std::unique_ptr<ODESolver> ODESolver::SelectImplicit(int ode_solver_type)
+{
+   using ode_ptr = std::unique_ptr<ODESolver>;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 21: return ode_ptr(new BackwardEulerSolver);
+      case 22: return ode_ptr(new SDIRK23Solver(2));
+      case 23: return ode_ptr(new SDIRK33Solver);
+
+      // Implicit A-stable methods (not L-stable)
+      case 32: return ode_ptr(new ImplicitMidpointSolver);
+      case 33: return ode_ptr(new SDIRK23Solver);
+      case 34: return ode_ptr(new SDIRK34Solver);
+
+      // Implicit generalized alpha
+      case 40:  return ode_ptr(new GeneralizedAlphaSolver(0.0));
+      case 41:  return ode_ptr(new GeneralizedAlphaSolver(0.1));
+      case 42:  return ode_ptr(new GeneralizedAlphaSolver(0.2));
+      case 43:  return ode_ptr(new GeneralizedAlphaSolver(0.3));
+      case 44:  return ode_ptr(new GeneralizedAlphaSolver(0.4));
+      case 45:  return ode_ptr(new GeneralizedAlphaSolver(0.5));
+      case 46:  return ode_ptr(new GeneralizedAlphaSolver(0.6));
+      case 47:  return ode_ptr(new GeneralizedAlphaSolver(0.7));
+      case 48:  return ode_ptr(new GeneralizedAlphaSolver(0.8));
+      case 49:  return ode_ptr(new GeneralizedAlphaSolver(0.9));
+      case 50:  return ode_ptr(new GeneralizedAlphaSolver(1.0));
+
+      // Implicit AM methods
+      case 51: return ode_ptr(new AM1Solver);
+      case 52: return ode_ptr(new AM2Solver);
+      case 53: return ode_ptr(new AM3Solver);
+      case 54: return ode_ptr(new AM4Solver);
+
+      default:
+         MFEM_ABORT("Unknown ODE solver type: " << ode_solver_type );
+   }
+}
+
+
+void ODEStateDataVector::SetSize( int vsize, MemoryType m_t)
+{
+   mem_type = m_t;
+   for (int i = 0; i < smax; i++)
+   {
+      idx[i] = smax - i - 1;
+      data[i].SetSize(vsize, mem_type);
+   }
+
+   ss = 0;
+}
+
+const Vector &ODEStateDataVector::Get(int i) const
+{
+   MFEM_ASSERT_INDEX_IN_RANGE(i,0,ss);
+   return data[idx[i]];
+}
+
+Vector &ODEStateDataVector::Get(int i)
+{
+   MFEM_ASSERT_INDEX_IN_RANGE(i,0,ss);
+   return data[idx[i]];
+}
+
+void ODEStateDataVector::Get(int i, Vector &vec) const
+{
+   MFEM_ASSERT_INDEX_IN_RANGE(i,0,ss);
+   vec = data[idx[i]];
+}
+
+void ODEStateDataVector::Set(int i, Vector &state)
+{
+   MFEM_ASSERT_INDEX_IN_RANGE(i,0,smax);
+   data[idx[i]] = state;
+}
+
+void ODEStateDataVector::Append(Vector &state)
+{
+   ShiftStages();
+   data[idx[0]] = state;
+   Increment();
+}
+
+void ODEStateDataVector::Print(std::ostream &os) const
+{
+   os << ss <<"/" <<smax<<std::endl;
+   idx.Print(os);
+   for (int i = 0; i < ss; i++) { data[idx[i]].Print(os); }
+}
+
+
 void ODESolver::Init(TimeDependentOperator &f_)
 {
    this->f = &f_;
@@ -344,104 +487,65 @@ const real_t RK8Solver::c[] =
 };
 
 
-AdamsBashforthSolver::AdamsBashforthSolver(int s_, const real_t *a_)
+AdamsBashforthSolver::AdamsBashforthSolver(int s_, const real_t *a_):
+   stages(s_), state(s_)
 {
-   smax = std::min(s_,5);
    a = a_;
-   k = new Vector[5];
-   dt_ = -1.0;
-
-   if (smax <= 2)
-   {
-      RKsolver = new RK2Solver();
-   }
-   else if (smax == 3)
-   {
-      RKsolver = new RK3SSPSolver();
-   }
-   else
-   {
-      RKsolver = new RK4Solver();
-   }
-}
-
-void AdamsBashforthSolver::GetStateVector(int i, Vector &state)
-{
-   MFEM_ASSERT( (i >= 0) && ( i < s ),
-                " AdamsBashforthSolver::GetStateVector \n" <<
-                " - Tried to get non-existent state "<<i);
-
-   state = k[idx[i]];
-}
-
-const Vector &AdamsBashforthSolver::GetStateVector(int i)
-{
-   MFEM_ASSERT( (i >= 0) && ( i < s ),
-                " AdamsBashforthSolver::GetStateVector \n" <<
-                " - Tried to get non-existent state "<<i);
-
-   return k[idx[i]];
-}
-
-
-void AdamsBashforthSolver::SetStateVector(int i, Vector &state)
-{
-   MFEM_ASSERT( (i >= 0) && ( i < smax ),
-                " AdamsBashforthSolver::SetStateVector \n" <<
-                " - Tried to set non-existent state "<<i);
-   k[idx[i]] = state;
-   s = std::max(i,s);
 }
 
 void AdamsBashforthSolver::Init(TimeDependentOperator &f_)
 {
    ODESolver::Init(f_);
-   RKsolver->Init(f_);
-   idx.SetSize(smax);
-   for (int i = 0; i < smax; i++)
-   {
-      idx[i] = (smax-i)%smax;
-      k[i].SetSize(f->Width());
-   }
-   s = 0;
+   if (RKsolver) { RKsolver->Init(f_); }
+   state.SetSize(f->Width(), mem_type);
+   dt_ = -1.0;
 }
 
 void AdamsBashforthSolver::Step(Vector &x, real_t &t, real_t &dt)
 {
-   if ( (dt_ > 0.0) && (fabs(dt-dt_) >10*std::numeric_limits<real_t>::epsilon()))
+   CheckTimestep(dt);
+
+   if (state.Size() >= stages -1)
    {
-      s = 0;
+      f->SetTime(t);
+      f->Mult(x, state[0]);
+      state.Increment();
+      for (int i = 0; i < stages; i++)
+      {
+         x.Add(a[i]*dt, state[i]);
+      }
+      t += dt;
+   }
+   else
+   {
+      f->Mult(x,state[0]);
+      RKsolver->Step(x,t,dt);
+      state.Increment();
+   }
+
+   state.ShiftStages();
+}
+
+void AdamsBashforthSolver::CheckTimestep(real_t dt)
+{
+   if (dt_ < 0.0)
+   {
+      dt_ = dt;
+      return;
+   }
+   else if (fabs(dt-dt_) >10*std::numeric_limits<real_t>::epsilon())
+   {
+      state.Reset();
       dt_ = dt;
 
       if (print())
       {
          mfem::out << "WARNING:" << std::endl;
          mfem::out << " - Time step changed" << std::endl;
-         mfem::out << " - Purging Adams-Bashforth history" << std::endl;
+         mfem::out << " - Purging time stepping history" << std::endl;
          mfem::out << " - Will run Runge-Kutta to rebuild history" << std::endl;
       }
    }
-
-   s++;
-   s = std::min(s, smax);
-   if (s == smax)
-   {
-      f->SetTime(t);
-      f->Mult(x, k[idx[0]]);
-      for (int i = 0; i < s; i++)
-      {
-         x.Add(a[i]*dt, k[idx[i]]);
-      }
-      t += dt;
-   }
-   else
-   {
-      f->Mult(x,k[idx[0]]);
-      RKsolver->Step(x,t,dt);
-   }
-
-   // Shift the index
-   for (int i = 0; i < smax; i++) { idx[i] = ++idx[i]%smax; }
 }
 
 const real_t AB1Solver::a[] =
@@ -455,110 +559,68 @@ const real_t AB4Solver::a[] =
 const real_t AB5Solver::a[] =
 {1901.0/720.0,-2774.0/720.0, 2616.0/720.0,-1274.0/720.0, 251.0/720.0};
 
-AdamsMoultonSolver::AdamsMoultonSolver(int s_, const real_t *a_)
+
+AdamsMoultonSolver::AdamsMoultonSolver(int s_, const real_t *a_):
+   stages(s_), state(s_)
 {
-   s = 0;
-   smax = std::min(s_+1,5);
    a = a_;
-   k = new Vector[5];
-   dt_ = -1.0;
-
-   if (smax <= 3)
-   {
-      RKsolver = new SDIRK23Solver();
-   }
-   else
-   {
-      RKsolver = new SDIRK34Solver();
-   }
-}
-
-const Vector &AdamsMoultonSolver::GetStateVector(int i)
-{
-   MFEM_ASSERT( (i >= 0) && ( i < s ),
-                " AdamsMoultonSolver::GetStateVector \n" <<
-                " - Tried to get non-existent state "<<i);
-   return k[idx[i+1]];
-}
-
-void AdamsMoultonSolver::GetStateVector(int i, Vector &state)
-{
-   MFEM_ASSERT( (i >= 0) && ( i < s ),
-                " AdamsMoultonSolver::GetStateVector \n" <<
-                " - Tried to get non-existent state "<<i);
-   state = k[idx[i+1]];
-}
-
-void AdamsMoultonSolver::SetStateVector(int i, Vector &state)
-{
-   MFEM_ASSERT( (i >= 0) && ( i < smax ),
-                " AdamsMoultonSolver::SetStateVector \n" <<
-                " - Tried to set non-existent state "<<i);
-   k[idx[i+1]] = state;
-   s = std::max(i,s);
 }
 
 void AdamsMoultonSolver::Init(TimeDependentOperator &f_)
 {
    ODESolver::Init(f_);
-   RKsolver->Init(f_);
-   int n = f->Width();
-   idx.SetSize(smax);
-   for (int i = 0; i < smax; i++)
-   {
-      idx[i] = (smax-i)%smax;
-      k[i].SetSize(n);
-   }
-   s = 0;
+   if (RKsolver) { RKsolver->Init(f_); }
+   state.SetSize(f->Width(), mem_type);
+   dt_ = -1.0;
 }
 
 void AdamsMoultonSolver::Step(Vector &x, real_t &t, real_t &dt)
 {
-   if ( (dt_ > 0.0) && (fabs(dt-dt_) >10*std::numeric_limits<real_t>::epsilon()))
+   if (dt_ < 0.0)
    {
-      s = 0;
+      dt_ = dt;
+   }
+   else if (fabs(dt-dt_) > 10*std::numeric_limits<real_t>::epsilon())
+   {
+      state.Reset();
       dt_ = dt;
 
       if (print())
       {
          mfem::out << "WARNING:" << std::endl;
          mfem::out << " - Time step changed" << std::endl;
-         mfem::out << " - Purging Adams-Moulton history" << std::endl;
+         mfem::out << " - Purging time stepping history" << std::endl;
          mfem::out << " - Will run Runge-Kutta to rebuild history" << std::endl;
       }
    }
 
-   if ((s == 0)&&(smax>1))
+   if ((state.Size() == 0)&&(stages>1))
    {
-      f->Mult(x,k[idx[1]]);
+      f->Mult(x,state[0]);
+      state.Increment();
    }
-   s++;
-   s = std::min(s, smax);
 
-   if (s >= smax-1)
+   if (state.Size() >= stages )
    {
       f->SetTime(t);
-      for (int i = 1; i < smax; i++)
+      for (int i = 0; i < stages; i++)
       {
-         x.Add(a[i]*dt, k[idx[i]]);
+         x.Add(a[i+1]*dt, state[i]);
       }
-      f->ImplicitSolve(a[0]*dt, x, k[idx[0]]);
-      x.Add(a[0]*dt, k[idx[0]]);
+      state.ShiftStages();
+      f->ImplicitSolve(a[0]*dt, x, state[0]);
+      x.Add(a[0]*dt, state[0]);
       t += dt;
    }
    else
    {
+      state.ShiftStages();
       RKsolver->Step(x,t,dt);
-      f->Mult(x,k[idx[0]]);
+      f->Mult(x,state[0]);
+      state.Increment();
    }
-
-
-   // Shift the index
-   for (int i = 0; i < smax; i++) { idx[i] = ++idx[i]%smax; }
 }
 
-const real_t AM0Solver::a[] =
-{1.0};
 const real_t AM1Solver::a[] =
 {0.5, 0.5};
 const real_t AM2Solver::a[] =
@@ -817,34 +879,7 @@ void GeneralizedAlphaSolver::Init(TimeDependentOperator &f_)
    ODESolver::Init(f_);
    k.SetSize(f->Width(), mem_type);
    y.SetSize(f->Width(), mem_type);
-   xdot.SetSize(f->Width(), mem_type);
-   xdot = 0.0;
-   nstate = 0;
-}
-
-const Vector &GeneralizedAlphaSolver::GetStateVector(int i)
-{
-   MFEM_ASSERT( (i == 0) && (nstate == 1),
-                "GeneralizedAlphaSolver::GetStateVector \n" <<
-                " - Tried to get non-existent state "<<i);
-   return xdot;
-}
-
-void GeneralizedAlphaSolver::GetStateVector(int i, Vector &state)
-{
-   MFEM_ASSERT( (i == 0) && (nstate == 1),
-                "GeneralizedAlphaSolver::GetStateVector \n" <<
-                " - Tried to get non-existent state "<<i);
-   state = xdot;
-}
-
-void GeneralizedAlphaSolver::SetStateVector(int i, Vector &state)
-{
-   MFEM_ASSERT( (i == 0),
-                "GeneralizedAlphaSolver::SetStateVector \n" <<
-                " - Tried to set non-existent state "<<i);
-   xdot = state;
-   nstate = 1;
+   state.SetSize(f->Width(), mem_type);
 }
 
 void GeneralizedAlphaSolver::SetRhoInf(real_t rho_inf)
@@ -884,17 +919,17 @@ void GeneralizedAlphaSolver::PrintProperties(std::ostream &os)
    }
 }
 
-// This routine assumes xdot is initialized.
+// This routine state[0] represents xdot
 void GeneralizedAlphaSolver::Step(Vector &x, real_t &t, real_t &dt)
 {
-   if (nstate == 0)
+   if (state.Size() == 0)
    {
-      f->Mult(x,xdot);
-      nstate = 1;
+      f->Mult(x,state[0]);
+      state.Increment();
    }
 
    // Set y = x + alpha_f*(1.0 - (gamma/alpha_m))*dt*xdot
-   add(x, alpha_f*(1.0 - (gamma/alpha_m))*dt, xdot, y);
+   add(x, alpha_f*(1.0 - (gamma/alpha_m))*dt, state[0], y);
 
    // Solve k = f(y + dt_eff*k)
    real_t dt_eff = (gamma*alpha_f/alpha_m)*dt;
@@ -902,11 +937,11 @@ void GeneralizedAlphaSolver::Step(Vector &x, real_t &t, real_t &dt)
    f->ImplicitSolve(dt_eff, y, k);
 
    // Update x and xdot
-   x.Add((1.0 - (gamma/alpha_m))*dt, xdot);
+   x.Add((1.0 - (gamma/alpha_m))*dt, state[0]);
    x.Add(       (gamma/alpha_m) *dt, k);
 
-   xdot *= (1.0-(1.0/alpha_m));
-   xdot.Add((1.0/alpha_m),k);
+   state[0] *= (1.0-(1.0/alpha_m));
+   state[0].Add((1.0/alpha_m),k);
 
    t += dt;
 }
@@ -1017,18 +1052,75 @@ SIAVSolver::Step(Vector &q, Vector &p, real_t &t, real_t &dt)
    }
 }
 
+std::string SecondOrderODESolver::Types =
+   "ODE solver: \n\t"
+   "  [0--10] - GeneralizedAlpha(0.1 * s),\n\t"
+   "  11 - Average Acceleration, 12 - Linear Acceleration\n\t"
+   "  13 - CentralDifference, 14 - FoxGoodwin";
+
+SecondOrderODESolver* SecondOrderODESolver::Select(int ode_solver_type)
+{
+   SecondOrderODESolver*  ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // Implicit methods
+      case 0: ode_solver = new GeneralizedAlpha2Solver(0.0); break;
+      case 1: ode_solver = new GeneralizedAlpha2Solver(0.1); break;
+      case 2: ode_solver = new GeneralizedAlpha2Solver(0.2); break;
+      case 3: ode_solver = new GeneralizedAlpha2Solver(0.3); break;
+      case 4: ode_solver = new GeneralizedAlpha2Solver(0.4); break;
+      case 5: ode_solver = new GeneralizedAlpha2Solver(0.5); break;
+      case 6: ode_solver = new GeneralizedAlpha2Solver(0.6); break;
+      case 7: ode_solver = new GeneralizedAlpha2Solver(0.7); break;
+      case 8: ode_solver = new GeneralizedAlpha2Solver(0.8); break;
+      case 9: ode_solver = new GeneralizedAlpha2Solver(0.9); break;
+      case 10: ode_solver = new GeneralizedAlpha2Solver(1.0); break;
+
+      case 11: ode_solver = new AverageAccelerationSolver(); break;
+      case 12: ode_solver = new LinearAccelerationSolver(); break;
+      case 13: ode_solver = new CentralDifferenceSolver(); break;
+      case 14: ode_solver = new FoxGoodwinSolver(); break;
+
+      default:
+         MFEM_ABORT("Unknown ODE solver type: " << ode_solver_type);
+   }
+   return ode_solver;
+}
+
+// In this routine state[0] represents d2xdt2
+void SecondOrderODESolver::EulerStep(Vector &x, Vector &dxdt, real_t &t,
+                                     real_t &dt)
+{
+   x.Add(dt, dxdt);
+
+   f->SetTime(t + dt);
+   f->ImplicitSolve(0.5*dt*dt, dt, x, dxdt, state[0]);
+
+   x   .Add(0.5*dt*dt, state[0]);
+   dxdt.Add(dt,    state[0]);
+   t += dt;
+}
+
+// In this routine state[0] represents d2xdt2
+void SecondOrderODESolver::MidPointStep(Vector &x, Vector &dxdt, real_t &t,
+                                        real_t &dt)
+{
+   x.Add(0.5*dt, dxdt);
+
+   f->SetTime(t + dt);
+   f->ImplicitSolve(0.25*dt*dt, 0.5*dt, x, dxdt, state[0]);
+
+   x.Add(0.5*dt, dxdt);
+   x.Add(0.5*dt*dt, state[0]);
+   dxdt.Add(dt, state[0]);
+   t += dt;
+}
+
 void SecondOrderODESolver::Init(SecondOrderTimeDependentOperator &f_)
 {
    this->f = &f_;
    mem_type = GetMemoryType(f_.GetMemoryClass());
-}
-
-void NewmarkSolver::Init(SecondOrderTimeDependentOperator &f_)
-{
-   SecondOrderODESolver::Init(f_);
-   d2xdt2.SetSize(f->Width());
-   d2xdt2 = 0.0;
-   first = true;
+   state.SetSize(f->Width(), mem_type);
 }
 
 void NewmarkSolver::PrintProperties(std::ostream &os)
@@ -1060,6 +1152,7 @@ void NewmarkSolver::PrintProperties(std::ostream &os)
    }
 }
 
+// In this routine state[0] represents d2xdt2
 void NewmarkSolver::Step(Vector &x, Vector &dxdt, real_t &t, real_t &dt)
 {
    real_t fac0 = 0.5 - beta;
@@ -1068,60 +1161,38 @@ void NewmarkSolver::Step(Vector &x, Vector &dxdt, real_t &t, real_t &dt)
    real_t fac4 = gamma;
 
    // In the first pass compute d2xdt2 directly from operator.
-   if (first)
+   if (state.Size() == 0)
    {
-      f->Mult(x, dxdt, d2xdt2);
-      first = false;
+      if (no_mult)
+      {
+         MidPointStep(x, dxdt, t, dt);
+         return;
+      }
+      else
+      {
+         f->Mult(x, dxdt, state[0]);
+      }
    }
    f->SetTime(t + dt);
 
    x.Add(dt, dxdt);
-   x.Add(fac0*dt*dt, d2xdt2);
-   dxdt.Add(fac2*dt, d2xdt2);
+   x.Add(fac0*dt*dt, state[0]);
+   dxdt.Add(fac2*dt, state[0]);
 
    f->SetTime(t + dt);
-   f->ImplicitSolve(fac3*dt*dt, fac4*dt, x, dxdt, d2xdt2);
+   f->ImplicitSolve(fac3*dt*dt, fac4*dt, x, dxdt, state[0]);
 
-   x   .Add(fac3*dt*dt, d2xdt2);
-   dxdt.Add(fac4*dt,    d2xdt2);
+   x   .Add(fac3*dt*dt, state[0]);
+   dxdt.Add(fac4*dt,    state[0]);
    t += dt;
 }
 
 void GeneralizedAlpha2Solver::Init(SecondOrderTimeDependentOperator &f_)
 {
    SecondOrderODESolver::Init(f_);
-   xa.SetSize(f->Width());
-   va.SetSize(f->Width());
-   aa.SetSize(f->Width());
-   d2xdt2.SetSize(f->Width());
-   d2xdt2 = 0.0;
-   nstate = 0;
-}
-
-const Vector &GeneralizedAlpha2Solver::GetStateVector(int i)
-{
-   MFEM_ASSERT( (i == 0) && (nstate == 1),
-                "GeneralizedAlpha2Solver::GetStateVector \n" <<
-                " - Tried to get non-existent state "<<i);
-   return d2xdt2;
-}
-
-
-void GeneralizedAlpha2Solver::GetStateVector(int i, Vector &state)
-{
-   MFEM_ASSERT( (i == 0) && (nstate == 1),
-                "GeneralizedAlpha2Solver::GetStateVector \n" <<
-                " - Tried to get non-existent state "<<i);
-   state = d2xdt2;
-}
-
-void GeneralizedAlpha2Solver::SetStateVector(int i, Vector &state)
-{
-   MFEM_ASSERT( (i == 0),
-                "GeneralizedAlpha2Solver::SetStateVector \n" <<
-                " - Tried to set non-existent state "<<i);
-   d2xdt2 = state;
-   nstate = 1;
+   xa.SetSize(f->Width(), mem_type);
+   va.SetSize(f->Width(), mem_type);
+   aa.SetSize(f->Width(), mem_type);
 }
 
 void GeneralizedAlpha2Solver::PrintProperties(std::ostream &os)
@@ -1153,6 +1224,7 @@ void GeneralizedAlpha2Solver::PrintProperties(std::ostream &os)
    }
 }
 
+// In this routine state[0] represents d2xdt2
 void GeneralizedAlpha2Solver::Step(Vector &x, Vector &dxdt,
                                    real_t &t, real_t &dt)
 {
@@ -1164,16 +1236,24 @@ void GeneralizedAlpha2Solver::Step(Vector &x, Vector &dxdt,
    real_t fac5 = alpha_m;
 
    // In the first pass compute d2xdt2 directly from operator.
-   if (nstate == 0)
+   if (state.Size() == 0)
    {
-      f->Mult(x, dxdt, d2xdt2);
-      nstate = 1;
+      if (no_mult)
+      {
+         MidPointStep(x, dxdt, t, dt);
+         return;
+      }
+      else
+      {
+         f->Mult(x, dxdt, state[0]);
+      }
+      state.Increment();
    }
 
    // Predict alpha levels
-   add(dxdt, fac0*dt, d2xdt2, va);
+   add(dxdt, fac0*dt, state[0], va);
    add(x, fac1*dt, va, xa);
-   add(dxdt, fac2*dt, d2xdt2, va);
+   add(dxdt, fac2*dt, state[0], va);
 
    // Solve alpha levels
    f->SetTime(t + dt);
@@ -1190,8 +1270,8 @@ void GeneralizedAlpha2Solver::Step(Vector &x, Vector &dxdt,
    dxdt *= 1.0 - 1.0/fac1;
    dxdt.Add (1.0/fac1, va);
 
-   d2xdt2 *= 1.0 - 1.0/fac5;
-   d2xdt2.Add (1.0/fac5, aa);
+   state[0] *= 1.0 - 1.0/fac5;
+   state[0].Add (1.0/fac5, aa);
 
    t += dt;
 }

linalg/ode.hpp

@@ -12,13 +12,99 @@
 #ifndef MFEM_ODE
 #define MFEM_ODE
 
+#include "../general/communication.hpp"
 #include "../config/config.hpp"
 #include "operator.hpp"
-#include "../general/communication.hpp"
+#include <vector>
+#include <memory>
 
 namespace mfem
 {
 
+/// An interface for storing state of previous timesteps
+class ODEStateData
+{
+public:
+   /// Get the maximum number of stored stages
+   virtual int MaxSize() const = 0;
+
+   /// Get the current number of stored stages
+   virtual int Size() const = 0;
+
+   /// Get the ith state vector
+   virtual const Vector &Get(int i) const = 0;
+
+   /// Get the ith state vector - non-const version
+   virtual Vector &Get(int i) = 0;
+
+   /// Get the ith state vector - with a copy
+   virtual void Get(int i, Vector &vec) const = 0;
+
+   /// Set the ith state vector
+   virtual void Set(int i, Vector &state) = 0;
+
+   /// Add state vector and increment state size
+   virtual void Append(Vector &state) = 0;
+
+   /// Virtual destructor
+   virtual ~ODEStateData() = default;
+};
+
+/// An implementation of ODEStateData that stores states in an std::vector<Vector>
+class ODEStateDataVector : public ODEStateData
+{
+private:
+   MemoryType mem_type;
+   int ss, smax;
+   std::vector<Vector> data;
+   Array<int> idx;
+
+public:
+   ODEStateDataVector (int smax): smax(smax)
+   {
+      data.resize(smax);
+      idx.SetSize(smax);
+      ss = 0;
+   };
+
+   /// Set the number of stages and the size of the vectors
+   void  SetSize(int vsize, MemoryType mem_type);
+
+   /// Shift the stage counter for the next timestep
+   inline void ShiftStages()
+   {
+      for (int i = 0; i < smax; i++) { idx[i] = (++idx[i])%smax; }
+   };
+
+   /// Increment the stage counter
+   void Increment() { ss++; ss = std::min(ss,smax); };
+
+   /// Reset the stage counter
+   void Reset() { ss = 0; };
+
+   /// Reference access to the ith vector.
+   inline Vector & operator[](int i) { return data[idx[i]]; };
+
+   /// Const reference access to the ith vector.
+   inline const Vector &operator[](int i) const { return data[idx[i]]; };
+
+   /// Print state data
+   void Print(std::ostream &os = mfem::out) const ;
+
+   int  MaxSize() const override { return smax; };
+
+   int  Size() const override { return ss; };
+
+   const Vector &Get(int i) const override;
+   Vector &Get(int i) override;
+   void Get(int i, Vector &vec) const override;
+
+   void Set(int i, Vector &state) override;
+
+   void Append(Vector &state) override;
+};
+
+
 /// Abstract class for solving systems of ODEs: dx/dt = f(x,t)
 class ODESolver
 {
@@ -92,26 +178,48 @@ public:
       while (t < tf) { Step(x, t, dt); }
    }
 
-   /// Function for getting and setting the state vectors
-   virtual int   GetMaxStateSize() { return 0; }
-   virtual int   GetStateSize() { return 0; }
-   virtual const Vector &GetStateVector(int i)
-   {
-      mfem_error("ODESolver has no state vectors");
-      Vector *s = NULL; return *s; // Make some compiler happy
-   }
-   virtual void  GetStateVector(int i, Vector &state)
-   {
-      mfem_error("ODESolver has no state vectors");
-   }
-   virtual void  SetStateVector(int i, Vector &state)
-   {
-      mfem_error("ODESolver has no state vectors");
-   }
+   /// Returns how many State vectors the ODE requires
+   virtual int GetStateSize() { return 0; };
+
+   // Help info for ODESolver options
+   static MFEM_EXPORT std::string ExplicitTypes;
+   static MFEM_EXPORT std::string ImplicitTypes;
+   static MFEM_EXPORT std::string Types;
+
+   /// Function for selecting the desired ODESolver (Explicit and Implicit)
+   /// Returns an ODESolver pointer based on an type
+   /// Caller gets ownership of the object and is responsible for its deletion
+   static MFEM_EXPORT std::unique_ptr<ODESolver> Select(const int ode_solver_type);
+
+   /// Function for selecting the desired Explicit ODESolver
+   /// Returns an ODESolver pointer based on an type
+   /// Caller gets ownership of the object and is responsible for its deletion
+   static MFEM_EXPORT std::unique_ptr<ODESolver> SelectExplicit(
+      const int ode_solver_type);
+
+   /// Function for selecting the desired Implicit ODESolver
+   /// Returns an ODESolver pointer based on an type
+   /// Caller gets ownership of the object and is responsible for its deletion
+   static MFEM_EXPORT std::unique_ptr<ODESolver> SelectImplicit(
+      const int ode_solver_type);
 
    virtual ~ODESolver() { }
 };
 
+/// Abstract class for an ODESolver that has state history implemented as ODEStateData
+class ODESolverWithStates : public ODESolver
+{
+public:
+   /// Returns the StateData
+   virtual ODEStateData& GetState() = 0;
+
+   /// Returns the StateData
+   virtual const ODEStateData& GetState() const = 0;
+
+   /// Returns how many State vectors the ODE requires
+   virtual int GetStateSize() { return GetState().MaxSize(); };
+};
+
 
 /// The classical forward Euler method
 class ForwardEulerSolver : public ODESolver
@@ -217,196 +325,13 @@ public:
 class RK8Solver : public ExplicitRKSolver
 {
 private:
-   static const real_t a[66], b[12], c[11];
+   static MFEM_EXPORT const real_t a[66], b[12], c[11];
 
 public:
    RK8Solver() : ExplicitRKSolver(12, a, b, c) { }
 };
 
 
-/** An explicit Adams-Bashforth method. */
-class AdamsBashforthSolver : public ODESolver
-{
-private:
-   int s, smax;
-   const real_t *a;
-   Vector *k;
-   Array<int> idx;
-   ODESolver *RKsolver;
-   real_t dt_;
-
-   inline bool print()
-   {
-#ifdef MFEM_USE_MPI
-      return Mpi::IsInitialized() ? Mpi::Root() : true;
-#else
-      return true;
-#endif
-   }
-
-public:
-   AdamsBashforthSolver(int s_, const real_t *a_);
-
-   void Init(TimeDependentOperator &f_) override;
-
-   void Step(Vector &x, real_t &t, real_t &dt) override;
-
-   int  GetMaxStateSize() override { return smax; };
-   int  GetStateSize() override { return s; };
-   const Vector &GetStateVector(int i) override;
-   void GetStateVector(int i, Vector &state) override;
-   void SetStateVector(int i, Vector &state) override;
-
-   ~AdamsBashforthSolver()
-   {
-      if (RKsolver) { delete RKsolver; }
-      delete [] k;
-   }
-};
-
-/** A 1-stage, 1st order AB method.  */
-class AB1Solver : public AdamsBashforthSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[1];
-
-public:
-   AB1Solver() : AdamsBashforthSolver(1, a) { }
-};
-
-/** A 2-stage, 2nd order AB method.  */
-class AB2Solver : public AdamsBashforthSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[2];
-
-public:
-   AB2Solver() : AdamsBashforthSolver(2, a) { }
-};
-
-/** A 3-stage, 3rd order AB method.  */
-class AB3Solver : public AdamsBashforthSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[3];
-
-public:
-   AB3Solver() : AdamsBashforthSolver(3, a) { }
-};
-
-/** A 4-stage, 4th order AB method.  */
-class AB4Solver : public AdamsBashforthSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[4];
-
-public:
-   AB4Solver() : AdamsBashforthSolver(4, a) { }
-};
-
-/** A 5-stage, 5th order AB method.  */
-class AB5Solver : public AdamsBashforthSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[5];
-
-public:
-   AB5Solver() : AdamsBashforthSolver(5, a) { }
-};
-
-
-/** An implicit Adams-Moulton method. */
-class AdamsMoultonSolver : public ODESolver
-{
-private:
-   int s, smax;
-   const real_t *a;
-   Vector *k;
-   Array<int> idx;
-   ODESolver *RKsolver;
-   real_t dt_;
-
-   inline bool print()
-   {
-#ifdef MFEM_USE_MPI
-      return Mpi::IsInitialized() ? Mpi::Root() : true;
-#else
-      return true;
-#endif
-   }
-
-public:
-   AdamsMoultonSolver(int s_, const real_t *a_);
-
-   void Init(TimeDependentOperator &f_) override;
-
-   void Step(Vector &x, real_t &t, real_t &dt) override;
-
-   int  GetMaxStateSize() override { return smax-1; };
-   int  GetStateSize() override { return s-1; };
-   const Vector &GetStateVector(int i) override;
-   void GetStateVector(int i, Vector &state) override;
-   void SetStateVector(int i, Vector &state) override;
-
-   ~AdamsMoultonSolver()
-   {
-      if (RKsolver) { delete RKsolver; }
-      delete [] k;
-   };
-};
-
-/** A 0-stage, 1st order AM method. */
-class AM0Solver : public AdamsMoultonSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[1];
-
-public:
-   AM0Solver() : AdamsMoultonSolver(0, a) { }
-};
-
-
-/** A 1-stage, 2nd order AM method. */
-class AM1Solver : public AdamsMoultonSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[2];
-
-public:
-   AM1Solver() : AdamsMoultonSolver(1, a) { }
-};
-
-/** A 2-stage, 3rd order AM method. */
-class AM2Solver : public AdamsMoultonSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[3];
-
-public:
-   AM2Solver() : AdamsMoultonSolver(2, a) { }
-};
-
-/** A 3-stage, 4th order AM method. */
-class AM3Solver : public AdamsMoultonSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[4];
-
-public:
-   AM3Solver() : AdamsMoultonSolver(3, a) { }
-};
-
-/** A 4-stage, 5th order AM method. */
-class AM4Solver : public AdamsMoultonSolver
-{
-private:
-   static MFEM_EXPORT const real_t a[5];
-
-public:
-   AM4Solver() : AdamsMoultonSolver(4, a) { }
-};
-
-
 /// Backward Euler ODE solver. L-stable.
 class BackwardEulerSolver : public ODESolver
 {
@@ -527,31 +452,183 @@ public:
 /// Generalized-alpha ODE solver from "A generalized-α method for integrating
 /// the filtered Navier-Stokes equations with a stabilized finite element
 /// method" by K.E. Jansen, C.H. Whiting and G.M. Hulbert.
-class GeneralizedAlphaSolver : public ODESolver
+class GeneralizedAlphaSolver : public ODESolverWithStates
 {
+   ODEStateDataVector state;
+
 protected:
-   mutable Vector xdot,k,y;
+
+   mutable Vector k,y;
    real_t alpha_f, alpha_m, gamma;
-   int  nstate;
 
    void SetRhoInf(real_t rho_inf);
-   void PrintProperties(std::ostream &out = mfem::out);
+   void PrintProperties(std::ostream &os = mfem::out);
 public:
 
-   GeneralizedAlphaSolver(real_t rho = 1.0) { SetRhoInf(rho); };
-
+   GeneralizedAlphaSolver(real_t rho = 1.0) : state(1) { SetRhoInf(rho); };
    void Init(TimeDependentOperator &f_) override;
-
    void Step(Vector &x, real_t &t, real_t &dt) override;
 
-   int  GetMaxStateSize() override { return 1; };
-   int  GetStateSize() override { return nstate; };
-   const Vector &GetStateVector(int i) override;
-   void GetStateVector(int i, Vector &state) override;
-   void SetStateVector(int i, Vector &state) override;
+   ODEStateData& GetState() override { return state; }
+   const ODEStateData& GetState() const override { return state; }
 };
 
 
+/** An explicit Adams-Bashforth method. */
+class AdamsBashforthSolver : public ODESolverWithStates
+{
+private:
+   const real_t *a;
+   const int stages;
+   real_t dt_;
+   ODEStateDataVector state;
+
+protected:
+   std::unique_ptr<ODESolver> RKsolver;
+
+   inline bool print()
+   {
+#ifdef MFEM_USE_MPI
+      return Mpi::IsInitialized() ? Mpi::Root() : true;
+#else
+      return true;
+#endif
+   }
+
+   void CheckTimestep(real_t dt);
+
+public:
+   AdamsBashforthSolver(int s_, const real_t *a_);
+   void Init(TimeDependentOperator &f_) override;
+   void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   ODEStateData& GetState() override { return state; }
+   const ODEStateData& GetState() const override { return state; }
+};
+
+/** A 1-stage, 1st order AB method.  */
+class AB1Solver : public AdamsBashforthSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[1];
+
+public:
+   AB1Solver() : AdamsBashforthSolver(1, a) { }
+};
+
+/** A 2-stage, 2nd order AB method.  */
+class AB2Solver : public AdamsBashforthSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[2];
+
+public:
+   AB2Solver() : AdamsBashforthSolver(2, a) { RKsolver.reset(new RK2Solver()); }
+};
+
+/** A 3-stage, 3rd order AB method.  */
+class AB3Solver : public AdamsBashforthSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[3];
+
+public:
+   AB3Solver() : AdamsBashforthSolver(3, a) { RKsolver.reset(new RK3SSPSolver()); }
+};
+
+/** A 4-stage, 4th order AB method.  */
+class AB4Solver : public AdamsBashforthSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[4];
+
+public:
+   AB4Solver() : AdamsBashforthSolver(4, a) { RKsolver.reset(new RK4Solver()); }
+};
+
+/** A 5-stage, 5th order AB method.  */
+class AB5Solver : public AdamsBashforthSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[5];
+
+public:
+   AB5Solver() : AdamsBashforthSolver(5, a) { RKsolver.reset(new RK6Solver()); }
+};
+
+
+/** An implicit Adams-Moulton method. */
+class AdamsMoultonSolver : public ODESolverWithStates
+{
+private:
+   const real_t *a;
+   const int stages;
+   real_t dt_;
+   ODEStateDataVector state;
+
+protected:
+   std::unique_ptr<ODESolver> RKsolver;
+
+   inline bool print()
+   {
+#ifdef MFEM_USE_MPI
+      return Mpi::IsInitialized() ? Mpi::Root() : true;
+#else
+      return true;
+#endif
+   }
+
+   void CheckTimestep(real_t dt);
+
+public:
+   AdamsMoultonSolver(int s_, const real_t *a_);
+   void Init(TimeDependentOperator &f_) override;
+   void Step(Vector &x, real_t &t, real_t &dt) override;
+
+   ODEStateData& GetState() override { return state; }
+   const ODEStateData& GetState() const override { return state; }
+};
+
+/** A 1-stage, 2nd order AM method. */
+class AM1Solver : public AdamsMoultonSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[2];
+
+public:
+   AM1Solver() : AdamsMoultonSolver(1, a) { RKsolver.reset(new SDIRK23Solver()); }
+};
+
+/** A 2-stage, 3rd order AM method. */
+class AM2Solver : public AdamsMoultonSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[3];
+
+public:
+   AM2Solver() : AdamsMoultonSolver(2, a) { RKsolver.reset(new SDIRK23Solver()); }
+};
+
+/** A 3-stage, 4th order AM method. */
+class AM3Solver : public AdamsMoultonSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[4];
+
+public:
+   AM3Solver() : AdamsMoultonSolver(3, a) { RKsolver.reset(new SDIRK23Solver()); }
+};
+
+/** A 4-stage, 5th order AM method. */
+class AM4Solver : public AdamsMoultonSolver
+{
+private:
+   static MFEM_EXPORT const real_t a[5];
+
+public:
+   AM4Solver() : AdamsMoultonSolver(4, a) { RKsolver.reset(new SDIRK34Solver()); }
+};
+
 /// The SIASolver class is based on the Symplectic Integration Algorithm
 /// described in "A Symplectic Integration Algorithm for Separable Hamiltonian
 /// Functions" by J. Candy and W. Rozmus, Journal of Computational Physics,
@@ -630,9 +707,10 @@ protected:
    /// Pointer to the associated TimeDependentOperator.
    SecondOrderTimeDependentOperator *f;  // f(.,.,t) : R^n x R^n --> R^n
    MemoryType mem_type;
+   ODEStateDataVector state;
 
 public:
-   SecondOrderODESolver() : f(NULL) { mem_type = MemoryType::HOST; }
+   SecondOrderODESolver() : f(NULL), state(1) { mem_type = MemoryType::HOST; }
 
    /// Associate a TimeDependentOperator with the ODE solver.
    /** This method has to be called:
@@ -680,6 +758,8 @@ public:
          sequence, then the ODE solver must be re-initialized by calling Init()
          between the two Step() calls. */
    virtual void Step(Vector &x, Vector &dxdt, real_t &t, real_t &dt) = 0;
+   void EulerStep(Vector &x, Vector &dxdt, real_t &t, real_t &dt);
+   void MidPointStep(Vector &x, Vector &dxdt, real_t &t, real_t &dt);
 
    /// Perform time integration from time @a t [in] to time @a tf [in].
    /** @param[in,out] x    Approximate solution.
@@ -705,22 +785,18 @@ public:
       while (t < tf) { Step(x, dxdt, t, dt); }
    }
 
-   /// Function for getting and setting the state vectors
-   virtual int   GetMaxStateSize() { return 0; };
-   virtual int   GetStateSize() { return 0; }
-   virtual const Vector &GetStateVector(int i)
-   {
-      mfem_error("ODESolver has no state vectors");
-      Vector *s = NULL; return *s; // Make some compiler happy
-   }
-   virtual void  GetStateVector(int i, Vector &state)
-   {
-      mfem_error("ODESolver has no state vectors");
-   }
-   virtual void  SetStateVector(int i, Vector &state)
-   {
-      mfem_error("ODESolver has no state vectors");
-   }
+   /// Functions for getting the state vectors
+   ODEStateData&  GetState() { return state; }
+   const ODEStateData&  GetState() const { return state; }
+
+   /// Returns how many State vectors the ODE requires
+   int GetStateSize() { return GetState().MaxSize(); };
+
+   /// Help info for SecondOrderODESolver options
+   static MFEM_EXPORT std::string Types;
+
+   /// Function selecting the desired SecondOrderODESolver
+   static MFEM_EXPORT SecondOrderODESolver *Select(const int ode_solver_type);
 
    virtual ~SecondOrderODESolver() { }
 };
@@ -731,17 +807,18 @@ public:
 class NewmarkSolver : public SecondOrderODESolver
 {
 private:
-   Vector d2xdt2;
-
    real_t beta, gamma;
-   bool first;
+   bool no_mult;
 
 public:
-   NewmarkSolver(real_t beta_ = 0.25, real_t gamma_ = 0.5) { beta = beta_; gamma = gamma_; };
+   NewmarkSolver(real_t beta_ = 0.25, real_t gamma_ = 0.5, bool no_mult_ = false)
+   {
+      beta = beta_;
+      gamma = gamma_;
+      no_mult = no_mult_;
+   };
 
-   void PrintProperties(std::ostream &out = mfem::out);
-
-   void Init(SecondOrderTimeDependentOperator &f_) override;
+   void PrintProperties(std::ostream &os = mfem::out);
 
    void Step(Vector &x, Vector &dxdt, real_t &t, real_t &dt) override;
 };
@@ -773,13 +850,14 @@ public:
 class GeneralizedAlpha2Solver : public SecondOrderODESolver
 {
 protected:
-   Vector xa,va,aa,d2xdt2;
+   Vector xa,va,aa;
    real_t alpha_f, alpha_m, beta, gamma;
-   int nstate;
+   bool no_mult;
 
 public:
-   GeneralizedAlpha2Solver(real_t rho_inf = 1.0)
+   GeneralizedAlpha2Solver(real_t rho_inf = 1.0, bool no_mult_ = false)
    {
+      no_mult = no_mult_;
       rho_inf = (rho_inf > 1.0) ? 1.0 : rho_inf;
       rho_inf = (rho_inf < 0.0) ? 0.0 : rho_inf;
 
@@ -789,17 +867,12 @@ public:
       gamma   = 0.5 + alpha_m - alpha_f;
    };
 
-   void PrintProperties(std::ostream &out = mfem::out);
+   void PrintProperties(std::ostream &os = mfem::out);
 
    void Init(SecondOrderTimeDependentOperator &f_) override;
 
    void Step(Vector &x, Vector &dxdt, real_t &t, real_t &dt) override;
 
-   int  GetMaxStateSize() override { return 1; };
-   int  GetStateSize() override { return nstate; };
-   const Vector &GetStateVector(int i) override;
-   void GetStateVector(int i, Vector &state) override;
-   void SetStateVector(int i, Vector &state) override;
 };
 
 /// The classical midpoint method.

linalg/petsc.hpp

@@ -36,10 +36,10 @@
 #if defined(PETSC_USE_COMPLEX)
 #error "MFEM does not work with PETSc compiled with complex numbers support"
 #endif
-#if defined(PETSC_USE_64BIT_INDICES) && !defined(HYPRE_BIGINT)
+#if defined(PETSC_USE_64BIT_INDICES) && !defined(HYPRE_BIGINT) && !defined(HYPRE_MIXEDINT)
 #error "Mismatch between HYPRE (32bit) and PETSc (64bit) integer types"
 #endif
-#if !defined(PETSC_USE_64BIT_INDICES) && defined(HYPRE_BIGINT)
+#if !defined(PETSC_USE_64BIT_INDICES) && (defined(HYPRE_BIGINT) || defined(HYPRE_MIXEDINT))
 #error "Mismatch between HYPRE (64bit) and PETSc (32bit) integer types"
 #endif
 

linalg/sundials.cpp

@@ -95,7 +95,7 @@ MFEM_DEPRECATED void* CVodeCreate(int lmm, SUNContext)
 
 /// (DEPRECATED) Wrapper function for backwards compatibility with SUNDIALS
 /// version < 6
-MFEM_DEPRECATED void* ARKStepCreate(ARKRhsFn fe, ARKRhsFn fi, realtype t0,
+MFEM_DEPRECATED void* ARKStepCreate(ARKRhsFn fe, ARKRhsFn fi, sunrealtype t0,
                                     N_Vector y0, SUNContext)
 {
    return ARKStepCreate(fe, fi, t0, y0);
@@ -127,7 +127,7 @@ MFEM_DEPRECATED N_Vector N_VNewEmpty_Parallel(MPI_Comm comm,
 /// (DEPRECATED) Wrapper function for backwards compatibility with SUNDIALS
 /// version < 6
 MFEM_DEPRECATED N_Vector SUN_Hip_OR_Cuda(N_VNewWithMemHelp)(sunindextype length,
-                                                            booleantype use_managed_mem,
+                                                            sunbooleantype use_managed_mem,
                                                             SUNMemoryHelper helper,
                                                             SUNContext)
 {
@@ -157,6 +157,16 @@ MFEM_DEPRECATED N_Vector N_VMake_MPIPlusX(MPI_Comm comm, N_Vector local_vector,
 
 #endif // SUNDIALS_VERSION_MAJOR < 6
 
+#if MFEM_SUNDIALS_VERSION < 70100
+#define MFEM_ARKode(FUNC) ARKStep##FUNC
+#else
+#define MFEM_ARKode(FUNC) ARKode##FUNC
+#endif
+
+// Macro STR(): expand the argument and add double quotes
+#define STR1(s) #s
+#define STR(s)  STR1(s)
+
 
 namespace mfem
 {
@@ -187,11 +197,21 @@ SundialsMemHelper &Sundials::GetMemHelper()
 Sundials::Sundials()
 {
 #ifdef MFEM_USE_MPI
-   MPI_Comm communicator = MPI_COMM_WORLD;
+   int mpi_initialized = 0;
+   MPI_Initialized(&mpi_initialized);
+   MPI_Comm communicator = mpi_initialized ? MPI_COMM_WORLD : MPI_COMM_NULL;
+#if SUNDIALS_VERSION_MAJOR < 7
    int return_val = SUNContext_Create((void*) &communicator, &context);
 #else
-   int return_val = SUNContext_Create(nullptr, &context);
+   int return_val = SUNContext_Create(communicator, &context);
 #endif
+#else // #ifdef MFEM_USE_MPI
+#if SUNDIALS_VERSION_MAJOR < 7
+   int return_val = SUNContext_Create(nullptr, &context);
+#else
+   int return_val = SUNContext_Create((SUNComm)(0), &context);
+#endif
+#endif // #ifdef MFEM_USE_MPI
    MFEM_VERIFY(return_val == 0, "Call to SUNContext_Create failed");
    SundialsMemHelper actual_helper(context);
    memHelper = std::move(actual_helper);
@@ -250,7 +270,11 @@ int SundialsMemHelper::SundialsMemHelper_Alloc(SUNMemoryHelper helper,
 #endif
                                               )
 {
+#if (SUNDIALS_VERSION_MAJOR < 7)
    SUNMemory sunmem = SUNMemoryNewEmpty();
+#else
+   SUNMemory sunmem = SUNMemoryNewEmpty(helper->sunctx);
+#endif
 
    sunmem->ptr = NULL;
    sunmem->own = SUNTRUE;
@@ -631,7 +655,7 @@ static int LSFree(SUNLinearSolver LS)
 // ---------------------------------------------------------------------------
 // CVODE interface
 // ---------------------------------------------------------------------------
-int CVODESolver::RHS(realtype t, const N_Vector y, N_Vector ydot,
+int CVODESolver::RHS(sunrealtype t, const N_Vector y, N_Vector ydot,
                      void *user_data)
 {
    // At this point the up-to-date data for N_Vector y and ydot is on the device.
@@ -648,7 +672,8 @@ int CVODESolver::RHS(realtype t, const N_Vector y, N_Vector ydot,
    return (0);
 }
 
-int CVODESolver::root(realtype t, N_Vector y, realtype *gout, void *user_data)
+int CVODESolver::root(sunrealtype t, N_Vector y, sunrealtype *gout,
+                      void *user_data)
 {
    CVODESolver *self = static_cast<CVODESolver*>(user_data);
 
@@ -668,8 +693,9 @@ void CVODESolver::SetRootFinder(int components, RootFunction func)
    MFEM_VERIFY(flag == CV_SUCCESS, "error in SetRootFinder()");
 }
 
-int CVODESolver::LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
-                             booleantype jok, booleantype *jcur, realtype gamma,
+int CVODESolver::LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy,
+                             SUNMatrix A, sunbooleantype jok,
+                             sunbooleantype *jcur, sunrealtype gamma,
                              void*, N_Vector, N_Vector, N_Vector)
 {
    // Get data from N_Vectors
@@ -683,7 +709,7 @@ int CVODESolver::LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
 }
 
 int CVODESolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
-                             N_Vector b, realtype tol)
+                             N_Vector b, sunrealtype tol)
 {
    SundialsNVector mfem_x(x);
    const SundialsNVector mfem_b(b);
@@ -859,7 +885,7 @@ void CVODESolver::UseSundialsLinearSolver()
    if (LSA != NULL) { SUNLinSolFree(LSA); LSA = NULL; }
 
    // Create linear solver
-   LSA = SUNLinSol_SPGMR(*Y, PREC_NONE, 0, Sundials::GetContext());
+   LSA = SUNLinSol_SPGMR(*Y, SUN_PREC_NONE, 0, Sundials::GetContext());
    MFEM_VERIFY(LSA, "error in SUNLinSol_SPGMR()");
 
    // Attach linear solver
@@ -1150,7 +1176,7 @@ void CVODESSolver::UseSundialsLinearSolverB()
    if (LSB != NULL) { SUNLinSolFree(LSB); LSB = NULL; }
 
    // Set default linear solver (Newton is the default Nonlinear Solver)
-   LSB = SUNLinSol_SPGMR(*yB, PREC_NONE, 0, Sundials::GetContext());
+   LSB = SUNLinSol_SPGMR(*yB, SUN_PREC_NONE, 0, Sundials::GetContext());
    MFEM_VERIFY(LSB, "error in SUNLinSol_SPGMR()");
 
    /* Attach the matrix and linear solver */
@@ -1158,11 +1184,11 @@ void CVODESSolver::UseSundialsLinearSolverB()
    MFEM_VERIFY(flag == CV_SUCCESS, "error in CVodeSetLinearSolverB()");
 }
 
-int CVODESSolver::LinSysSetupB(realtype t, N_Vector y, N_Vector yB,
+int CVODESSolver::LinSysSetupB(sunrealtype t, N_Vector y, N_Vector yB,
                                N_Vector fyB, SUNMatrix AB,
-                               booleantype jokB, booleantype *jcurB,
-                               realtype gammaB, void *user_data, N_Vector tmp1,
-                               N_Vector tmp2, N_Vector tmp3)
+                               sunbooleantype jokB, sunbooleantype *jcurB,
+                               sunrealtype gammaB, void *user_data,
+                               N_Vector tmp1, N_Vector tmp2, N_Vector tmp3)
 {
    // Get data from N_Vectors
    const SundialsNVector mfem_y(y);
@@ -1178,7 +1204,7 @@ int CVODESSolver::LinSysSetupB(realtype t, N_Vector y, N_Vector yB,
 }
 
 int CVODESSolver::LinSysSolveB(SUNLinearSolver LS, SUNMatrix AB, N_Vector yB,
-                               N_Vector Rb, realtype tol)
+                               N_Vector Rb, sunrealtype tol)
 {
    SundialsNVector mfem_yB(yB);
    const SundialsNVector mfem_Rb(Rb);
@@ -1216,7 +1242,7 @@ void CVODESSolver::SetWFTolerances(EWTFunction func)
 
 // CVODESSolver static functions
 
-int CVODESSolver::RHSQ(realtype t, const N_Vector y, N_Vector qdot,
+int CVODESSolver::RHSQ(sunrealtype t, const N_Vector y, N_Vector qdot,
                        void *user_data)
 {
    CVODESSolver *self = static_cast<CVODESSolver*>(user_data);
@@ -1229,7 +1255,7 @@ int CVODESSolver::RHSQ(realtype t, const N_Vector y, N_Vector qdot,
    return 0;
 }
 
-int CVODESSolver::RHSQB(realtype t, N_Vector y, N_Vector yB, N_Vector qBdot,
+int CVODESSolver::RHSQB(sunrealtype t, N_Vector y, N_Vector yB, N_Vector qBdot,
                         void *user_dataB)
 {
    CVODESSolver *self = static_cast<CVODESSolver*>(user_dataB);
@@ -1243,7 +1269,7 @@ int CVODESSolver::RHSQB(realtype t, N_Vector y, N_Vector yB, N_Vector qBdot,
    return 0;
 }
 
-int CVODESSolver::RHSB(realtype t, N_Vector y, N_Vector yB, N_Vector yBdot,
+int CVODESSolver::RHSB(sunrealtype t, N_Vector y, N_Vector yB, N_Vector yBdot,
                        void *user_dataB)
 {
    CVODESSolver *self = static_cast<CVODESSolver*>(user_dataB);
@@ -1341,7 +1367,7 @@ CVODESSolver::~CVODESSolver()
 // ARKStep interface
 // ---------------------------------------------------------------------------
 
-int ARKStepSolver::RHS1(realtype t, const N_Vector y, N_Vector result,
+int ARKStepSolver::RHS1(sunrealtype t, const N_Vector y, N_Vector result,
                         void *user_data)
 {
    // Get data from N_Vectors
@@ -1373,7 +1399,7 @@ int ARKStepSolver::RHS1(realtype t, const N_Vector y, N_Vector result,
    return (0);
 }
 
-int ARKStepSolver::RHS2(realtype t, const N_Vector y, N_Vector result,
+int ARKStepSolver::RHS2(sunrealtype t, const N_Vector y, N_Vector result,
                         void *user_data)
 {
    // Get data from N_Vectors
@@ -1399,9 +1425,9 @@ int ARKStepSolver::RHS2(realtype t, const N_Vector y, N_Vector result,
    return (0);
 }
 
-int ARKStepSolver::LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
-                               SUNMatrix, booleantype jok, booleantype *jcur,
-                               realtype gamma,
+int ARKStepSolver::LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy,
+                               SUNMatrix A, SUNMatrix, sunbooleantype jok,
+                               sunbooleantype *jcur, sunrealtype gamma,
                                void*, N_Vector, N_Vector, N_Vector)
 {
    // Get data from N_Vectors
@@ -1419,7 +1445,7 @@ int ARKStepSolver::LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
 }
 
 int ARKStepSolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
-                               N_Vector b, realtype tol)
+                               N_Vector b, sunrealtype tol)
 {
    SundialsNVector mfem_x(x);
    const SundialsNVector mfem_b(b);
@@ -1433,7 +1459,7 @@ int ARKStepSolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
    return (self->f->SUNImplicitSolve(mfem_b, mfem_x, tol));
 }
 
-int ARKStepSolver::MassSysSetup(realtype t, SUNMatrix M,
+int ARKStepSolver::MassSysSetup(sunrealtype t, SUNMatrix M,
                                 void*, N_Vector, N_Vector, N_Vector)
 {
    ARKStepSolver *self = static_cast<ARKStepSolver*>(GET_CONTENT(M));
@@ -1444,7 +1470,7 @@ int ARKStepSolver::MassSysSetup(realtype t, SUNMatrix M,
 }
 
 int ARKStepSolver::MassSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
-                                N_Vector b, realtype tol)
+                                N_Vector b, sunrealtype tol)
 {
    SundialsNVector mfem_x(x);
    const SundialsNVector mfem_b(b);
@@ -1464,7 +1490,7 @@ int ARKStepSolver::MassMult1(SUNMatrix M, N_Vector x, N_Vector v)
    return (self->f->SUNMassMult(mfem_x, mfem_v));
 }
 
-int ARKStepSolver::MassMult2(N_Vector x, N_Vector v, realtype t,
+int ARKStepSolver::MassMult2(N_Vector x, N_Vector v, sunrealtype t,
                              void* mtimes_data)
 {
    const SundialsNVector mfem_x(x);
@@ -1535,7 +1561,7 @@ void ARKStepSolver::Init(TimeDependentOperator &f_)
       // Free existing solver memory and re-create with new vector size
       if (resize)
       {
-         ARKStepFree(&sundials_mem);
+         MFEM_ARKode(Free)(&sundials_mem);
          sundials_mem = NULL;
       }
    }
@@ -1573,12 +1599,15 @@ void ARKStepSolver::Init(TimeDependentOperator &f_)
       MFEM_VERIFY(sundials_mem, "error in ARKStepCreate()");
 
       // Attach the ARKStepSolver as user-defined data
-      flag = ARKStepSetUserData(sundials_mem, this);
-      MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetUserData()");
+      flag = MFEM_ARKode(SetUserData)(sundials_mem, this);
+      MFEM_VERIFY(flag == ARK_SUCCESS,
+                  "error in " STR(MFEM_ARKode(SetUserData)) "()");
 
       // Set default tolerances
-      flag = ARKStepSStolerances(sundials_mem, default_rel_tol, default_abs_tol);
-      MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetSStolerances()");
+      flag = MFEM_ARKode(SStolerances)(sundials_mem, default_rel_tol,
+                                       default_abs_tol);
+      MFEM_VERIFY(flag == ARK_SUCCESS,
+                  "error in " STR(MFEM_ARKode(SStolerances)) "()");
 
       // If implicit, attach MFEM linear solver by default
       if (use_implicit) { UseMFEMLinearSolver(); }
@@ -1617,15 +1646,16 @@ void ARKStepSolver::Step(Vector &x, real_t &t, real_t &dt)
 
    // Integrate the system
    double tout = t + dt;
-   flag = ARKStepEvolve(sundials_mem, tout, *Y, &t, step_mode);
-   MFEM_VERIFY(flag >= 0, "error in ARKStepEvolve()");
+   flag = MFEM_ARKode(Evolve)(sundials_mem, tout, *Y, &t, step_mode);
+   MFEM_VERIFY(flag >= 0, "error in " STR(MFEM_ARKode(Evolve)) "()");
 
    // Make sure host is up to date
    Y->HostRead();
 
    // Return the last incremental step size
-   flag = ARKStepGetLastStep(sundials_mem, &dt);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepGetLastStep()");
+   flag = MFEM_ARKode(GetLastStep)(sundials_mem, &dt);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(GetLastStep)) "()");
 }
 
 void ARKStepSolver::UseMFEMLinearSolver()
@@ -1651,12 +1681,14 @@ void ARKStepSolver::UseMFEMLinearSolver()
    A->ops->destroy = MatDestroy;
 
    // Attach the linear solver and matrix
-   flag = ARKStepSetLinearSolver(sundials_mem, LSA, A);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetLinearSolver()");
+   flag = MFEM_ARKode(SetLinearSolver)(sundials_mem, LSA, A);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetLinearSolver)) "()");
 
    // Set the linear system evaluation function
-   flag = ARKStepSetLinSysFn(sundials_mem, ARKStepSolver::LinSysSetup);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetLinSysFn()");
+   flag = MFEM_ARKode(SetLinSysFn)(sundials_mem, ARKStepSolver::LinSysSetup);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetLinSysFn)) "()");
 }
 
 void ARKStepSolver::UseSundialsLinearSolver()
@@ -1666,12 +1698,13 @@ void ARKStepSolver::UseSundialsLinearSolver()
    if (LSA != NULL) { SUNLinSolFree(LSA); LSA = NULL; }
 
    // Create linear solver
-   LSA = SUNLinSol_SPGMR(*Y, PREC_NONE, 0, Sundials::GetContext());
+   LSA = SUNLinSol_SPGMR(*Y, SUN_PREC_NONE, 0, Sundials::GetContext());
    MFEM_VERIFY(LSA, "error in SUNLinSol_SPGMR()");
 
    // Attach linear solver
-   flag = ARKStepSetLinearSolver(sundials_mem, LSA, NULL);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetLinearSolver()");
+   flag = MFEM_ARKode(SetLinearSolver)(sundials_mem, LSA, NULL);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetLinearSolver)) "()");
 }
 
 void ARKStepSolver::UseMFEMMassLinearSolver(int tdep)
@@ -1698,12 +1731,14 @@ void ARKStepSolver::UseMFEMMassLinearSolver(int tdep)
    M->ops->destroy = MatDestroy;
 
    // Attach the linear solver and matrix
-   flag = ARKStepSetMassLinearSolver(sundials_mem, LSM, M, tdep);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetLinearSolver()");
+   flag = MFEM_ARKode(SetMassLinearSolver)(sundials_mem, LSM, M, tdep);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMassLinearSolver)) "()");
 
    // Set the linear system function
-   flag = ARKStepSetMassFn(sundials_mem, ARKStepSolver::MassSysSetup);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMassFn()");
+   flag = MFEM_ARKode(SetMassFn)(sundials_mem, ARKStepSolver::MassSysSetup);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMassFn)) "()");
 
    // Check that the ODE is not expressed in EXPLICIT form
    MFEM_VERIFY(!f->isExplicit(), "ODE operator is expressed in EXPLICIT form")
@@ -1716,17 +1751,19 @@ void ARKStepSolver::UseSundialsMassLinearSolver(int tdep)
    if (LSM != NULL) { SUNLinSolFree(LSM); LSM = NULL; }
 
    // Create linear solver
-   LSM = SUNLinSol_SPGMR(*Y, PREC_NONE, 0, Sundials::GetContext());
+   LSM = SUNLinSol_SPGMR(*Y, SUN_PREC_NONE, 0, Sundials::GetContext());
    MFEM_VERIFY(LSM, "error in SUNLinSol_SPGMR()");
 
    // Attach linear solver
-   flag = ARKStepSetMassLinearSolver(sundials_mem, LSM, NULL, tdep);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMassLinearSolver()");
+   flag = MFEM_ARKode(SetMassLinearSolver)(sundials_mem, LSM, NULL, tdep);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMassLinearSolver)) "()");
 
    // Attach matrix multiplication function
-   flag = ARKStepSetMassTimes(sundials_mem, NULL, ARKStepSolver::MassMult2,
-                              this);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMassTimes()");
+   flag = MFEM_ARKode(SetMassTimes)(sundials_mem, NULL,
+                                    ARKStepSolver::MassMult2, this);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMassTimes)) "()");
 
    // Check that the ODE is not expressed in EXPLICIT form
    MFEM_VERIFY(!f->isExplicit(), "ODE operator is expressed in EXPLICIT form")
@@ -1739,20 +1776,23 @@ void ARKStepSolver::SetStepMode(int itask)
 
 void ARKStepSolver::SetSStolerances(double reltol, double abstol)
 {
-   flag = ARKStepSStolerances(sundials_mem, reltol, abstol);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSStolerances()");
+   flag = MFEM_ARKode(SStolerances)(sundials_mem, reltol, abstol);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SStolerances)) "()");
 }
 
 void ARKStepSolver::SetMaxStep(double dt_max)
 {
-   flag = ARKStepSetMaxStep(sundials_mem, dt_max);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMaxStep()");
+   flag = MFEM_ARKode(SetMaxStep)(sundials_mem, dt_max);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMaxStep)) "()");
 }
 
 void ARKStepSolver::SetOrder(int order)
 {
-   flag = ARKStepSetOrder(sundials_mem, order);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetOrder()");
+   flag = MFEM_ARKode(SetOrder)(sundials_mem, order);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetOrder)) "()");
 }
 
 void ARKStepSolver::SetERKTableNum(ARKODE_ERKTableID table_id)
@@ -1776,8 +1816,9 @@ void ARKStepSolver::SetIMEXTableNum(ARKODE_ERKTableID etable_id,
 
 void ARKStepSolver::SetFixedStep(double dt)
 {
-   flag = ARKStepSetFixedStep(sundials_mem, dt);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetFixedStep()");
+   flag = MFEM_ARKode(SetFixedStep)(sundials_mem, dt);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetFixedStep)) "()");
 }
 
 void ARKStepSolver::PrintInfo() const
@@ -1799,18 +1840,19 @@ void ARKStepSolver::PrintInfo() const
                                      &netfails);
    MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepGetTimestepperStats()");
 
-   flag = ARKStepGetStepStats(sundials_mem,
-                              &nsteps,
-                              &hinused,
-                              &hlast,
-                              &hcur,
-                              &tcur);
+   flag = MFEM_ARKode(GetStepStats)(sundials_mem,
+                                    &nsteps,
+                                    &hinused,
+                                    &hlast,
+                                    &hcur,
+                                    &tcur);
 
    // Get nonlinear solver stats
-   flag = ARKStepGetNonlinSolvStats(sundials_mem,
-                                    &nniters,
-                                    &nncfails);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepGetNonlinSolvStats()");
+   flag = MFEM_ARKode(GetNonlinSolvStats)(sundials_mem,
+                                          &nniters,
+                                          &nncfails);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(GetNonlinSolvStats)) "()");
 
    mfem::out <<
              "ARKStep:\n"
@@ -1838,7 +1880,7 @@ ARKStepSolver::~ARKStepSolver()
    SUNMatDestroy(A);
    SUNLinSolFree(LSA);
    SUNNonlinSolFree(NLS);
-   ARKStepFree(&sundials_mem);
+   MFEM_ARKode(Free)(&sundials_mem);
 }
 
 // ---------------------------------------------------------------------------
@@ -1861,7 +1903,7 @@ int KINSolver::Mult(const N_Vector u, N_Vector fu, void *user_data)
 
 // Wrapper for computing Jacobian-vector products
 int KINSolver::GradientMult(N_Vector v, N_Vector Jv, N_Vector u,
-                            booleantype *new_u, void *user_data)
+                            sunbooleantype *new_u, void *user_data)
 {
    const SundialsNVector mfem_v(v);
    SundialsNVector mfem_Jv(Jv);
@@ -1901,7 +1943,7 @@ int KINSolver::LinSysSetup(N_Vector u, N_Vector, SUNMatrix J,
 
 // Wrapper for solving linear systems J u = b
 int KINSolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector u,
-                           N_Vector b, realtype)
+                           N_Vector b, sunrealtype)
 {
    SundialsNVector mfem_u(u), mfem_b(b);
    KINSolver *self = static_cast<KINSolver*>(GET_CONTENT(LS));
@@ -1960,7 +2002,11 @@ KINSolver::KINSolver(int strategy, bool oper_grad)
    f_scale = new SundialsNVector();
 
    // Default abs_tol and print_level
+#if MFEM_SUNDIALS_VERSION < 70000
    abs_tol     = pow(UNIT_ROUNDOFF, 1.0/3.0);
+#else
+   abs_tol     = pow(SUN_UNIT_ROUNDOFF, 1.0/3.0);
+#endif
    print_level = 0;
 }
 
@@ -1974,7 +2020,11 @@ KINSolver::KINSolver(MPI_Comm comm, int strategy, bool oper_grad)
    f_scale = new SundialsNVector(comm);
 
    // Default abs_tol and print_level
+#if MFEM_SUNDIALS_VERSION < 70000
    abs_tol     = pow(UNIT_ROUNDOFF, 1.0/3.0);
+#else
+   abs_tol     = pow(SUN_UNIT_ROUNDOFF, 1.0/3.0);
+#endif
    print_level = 0;
 }
 #endif
@@ -2086,7 +2136,7 @@ void KINSolver::SetOperator(const Operator &op)
          if (A != NULL) { SUNMatDestroy(A); A = NULL; }
          if (LSA != NULL) { SUNLinSolFree(LSA); LSA = NULL; }
 
-         LSA = SUNLinSol_SPGMR(*Y, PREC_NONE, 0, Sundials::GetContext());
+         LSA = SUNLinSol_SPGMR(*Y, SUN_PREC_NONE, 0, Sundials::GetContext());
          MFEM_VERIFY(LSA, "error in SUNLinSol_SPGMR()");
 
          flag = KINSetLinearSolver(sundials_mem, LSA, NULL);
@@ -2155,12 +2205,12 @@ void KINSolver::SetJFNKSolver(Solver &solver)
    if (LSA != NULL) { SUNLinSolFree(LSA); LSA = NULL; }
 
    // Setup FGMRES
-   LSA = SUNLinSol_SPFGMR(*Y, prec ? PREC_RIGHT : PREC_NONE, maxli,
+   LSA = SUNLinSol_SPFGMR(*Y, prec ? SUN_PREC_RIGHT : SUN_PREC_NONE, maxli,
                           Sundials::GetContext());
    MFEM_VERIFY(LSA, "error in SUNLinSol_SPFGMR()");
 
    flag = SUNLinSol_SPFGMRSetMaxRestarts(LSA, maxlrs);
-   MFEM_VERIFY(flag == SUNLS_SUCCESS, "error in SUNLinSol_SPFGMR()");
+   MFEM_VERIFY(flag == SUN_SUCCESS, "error in SUNLinSol_SPFGMR()");
 
    flag = KINSetLinearSolver(sundials_mem, LSA, NULL);
    MFEM_VERIFY(flag == KIN_SUCCESS, "error in KINSetLinearSolver()");
@@ -2317,18 +2367,21 @@ void KINSolver::Mult(Vector &x,
 
    if (rank == 0)
    {
+#if MFEM_SUNDIALS_VERSION < 70000
       flag = KINSetPrintLevel(sundials_mem, print_level);
       MFEM_VERIFY(flag == KIN_SUCCESS, "KINSetPrintLevel() failed!");
+#endif
+      // NOTE: there is no KINSetPrintLevel in SUNDIALS v7!
 
 #ifdef SUNDIALS_BUILD_WITH_MONITORING
       if (jfnk && print_level)
       {
          flag = SUNLinSolSetInfoFile_SPFGMR(LSA, stdout);
-         MFEM_VERIFY(flag == SUNLS_SUCCESS,
+         MFEM_VERIFY(flag == SUN_SUCCESS,
                      "error in SUNLinSolSetInfoFile_SPFGMR()");
 
          flag = SUNLinSolSetPrintLevel_SPFGMR(LSA, 1);
-         MFEM_VERIFY(flag == SUNLS_SUCCESS,
+         MFEM_VERIFY(flag == SUN_SUCCESS,
                      "error in SUNLinSolSetPrintLevel_SPFGMR()");
       }
 #endif

linalg/sundials.hpp

@@ -54,6 +54,10 @@
 
 #include <functional>
 
+#define MFEM_SUNDIALS_VERSION \
+   (SUNDIALS_VERSION_MAJOR*10000 + SUNDIALS_VERSION_MINOR*100 + \
+    SUNDIALS_VERSION_PATCH)
+
 #if (SUNDIALS_VERSION_MAJOR < 6)
 
 /// (DEPRECATED) Map SUNDIALS version >= 6 datatypes and constants to
@@ -68,13 +72,30 @@ constexpr ARKODE_ERKTableID ARKODE_FEHLBERG_13_7_8 = FEHLBERG_13_7_8;
 /// arbitrary type for more compact backwards compatibility
 using SUNContext = void*;
 
+/// 'sunrealtype' was first introduced in v6.0.0
+typedef realtype sunrealtype;
+/// 'sunbooleantype' was first introduced in v6.0.0
+typedef booleantype sunbooleantype;
+
+/// New constant names introduced in v6.0.0
+enum { SUN_PREC_NONE, SUN_PREC_LEFT, SUN_PREC_RIGHT, SUN_PREC_BOTH };
+
 // KIN_ORTH_MGS was introduced in SUNDIALS v6; here, we define it just so that
 // it can be used as the default option in the second parameter of
 // KINSolver::EnableAndersonAcc -- the actual value of the parameter will be
 // ignored when using SUNDIALS < v6.
 #define KIN_ORTH_MGS 0
 
-#endif // SUNDIALS_VERSION_MAJOR < 6
+#endif // #if SUNDIALS_VERSION_MAJOR < 6
+
+#if (SUNDIALS_VERSION_MAJOR < 7)
+
+/** @brief The enum constant SUN_SUCCESS was added in v7 as a replacement of
+    various *_SUCCESS macros that were removed in v7. */
+enum { SUN_SUCCESS = 0 };
+
+#endif // #if SUNDIALS_VERSION_MAJOR < 7
+
 
 namespace mfem
 {
@@ -244,7 +265,14 @@ public:
 
 #ifdef MFEM_USE_MPI
    /// Returns the MPI communicator for the internal N_Vector x.
-   inline MPI_Comm GetComm() const { return *static_cast<MPI_Comm*>(N_VGetCommunicator(x)); }
+   inline MPI_Comm GetComm() const
+   {
+#if SUNDIALS_VERSION_MAJOR < 7
+      return *static_cast<MPI_Comm*>(N_VGetCommunicator(x));
+#else
+      return N_VGetCommunicator(x);
+#endif
+   }
 
    /// Returns the MPI global length for the internal N_Vector x.
    inline long GlobalSize() const { return N_VGetLength(x); }
@@ -396,24 +424,26 @@ protected:
    int root_components; /// Number of components in gout
 
    /// Wrapper to compute the ODE rhs function.
-   static int RHS(realtype t, const N_Vector y, N_Vector ydot, void *user_data);
+   static int RHS(sunrealtype t, const N_Vector y, N_Vector ydot,
+                  void *user_data);
 
    /// Setup the linear system $ A x = b $.
-   static int LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
-                          booleantype jok, booleantype *jcur,
-                          realtype gamma, void *user_data, N_Vector tmp1,
+   static int LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy, SUNMatrix A,
+                          sunbooleantype jok, sunbooleantype *jcur,
+                          sunrealtype gamma, void *user_data, N_Vector tmp1,
                           N_Vector tmp2, N_Vector tmp3);
 
    /// Solve the linear system $ A x = b $.
    static int LinSysSolve(SUNLinearSolver LS, SUNMatrix A, N_Vector x,
-                          N_Vector b, realtype tol);
+                          N_Vector b, sunrealtype tol);
 
    /// Prototype to define root finding for CVODE
-   static int root(realtype t, N_Vector y, realtype *gout, void *user_data);
+   static int root(sunrealtype t, N_Vector y, sunrealtype *gout,
+                   void *user_data);
 
    /// Typedef for root finding functions
-   typedef std::function<int(realtype t, Vector y, Vector gout, CVODESolver *)>
-   RootFunction;
+   typedef std::function<int(sunrealtype t, Vector y, Vector gout,
+                             CVODESolver *)> RootFunction;
 
    /// A class member to facilitate pointing to a user-specified root function
    RootFunction root_func;
@@ -421,7 +451,8 @@ protected:
    /// Typedef declaration for error weight functions
    typedef std::function<int(Vector y, Vector w, CVODESolver*)> EWTFunction;
 
-   /// A class member to facilitate pointing to a user-specified error weight function
+   /** @brief A class member to facilitate pointing to a user-specified error
+       weight function */
    EWTFunction ewt_func;
 
 public:
@@ -455,7 +486,7 @@ public:
        @note If this method is called a second time with a different problem
        size, then any non-default user-set options will be lost and will need
        to be set again. */
-   void Init(TimeDependentOperator &f_);
+   void Init(TimeDependentOperator &f_) override;
 
    /// Integrate the ODE with CVODE using the specified step mode.
    /** @param[in,out] x  On output, the solution vector at the requested output
@@ -531,14 +562,15 @@ protected:
    int indexB; ///< backward problem index
 
    /// Wrapper to compute the ODE RHS Quadrature function.
-   static int RHSQ(realtype t, const N_Vector y, N_Vector qdot, void *user_data);
+   static int RHSQ(sunrealtype t, const N_Vector y, N_Vector qdot,
+                   void *user_data);
 
    /// Wrapper to compute the ODE RHS backward function.
-   static int RHSB(realtype t, N_Vector y,
+   static int RHSB(sunrealtype t, N_Vector y,
                    N_Vector yB, N_Vector yBdot, void *user_dataB);
 
    /// Wrapper to compute the ODE RHS Backwards Quadrature function.
-   static int RHSQB(realtype t, N_Vector y, N_Vector yB,
+   static int RHSQB(sunrealtype t, N_Vector y, N_Vector yB,
                     N_Vector qBdot, void *user_dataB);
 
    /// Error control function
@@ -654,15 +686,15 @@ public:
    void SetSVtolerancesB(double reltol, Vector abstol);
 
    /// Setup the linear system A x = b
-   static int LinSysSetupB(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
+   static int LinSysSetupB(sunrealtype t, N_Vector y, N_Vector yB, N_Vector fyB,
                            SUNMatrix A,
-                           booleantype jok, booleantype *jcur,
-                           realtype gamma, void *user_data, N_Vector tmp1,
+                           sunbooleantype jok, sunbooleantype *jcur,
+                           sunrealtype gamma, void *user_data, N_Vector tmp1,
                            N_Vector tmp2, N_Vector tmp3);
 
    /// Solve the linear system A x = b
    static int LinSysSolveB(SUNLinearSolver LS, SUNMatrix A, N_Vector x,
-                           N_Vector b, realtype tol);
+                           N_Vector b, sunrealtype tol);
 
 
    /// Destroy the associated CVODES memory and SUNDIALS objects.
@@ -695,33 +727,35 @@ protected:
        RHS1 is explicit RHS and RHS2 the implicit RHS for IMEX integration. When
        purely implicit or explicit only RHS1 is used. */
    ///@{
-   static int RHS1(realtype t, const N_Vector y, N_Vector ydot, void *user_data);
-   static int RHS2(realtype t, const N_Vector y, N_Vector ydot, void *user_data);
+   static int RHS1(sunrealtype t, const N_Vector y, N_Vector ydot,
+                   void *user_data);
+   static int RHS2(sunrealtype t, const N_Vector y, N_Vector ydot,
+                   void *user_data);
    ///@}
 
    /// Setup the linear system $ A x = b $.
-   static int LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
-                          SUNMatrix M, booleantype jok, booleantype *jcur,
-                          realtype gamma, void *user_data, N_Vector tmp1,
+   static int LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy, SUNMatrix A,
+                          SUNMatrix M, sunbooleantype jok, sunbooleantype *jcur,
+                          sunrealtype gamma, void *user_data, N_Vector tmp1,
                           N_Vector tmp2, N_Vector tmp3);
 
    /// Solve the linear system $ A x = b $.
    static int LinSysSolve(SUNLinearSolver LS, SUNMatrix A, N_Vector x,
-                          N_Vector b, realtype tol);
+                          N_Vector b, sunrealtype tol);
 
    /// Setup the linear system $ M x = b $.
-   static int MassSysSetup(realtype t, SUNMatrix M, void *user_data,
+   static int MassSysSetup(sunrealtype t, SUNMatrix M, void *user_data,
                            N_Vector tmp1, N_Vector tmp2, N_Vector tmp3);
 
    /// Solve the linear system $ M x = b $.
    static int MassSysSolve(SUNLinearSolver LS, SUNMatrix M, N_Vector x,
-                           N_Vector b, realtype tol);
+                           N_Vector b, sunrealtype tol);
 
    /// Compute the matrix-vector product $ v = M x $.
    static int MassMult1(SUNMatrix M, N_Vector x, N_Vector v);
 
    /// Compute the matrix-vector product $v = M_t x $ at time t.
-   static int MassMult2(N_Vector x, N_Vector v, realtype t,
+   static int MassMult2(N_Vector x, N_Vector v, sunrealtype t,
                         void* mtimes_data);
 
 public:
@@ -757,7 +791,7 @@ public:
        @note If this method is called a second time with a different problem
        size, then any non-default user-set options will be lost and will need
        to be set again. */
-   void Init(TimeDependentOperator &f_);
+   void Init(TimeDependentOperator &f_) override;
 
    /// Integrate the ODE with ARKode using the specified step mode.
    /**
@@ -871,7 +905,7 @@ protected:
 
    /// Wrapper to compute the Jacobian-vector product $ J(u) v = Jv $.
    static int GradientMult(N_Vector v, N_Vector Jv, N_Vector u,
-                           booleantype *new_u, void *user_data);
+                           sunbooleantype *new_u, void *user_data);
 
    /// Setup the linear system $ J u = b $.
    static int LinSysSetup(N_Vector u, N_Vector fu, SUNMatrix J,
@@ -879,7 +913,7 @@ protected:
 
    /// Solve the linear system $ J u = b $.
    static int LinSysSolve(SUNLinearSolver LS, SUNMatrix J, N_Vector u,
-                          N_Vector b, realtype tol);
+                          N_Vector b, sunrealtype tol);
 
    /// Setup the preconditioner.
    static int PrecSetup(N_Vector uu,

mesh/CMakeLists.txt

@@ -32,6 +32,7 @@ set(SRCS
   vtk.cpp
   wedge.cpp
   submesh/submesh.cpp
+  submesh/ncsubmesh.cpp
   submesh/submesh_utils.cpp
   submesh/transfermap.cpp
   )
@@ -58,6 +59,7 @@ set(HDRS
   vertex.hpp
   vtk.hpp
   wedge.hpp
+  submesh/ncsubmesh.hpp
   submesh/submesh.hpp
   submesh/submesh_utils.hpp
   submesh/transfer_category.hpp
@@ -68,15 +70,17 @@ if (MFEM_USE_MPI)
   list(APPEND SRCS
     pmesh.cpp
     pncmesh.cpp
-    submesh/ptransfermap.cpp
-    submesh/psubmesh.cpp)
+    submesh/pncsubmesh.cpp
+    submesh/psubmesh.cpp
+    submesh/ptransfermap.cpp)
   # If this list (HDRS -> HEADERS) is used for install, we probably want the
   # headers added all the time.
   list(APPEND HDRS
     pmesh.hpp
     pncmesh.hpp
-    submesh/ptransfermap.hpp
-    submesh/psubmesh.hpp)
+    submesh/pncsubmesh.hpp
+    submesh/psubmesh.hpp
+    submesh/ptransfermap.hpp)
 endif()
 
 if (MFEM_USE_PUMI)

mesh/mesh.cpp

@@ -2033,6 +2033,18 @@ int Mesh::AddBdrElement(Element *elem)
    return NumOfBdrElements++;
 }
 
+void Mesh::AddBdrElements(Array<Element *> &bdr_elems,
+                          const Array<int> &new_be_to_face)
+{
+   boundary.Reserve(boundary.Size() + bdr_elems.Size());
+   MFEM_ASSERT(bdr_elems.Size() == new_be_to_face.Size(), "wrong size");
+   for (int i = 0; i < bdr_elems.Size(); i++)
+   {
+      AddBdrElement(bdr_elems[i]);
+   }
+   be_to_face.Append(new_be_to_face);
+}
+
 int Mesh::AddBdrSegment(int v1, int v2, int attr)
 {
    CheckEnlarge(boundary, NumOfBdrElements);
@@ -7346,6 +7358,12 @@ void Mesh::GetBdrElementAdjacentElement2(
    info = fi.Elem1Inf + ori;
 }
 
+void Mesh::SetAttribute(int i, int attr)
+{
+  elements[i]->SetAttribute(attr);
+  if (ncmesh) ncmesh->SetAttribute(i, attr);
+}
+
 Element::Type Mesh::GetElementType(int i) const
 {
    return elements[i]->GetType();
@@ -7672,7 +7690,6 @@ void Mesh::AddQuadFaceElement(int lf, int gf, int el,
 void Mesh::GenerateFaces()
 {
    int nfaces = GetNumFaces();
-
    for (auto &f : faces)
    {
       FreeElement(f);

mesh/mesh.hpp

@@ -993,6 +993,17 @@ public:
    /// @note Ownership of @a elem will pass to the Mesh object
    int AddBdrElement(Element *elem);
 
+   /**
+    * @brief Add an array of boundary elements to the mesh, along with map from
+    * the elements to their faces
+    * @param[in] bdr_elems The set of boundary element pointers, ownership of
+    * the pointers will be transferred to the Mesh object
+    * @param[in] be_to_face The map from the boundary element index to the face
+    * index
+    */
+   void AddBdrElements(Array<Element *> &bdr_elems,
+                       const Array<int> &be_to_face);
+
    int AddBdrSegment(int v1, int v2, int attr = 1);
    int AddBdrSegment(const int *vi, int attr = 1);
 
@@ -1102,6 +1113,15 @@ public:
        have two adjacent faces in 3D, or edges in 2D. */
    void RemoveInternalBoundaries();
 
+   /**
+    * @brief Clear the boundary element to edge map.
+    */
+   void DeleteBoundaryElementToEdge()
+   {
+      delete bel_to_edge;
+      bel_to_edge = nullptr;
+   }
+
    /// @}
 
    /// @name Element ordering methods
@@ -1366,7 +1386,7 @@ public:
    int GetAttribute(int i) const { return elements[i]->GetAttribute(); }
 
    /// Set the attribute of element i.
-   void SetAttribute(int i, int attr) { elements[i]->SetAttribute(attr); }
+   void SetAttribute(int i, int attr);
 
    /// Return the attribute of boundary element i.
    int GetBdrAttribute(int i) const { return boundary[i]->GetAttribute(); }

mesh/mesh_headers.hpp

@@ -25,6 +25,7 @@
 #include "ncmesh.hpp"
 #include "mesh.hpp"
 #include "mesh_operators.hpp"
+#include "submesh/ncsubmesh.hpp"
 #include "submesh/submesh.hpp"
 #include "submesh/submesh_utils.hpp"
 #include "submesh/transfermap.hpp"
@@ -36,6 +37,7 @@
 #ifdef MFEM_USE_MPI
 #include "pncmesh.hpp"
 #include "pmesh.hpp"
+#include "submesh/pncsubmesh.hpp"
 #include "submesh/psubmesh.hpp"
 #include "submesh/ptransfermap.hpp"
 #endif

mesh/ncmesh.cpp

@@ -58,23 +58,25 @@ void NCMesh::GeomInfo::InitGeom(Geometry::Type geom)
 {
    if (initialized) { return; }
 
-   mfem::Element *elem = NULL;
-   switch (geom)
+   auto elem = [&]()
    {
-      case Geometry::CUBE: elem = new Hexahedron; break;
-      case Geometry::PRISM: elem = new Wedge; break;
-      case Geometry::TETRAHEDRON: elem = new Tetrahedron; break;
-      case Geometry::PYRAMID: elem = new Pyramid; break;
-      case Geometry::SQUARE: elem = new Quadrilateral; break;
-      case Geometry::TRIANGLE: elem = new Triangle; break;
-      case Geometry::SEGMENT: elem = new Segment; break;
-      default: MFEM_ABORT("unsupported geometry " << geom);
-   }
+      switch (geom)
+      {
+         case Geometry::CUBE: return std::unique_ptr<mfem::Element>(new Hexahedron);
+         case Geometry::PRISM: return std::unique_ptr<mfem::Element>(new Wedge);
+         case Geometry::TETRAHEDRON: return std::unique_ptr<mfem::Element>
+                                               (new Tetrahedron);
+         case Geometry::PYRAMID: return std::unique_ptr<mfem::Element>(new Pyramid);
+         case Geometry::SQUARE: return std::unique_ptr<mfem::Element>(new Quadrilateral);
+         case Geometry::TRIANGLE: return std::unique_ptr<mfem::Element>(new Triangle);
+         case Geometry::SEGMENT: return std::unique_ptr<mfem::Element>(new Segment);
+         default: MFEM_ABORT("unsupported geometry " << geom);
+      }
+   }();
 
    nv = elem->GetNVertices();
    ne = elem->GetNEdges();
    nf = elem->GetNFaces();
-
    for (int i = 0; i < ne; i++)
    {
       for (int j = 0; j < 2; j++)
@@ -119,19 +121,9 @@ void NCMesh::GeomInfo::InitGeom(Geometry::Type geom)
       }
    }
 
-   delete elem;
    initialized = true;
 }
 
-static void CheckSupportedGeom(Geometry::Type geom)
-{
-   MFEM_VERIFY(geom == Geometry::SEGMENT ||
-               geom == Geometry::TRIANGLE || geom == Geometry::SQUARE ||
-               geom == Geometry::CUBE || geom == Geometry::PRISM ||
-               geom == Geometry::PYRAMID || geom == Geometry::TETRAHEDRON,
-               "Element type " << geom << " is not supported by NCMesh.");
-}
-
 NCMesh::NCMesh(const Mesh *mesh)
    : shadow(1024, 2048)
 {
@@ -157,7 +149,7 @@ NCMesh::NCMesh(const Mesh *mesh)
       }
 
       // create NCMesh::Element for this mfem::Element
-      int root_id = AddElement(Element(geom, elem->GetAttribute()));
+      int root_id = AddElement(geom, elem->GetAttribute());
       MFEM_ASSERT(root_id == i, "");
       Element &root_elem = elements[root_id];
 
@@ -248,11 +240,18 @@ NCMesh::NCMesh(const NCMesh &other)
    , nodes(other.nodes)
    , faces(other.faces)
    , elements(other.elements)
+   , free_element_ids(other.free_element_ids)
+   , root_state(other.root_state)
+   , coordinates(other.coordinates)
+   , NEdges(other.NEdges)
+   , NFaces(other.NFaces)
+   , NGhostEdges(other.NGhostEdges)
+   , NGhostFaces(other.NGhostFaces)
+   , boundary_faces(other.boundary_faces)
+   , face_geom(other.face_geom)
+   , element_vertex(other.element_vertex)
    , shadow(1024, 2048)
 {
-   other.free_element_ids.Copy(free_element_ids);
-   other.root_state.Copy(root_state);
-   other.coordinates.Copy(coordinates);
    Update();
 }
 
@@ -351,8 +350,8 @@ int NCMesh::GetMidFaceNode(int en1, int en2, int en3, int en4)
 
 void NCMesh::ReferenceElement(int elem)
 {
-   Element &el = elements[elem];
-   int* node = el.node;
+   const Element &el = elements[elem];
+   const int* node = el.node;
    GeomInfo& gi = GI[el.Geom()];
 
    // reference all vertices
@@ -507,7 +506,7 @@ int NCMesh::NewHexahedron(int n0, int n1, int n2, int n3,
                           int fattr3, int fattr4, int fattr5)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::CUBE, attr));
+   int new_id = AddElement(Geometry::CUBE, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2, el.node[3] = n3;
@@ -537,7 +536,7 @@ int NCMesh::NewWedge(int n0, int n1, int n2,
                      int fattr2, int fattr3, int fattr4)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::PRISM, attr));
+   int new_id = AddElement(Geometry::PRISM, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2;
@@ -566,7 +565,7 @@ int NCMesh::NewTetrahedron(int n0, int n1, int n2, int n3, int attr,
                            int fattr0, int fattr1, int fattr2, int fattr3)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::TETRAHEDRON, attr));
+   int new_id = AddElement(Geometry::TETRAHEDRON, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2, el.node[3] = n3;
@@ -592,7 +591,7 @@ int NCMesh::NewPyramid(int n0, int n1, int n2, int n3, int n4, int attr,
                        int fattr4)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::PYRAMID, attr));
+   int new_id = AddElement(Geometry::PYRAMID, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2, el.node[3] = n3;
@@ -622,7 +621,7 @@ int NCMesh::NewQuadrilateral(int n0, int n1, int n2, int n3,
                              int eattr0, int eattr1, int eattr2, int eattr3)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::SQUARE, attr));
+   int new_id = AddElement(Geometry::SQUARE, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2, el.node[3] = n3;
@@ -647,7 +646,7 @@ int NCMesh::NewTriangle(int n0, int n1, int n2,
                         int attr, int eattr0, int eattr1, int eattr2)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::TRIANGLE, attr));
+   int new_id = AddElement(Geometry::TRIANGLE, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2;
@@ -672,7 +671,7 @@ int NCMesh::NewTriangle(int n0, int n1, int n2,
 int NCMesh::NewSegment(int n0, int n1, int attr, int vattr1, int vattr2)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::SEGMENT, attr));
+   int new_id = AddElement(Geometry::SEGMENT, attr);
    Element &el = elements[new_id];
    el.node[0] = n0, el.node[1] = n1;
 
@@ -2167,7 +2166,6 @@ void NCMesh::UpdateLeafElements()
    // final (Mesh) indices of leaves
    leaf_elements.Append(ghosts);
    leaf_sfc_index.SetSize(leaf_elements.Size());
-
    for (int i = 0; i < leaf_elements.Size(); i++)
    {
       Element &el = elements[leaf_elements[i]];
@@ -2234,7 +2232,6 @@ void NCMesh::UpdateVertices()
    }
 
    // STEP 2: assign indices of top-level local vertices, in original order
-
    NVertices = 0;
    for (auto &node : nodes)
    {
@@ -2246,7 +2243,6 @@ void NCMesh::UpdateVertices()
 
    // STEP 3: go over all elements (local and ghost) in SFC order and assign
    // remaining local vertices in that order.
-
    Array<int> sfc_order(leaf_elements.Size());
    for (int i = 0; i < sfc_order.Size(); i++)
    {
@@ -2264,7 +2260,6 @@ void NCMesh::UpdateVertices()
    }
 
    // STEP 4: create the mapping from Mesh vertex index to NCMesh node index
-
    vertex_nodeId.SetSize(NVertices);
    for (auto node = nodes.begin(); node != nodes.end(); ++node)
    {
@@ -2277,7 +2272,6 @@ void NCMesh::UpdateVertices()
 
    // STEP 5: assign remaining ghost vertices, ignore vertices beyond the ghost
    // layer
-
    NGhostVertices = 0;
    for (int i = 0; i < sfc_order.Size(); i++)
    {
@@ -2361,6 +2355,8 @@ void NCMesh::InitRootState(int root_count)
    root_state.SetSize(root_count);
    root_state = 0;
 
+   if (elements.Size() == 0) { return; }
+
    char* node_order;
    int nch;
 
@@ -2610,11 +2606,10 @@ void NCMesh::OnMeshUpdated(Mesh *mesh)
    {
       const int *ev = edge_vertex->GetRow(i);
       Node* node = nodes.Find(vertex_nodeId[ev[0]], vertex_nodeId[ev[1]]);
-
       MFEM_ASSERT(node && node->HasEdge(),
                   "edge (" << ev[0] << "," << ev[1] << ") not found, "
-                  "node = " << node);
-
+                  "node = " << node << " node->HasEdge() "
+                  << (node != nullptr ? node->HasEdge() : false));
       node->edge_index = i;
    }
 
@@ -2709,7 +2704,6 @@ void NCMesh::OnMeshUpdated(Mesh *mesh)
          if (face.index < 0)
          {
             face.index = NFaces + (nghosts++);
-
             // store the face geometry
             static const Geometry::Type types[5] =
             {
@@ -2793,10 +2787,186 @@ bool NCMesh::TriFaceSplit(int v1, int v2, int v3, int mid[3]) const
    if (mid) { mid[0] = e1, mid[1] = e2, mid[2] = e3; }
 
    // This is necessary but not sufficient to determine if a face has been
-   // split.
+   // split. All edges might have been split due to edge attached faces being
+   // refined. Need to check for existence of face made up of midpoints.
    return true;
 }
 
+bool contains_node(const std::array<int, 4> &nodes, int n)
+{
+   return std::find(nodes.begin(), nodes.end(), n) != nodes.end();
+};
+
+int NCMesh::ParentFaceNodes(std::array<int, 4> &face_nodes) const
+{
+   const bool is_tri = face_nodes[3] == -1;
+   const bool is_segment = (face_nodes[0] == face_nodes[1] &&
+                            face_nodes[2] == face_nodes[3]);
+   const bool is_quad = *std::min_element(face_nodes.begin(),
+                                          face_nodes.end()) >= 0;
+
+   MFEM_ASSERT((is_tri && !is_segment && !is_quad)
+               || (!is_tri && is_segment && !is_quad) || (!is_tri && !is_segment &&
+                                                          is_quad), "Inconsistent node geometry");
+
+   bool all_nodes_root = true;
+   for (auto x : face_nodes)
+   {
+      all_nodes_root = all_nodes_root && (x < 0 || (nodes[x].p1 == nodes[x].p2));
+   }
+   // This face is a root face -> nothing to do.
+   if (all_nodes_root) { return -1; }
+
+   int child = -1; // The index into parent.child that this face corresponds to.
+   auto parent_nodes = face_nodes;
+   if (is_quad)
+   {
+      // Logic for coarsening anisotropic faces is more complex, needs
+      // identification and handling of multiple "crux" points. Will require
+      // inspection of edge nodes.
+      MFEM_VERIFY(Iso,
+                  "ParentFaceNodes does not support anisotropic refinement yet!");
+
+      // Finds the first node whose parents aren't in the face_nodes. This is
+      // also the index of the child location in the parent face. Treated
+      // separately as ultimately multiple crux will need to be handled for
+      // anisotropic faces.
+      const auto crux = [&]()
+      {
+         for (int i = 0; i < static_cast<int>(face_nodes.size()); i++)
+         {
+            if ((!contains_node(face_nodes, nodes[face_nodes[i]].p1)
+                 && !contains_node(face_nodes, nodes[face_nodes[i]].p2))
+                || (nodes[face_nodes[i]].p1 == nodes[face_nodes[i]].p2) /* top level node */)
+            {
+               return i;
+            }
+         }
+         return -1;
+      }();
+      MFEM_ASSERT(crux != -1, "A root face should have been returned early");
+
+      // Loop over nodes, starting from diagonal to child, wrapping and skipping
+      // child. This will visit the node opposite child twice, thereby
+      // coarsening to the diagonally opposite. NOTE: This assumes that the
+      // nodes for a square are numbered (0 -> 1 -> 2 -> 3 -> 0).
+      for (int i = 0; i < static_cast<int>(face_nodes.size()) + 1; i++)
+      {
+         int ind = (crux + i + 2) %
+                   4; // Start and end with coarsening of the diagonally opposite
+         if (ind == crux) { continue; }
+         auto &x = parent_nodes[ind];
+
+         // Check against parent_nodes rather than face_nodes so on second lap
+         // the node opposite crux will coarsen again to the diagonally across
+         // in the parent face. A top level node has p1 == p2, thus these
+         // modifications do nothing.
+         if (contains_node(parent_nodes, nodes[x].p1))
+         {
+            MFEM_ASSERT(nodes[x].p2 == nodes[x].p1 ||
+                        !contains_node(parent_nodes, nodes[x].p2), "!");
+            x = nodes[x].p2;
+         }
+         else if (contains_node(parent_nodes, nodes[x].p2))
+         {
+            MFEM_ASSERT(nodes[x].p2 == nodes[x].p1 ||
+                        !contains_node(parent_nodes, nodes[x].p1), "!");
+            x = nodes[x].p1;
+         }
+         else { /* do nothing */ }
+      }
+   }
+   else if (is_tri)
+   {
+      for (int i = 0; i < 3; i++)
+      {
+         auto x = face_nodes[i];
+         if (x == -1) { continue; }
+         if (contains_node(face_nodes, nodes[x].p1))
+         {
+            MFEM_ASSERT(nodes[x].p2 == nodes[x].p1 ||
+                        !contains_node(face_nodes, nodes[x].p2), "!");
+            parent_nodes[i] = nodes[x].p2;
+         }
+         else if (contains_node(face_nodes, nodes[x].p2))
+         {
+            MFEM_ASSERT(nodes[x].p2 == nodes[x].p1 ||
+                        !contains_node(face_nodes, nodes[x].p1), "!");
+            parent_nodes[i] = nodes[x].p1;
+         }
+         else { /* do nothing */ }
+      }
+
+      if (std::equal(face_nodes.begin(), face_nodes.end(), parent_nodes.begin()))
+      {
+         // Having excluded root faces, this must be an interior face. We need
+         // to handle the special case of the interior face of the parent face.
+         std::array<std::array<int, 2>, 6> parent_pairs;
+         for (std::size_t i = 0; i < face_nodes.size() - 1; i++)
+         {
+            parent_pairs[i][0] = nodes[face_nodes[i]].p1;
+            parent_pairs[i][1] = nodes[face_nodes[i]].p2;
+         }
+         // Each node gets mapped to the common node from its parents and the
+         // predecessor node's parents.
+         for (int i = 0; i < 3; i++)
+         {
+            // Parenting convention here assumes parent face has the SAME
+            // orientation as the original. This is true on exterior boundaries,
+            // but for an interior boundary the master face will have an
+            // opposing orientation. TODO: Possibly fix for interior boundaries.
+            const auto &prev = parent_pairs[(i - 1 + 3) % 3]; // (0 -> 2, 1 -> 0, 2 -> 1)
+            const auto &next = parent_pairs[(i + 1 + 3) % 3]; // (0 -> 1, 1 -> 2, 2 -> 0)
+            for (auto x : next)
+            {
+               if (std::find(prev.begin(), prev.end(), x) != prev.end()) { parent_nodes[i] = x; }
+            }
+         }
+         child = 3; // The interior face is the final child.
+      }
+   }
+   else if (is_segment)
+   {
+      // Given this isn't a root face, one node must be the parent of the other.
+      if (face_nodes[0] == nodes[face_nodes[1]].p1)
+      {
+         face_nodes[1] = nodes[face_nodes[1]].p2;
+      }
+      else if (face_nodes[0] == nodes[face_nodes[1]].p2)
+      {
+         face_nodes[1] = nodes[face_nodes[1]].p1;
+      }
+      else if (face_nodes[1] == nodes[face_nodes[0]].p1)
+      {
+         face_nodes[0] = nodes[face_nodes[0]].p2;
+      }
+      else if (face_nodes[1] == nodes[face_nodes[0]].p2)
+      {
+         face_nodes[0] = nodes[face_nodes[0]].p1;
+      }
+      else
+      {
+         MFEM_ABORT("Internal logic error!");
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Unrecognized face geometry!");
+   }
+   for (int i = 0; i < 4 && face_nodes[i] >= 0; i++)
+   {
+      if (face_nodes[i] == parent_nodes[i])
+      {
+         MFEM_ASSERT(child == -1,
+                     "This face cannot be more than one child of the parent face!");
+         child = i;
+      }
+   }
+   MFEM_ASSERT(child != -1, "Root elements must have exited early!");
+   std::swap(face_nodes, parent_nodes);
+   return child;
+}
+
 int NCMesh::find_node(const Element &el, int node)
 {
    for (int i = 0; i < MaxElemNodes; i++)
@@ -3556,7 +3726,8 @@ NCMesh::NCList::BuildIndex() const
       int max_master_index = max_master != nullptr ? max_master->index : -1;
       int max_slave_index = max_slave != nullptr ? max_slave->index : -1;
 
-      inv_index.reserve(std::max({max_conforming_index, max_master_index, max_slave_index}));
+      inv_index.reserve(max(max_conforming_index, max_master_index, max_slave_index,
+                            0));
       for (int i = 0; i < conforming.Size(); i++)
       {
          inv_index.emplace(conforming[i].index, std::make_pair(MeshIdType::CONFORMING,
@@ -3571,8 +3742,6 @@ NCMesh::NCList::BuildIndex() const
          inv_index.emplace(slaves[i].index, std::make_pair(MeshIdType::SLAVE, i));
       }
    }
-   MFEM_ASSERT(inv_index.size() > 0,
-               "Empty inverse index, member lists must be populated before BuildIndex is called!");
 }
 
 //// Neighbors /////////////////////////////////////////////////////////////////
@@ -5260,12 +5429,21 @@ void NCMesh::GetElementFacesAttributes(int leaf_elem,
       face_attribs[i] = face->attribute;
    }
 }
-
 void NCMesh::FindFaceNodes(int face, int node[4]) const
+{
+   auto tmp = FindFaceNodes(face);
+   std::copy(tmp.begin(), tmp.end(), node);
+}
+
+std::array<int, 4> NCMesh::FindFaceNodes(int face) const
+{
+   return FindFaceNodes(faces[face]);
+}
+
+std::array<int, 4> NCMesh::FindFaceNodes(const Face &fa) const
 {
    // Obtain face nodes from one of its elements (note that face->p1, p2, p3
    // cannot be used directly since they are not in order and p4 is missing).
-   const Face &fa = faces[face];
    int elem = fa.elem[0];
    if (elem < 0) { elem = fa.elem[1]; }
    MFEM_ASSERT(elem >= 0, "Face has no elements?");
@@ -5277,10 +5455,12 @@ void NCMesh::FindFaceNodes(int face, int node[4]) const
                            find_node(el, fa.p3));
 
    const int* fv = GI[el.Geom()].faces[f];
+   std::array<int, 4> node;
    for (int i = 0; i < 4; i++)
    {
       node[i] = el.node[fv[i]];
    }
+   return node;
 }
 
 void NCMesh::GetBoundaryClosure(const Array<int> &bdr_attr_is_ess,
@@ -5294,13 +5474,11 @@ void NCMesh::GetBoundaryClosure(const Array<int> &bdr_attr_is_ess,
    if (Dim == 3)
    {
       GetFaceList(); // make sure 'boundary_faces' is up to date
-
       for (int f : boundary_faces)
       {
          if (bdr_attr_is_ess[faces[f].attribute - 1])
          {
-            int node[4];
-            FindFaceNodes(f, node);
+            auto node = FindFaceNodes(f);
             int nfv = (node[3] < 0) ? 3 : 4;
 
             for (int j = 0; j < nfv; j++)
@@ -5334,6 +5512,7 @@ void NCMesh::GetBoundaryClosure(const Array<int> &bdr_attr_is_ess,
    }
    else if (Dim == 2)
    {
+      GetFaceList();
       GetEdgeList(); // make sure 'boundary_faces' is up to date
 
       for (int f : boundary_faces)
@@ -5554,9 +5733,7 @@ void NCMesh::LimitNCLevel(int max_nc_level)
    {
       Array<Refinement> refinements;
       GetLimitRefinements(refinements, max_nc_level);
-
       if (!refinements.Size()) { break; }
-
       Refine(refinements);
    }
 }
@@ -5847,12 +6024,15 @@ void NCMesh::InitRootElements()
 
    // count the root elements
    int nroots = 0;
-   while (nroots < elements.Size() &&
-          elements[nroots].parent == -1)
-   {
-      nroots++;
-   }
-   MFEM_VERIFY(nroots, "invalid mesh file: no root elements found.");
+   for (const auto &e : elements)
+      if (e.parent == -1)
+      {
+         ++nroots;
+      }
+   MFEM_VERIFY(nroots > 0 ||
+               elements.Size() == 0,
+               "invalid mesh file: no root elements in non-empty mesh found.");
+
 
    // check that only the first 'nroot' elements are roots (have no parent)
    for (int i = nroots; i < elements.Size(); i++)
@@ -5892,6 +6072,9 @@ NCMesh::NCMesh(std::istream &input, int version, int &curved, int &is_nc)
    std::string ident;
    int count;
 
+   // Skip the version string
+   skip_comment_lines(input, 'M');
+
    // load dimension
    skip_comment_lines(input, '#');
    input >> ident;
@@ -6018,9 +6201,10 @@ NCMesh::NCMesh(std::istream &input, int version, int &curved, int &is_nc)
    {
       LoadCoordinates(input);
 
-      MFEM_VERIFY(coordinates.Size()/3 >= CountTopLevelNodes(),
+      MFEM_VERIFY(coordinates.Size() >= 3*CountTopLevelNodes(),
                   "Invalid mesh file: not all top-level nodes are covered by "
-                  "the 'coordinates' section of the mesh file.");
+                  "the 'coordinates' section of the mesh file: " << coordinates.Size() << ' ' <<
+                  3*CountTopLevelNodes());
       curved = 0;
    }
    else if (ident == "nodes")
@@ -6082,7 +6266,7 @@ void NCMesh::LoadCoarseElements(std::istream &input)
       int ref_type;
       input >> ref_type;
 
-      int elem = AddElement(Element(Geometry::INVALID, 0));
+      int elem = AddElement(Geometry::INVALID, 0);
       Element &el = elements[elem];
       el.ref_type = ref_type;
 
@@ -6169,7 +6353,7 @@ void NCMesh::LoadLegacyFormat(std::istream &input, int &curved, int &is_nc)
       CheckSupportedGeom(type);
       GI[geom].InitGeom(type);
 
-      int eid = AddElement(Element(type, attr));
+      int eid = AddElement(type, attr);
       MFEM_ASSERT(eid == i, "");
 
       Element &el = elements[eid];

mesh/ncmesh.hpp

@@ -29,10 +29,10 @@
 namespace mfem
 {
 
-/** Represents the index of an element to refine, plus a refinement type.
-    The refinement type is needed for anisotropic refinement of quads and hexes.
-    Bits 0,1 and 2 of 'ref_type' specify whether the element should be split
-    in the X, Y and Z directions, respectively (Z is ignored for quads). */
+/** Represents the index of an element to refine, plus a refinement type. The
+    refinement type is needed for anisotropic refinement of quads and hexes.
+    Bits 0,1 and 2 of 'ref_type' specify whether the element should be split in
+    the X, Y and Z directions, respectively (Z is ignored for quads). */
 struct Refinement
 {
    enum : char { X = 1, Y = 2, Z = 4, XY = 3, XZ = 5, YZ = 6, XYZ = 7 };
@@ -45,7 +45,6 @@ struct Refinement
       : index(index), ref_type(type) {}
 };
 
-
 /// Defines the position of a fine element within a coarse element.
 struct Embedding
 {
@@ -54,7 +53,8 @@ struct Embedding
 
    /** The (geom, matrix) pair determines the sub-element transformation for the
        fine element: CoarseFineTransformations::point_matrices[geom](matrix) is
-       the point matrix of the region within the coarse element reference domain.*/
+       the point matrix of the region within the coarse element reference
+       domain.*/
    unsigned geom : 4;
    unsigned matrix : 27;
 
@@ -66,7 +66,6 @@ struct Embedding
       : parent(elem), geom(geom), matrix(matrix), ghost(ghost) {}
 };
 
-
 /// Defines the coarse-fine transformations of all fine elements.
 struct CoarseFineTransformations
 {
@@ -96,24 +95,23 @@ void Swap(CoarseFineTransformations &a, CoarseFineTransformations &b);
 
 struct MatrixMap; // for internal use
 
-
-/** \brief A class for non-conforming AMR. The class is not used directly
- *  by the user, rather it is an extension of the Mesh class.
+/** \brief A class for non-conforming AMR. The class is not used directly by the
+ *  user, rather it is an extension of the Mesh class.
  *
  *  In general, the class is used by MFEM as follows:
  *
- *  1. NCMesh is constructed from elements of an existing Mesh. The elements
- *     are copied and become roots of the refinement hierarchy.
+ *  1. NCMesh is constructed from elements of an existing Mesh. The elements are
+ *     copied and become roots of the refinement hierarchy.
  *
  *  2. Some elements are refined with the Refine() method. Both isotropic and
  *     anisotropic refinements of quads/hexes are supported.
  *
- *  3. A new Mesh is created from NCMesh containing the leaf elements.
- *     This new Mesh may have non-conforming (hanging) edges and faces and
- *     is the one seen by the user.
+ *  3. A new Mesh is created from NCMesh containing the leaf elements. This new
+ *     Mesh may have non-conforming (hanging) edges and faces and is the one
+ *     seen by the user.
  *
- *  4. FiniteElementSpace asks NCMesh for a list of conforming, master and
- *     slave edges/faces and creates the conforming interpolation matrix P.
+ *  4. FiniteElementSpace asks NCMesh for a list of conforming, master and slave
+ *     edges/faces and creates the conforming interpolation matrix P.
  *
  *  5. A continuous/conforming solution is obtained by solving P'*A*P x = P'*b.
  *
@@ -121,8 +119,10 @@ struct MatrixMap; // for internal use
  */
 class NCMesh
 {
+protected:
+   NCMesh() = default;
 public:
-   //// Initialize with elements from an existing 'mesh'.
+   //// Initialize with elements from an existing Mesh.
    explicit NCMesh(const Mesh *mesh);
 
    /** Load from a stream. The id header is assumed to have been read already
@@ -155,8 +155,8 @@ public:
    virtual int GetNGhostElements() const { return 0; }
 
    /** Perform the given batch of refinements. Please note that in the presence
-       of anisotropic splits additional refinements may be necessary to keep
-       the mesh consistent. However, the function always performs at least the
+       of anisotropic splits additional refinements may be necessary to keep the
+       mesh consistent. However, the function always performs at least the
        requested refinements. */
    virtual void Refine(const Array<Refinement> &refinements);
 
@@ -172,14 +172,16 @@ public:
    const Table &GetDerefinementTable();
 
    /** Check derefinements returned by GetDerefinementTable and mark those that
-       can be done safely so that the maximum NC level condition is not violated.
-       On return, level_ok.Size() == deref_table.Size() and contains 0/1s. */
+       can be done safely so that the maximum NC level condition is not
+       violated. On return, level_ok.Size() == deref_table.Size() and contains
+       0/1s. */
    virtual void CheckDerefinementNCLevel(const Table &deref_table,
                                          Array<int> &level_ok, int max_nc_level);
 
-   /** Perform a subset of the possible derefinements (see GetDerefinementTable).
-       Note that if anisotropic refinements are present in the mesh, some of the
-       derefinements may have to be skipped to preserve mesh consistency. */
+   /** Perform a subset of the possible derefinements (see
+       GetDerefinementTable). Note that if anisotropic refinements are present
+       in the mesh, some of the derefinements may have to be skipped to preserve
+       mesh consistency. */
    virtual void Derefine(const Array<int> &derefs);
 
    // master/slave lists
@@ -340,9 +342,9 @@ public:
    const CoarseFineTransformations& GetRefinementTransforms() const;
 
    /** After derefinement, calculate the relations of previous fine elements
-       (some of which may no longer exist) to the current leaf elements.
-       Unlike for refinement, Derefine() may only be called once before this
-       function so there is no MarkFineLevel(). */
+       (some of which may no longer exist) to the current leaf elements. Unlike
+       for refinement, Derefine() may only be called once before this function
+       so there is no MarkFineLevel(). */
    const CoarseFineTransformations& GetDerefinementTransforms() const;
 
    /// Free all internal data created by the above three functions.
@@ -359,8 +361,8 @@ public:
    static void GridSfcOrdering2D(int width, int height,
                                  Array<int> &coords);
 
-   /** Return a space filling curve for a 3D rectangular grid of elements.
-       The Hilbert-curve-like algorithm works well for even dimensions. For odd
+   /** Return a space filling curve for a 3D rectangular grid of elements. The
+       Hilbert-curve-like algorithm works well for even dimensions. For odd
        width/height/depth it tends to produce some diagonal (edge-neighbor)
        steps. Even dimensions are recommended. */
    static void GridSfcOrdering3D(int width, int height, int depth,
@@ -428,17 +430,20 @@ public:
    /// Return the number of root elements.
    int GetNumRootElements() { return root_state.Size(); }
 
-   /// Return the distance of leaf 'i' from the root.
+   /// Return the distance of leaf @a i from the root.
    int GetElementDepth(int i) const;
 
    /** Return the size reduction compared to the root element (ignoring local
        stretching and curvature). */
    int GetElementSizeReduction(int i) const;
 
-   /// Return the faces and face attributes of leaf element 'i'.
+   /// Return the faces and face attributes of leaf element @a i.
    void GetElementFacesAttributes(int i, Array<int> &faces,
                                   Array<int> &fattr) const;
 
+   /// Set the attribute of leaf element @a i, which is a Mesh element index.
+   void SetAttribute(int i, int attr)
+   { elements[leaf_elements[i]].attribute = attr; }
 
    /** I/O: Print the mesh in "MFEM NC mesh v1.0" format. If @a comments is
        non-empty, it will be printed after the first line of the file, and each
@@ -459,8 +464,26 @@ public:
 
    int PrintMemoryDetail() const;
 
-   typedef std::int64_t RefCoord;
+   using RefCoord = std::int64_t;
 
+   static constexpr int MaxElemNodes =
+      8;       ///< Number of nodes an element can have
+   static constexpr int MaxElemEdges =
+      12;      ///< Number of edges an element can have
+   static constexpr int MaxElemFaces =
+      6;       ///< Number of faces an element can have
+   static constexpr int MaxElemChildren =
+      10;      ///< Number of children an element can have
+   static constexpr int MaxFaceNodes =
+      4;      ///< Number of faces an element can have
+
+   /**
+    * @brief Given a node index, return the vertex index associated
+    *
+    * @param node
+    * @return int
+    */
+   int GetNodeVertex(int node) { return nodes[node].vert_index; }
 
 protected: // non-public interface for the Mesh class
 
@@ -473,8 +496,8 @@ protected: // non-public interface for the Mesh class
        Face::index) after a new mesh was created from us. */
    void OnMeshUpdated(Mesh *mesh);
 
-   /** Delete top-level vertex coordinates if the Mesh became curved, e.g.,
-       by calling Mesh::SetCurvature or otherwise setting the Nodes. */
+   /** Delete top-level vertex coordinates if the Mesh became curved, e.g., by
+       calling Mesh::SetCurvature or otherwise setting the Nodes. */
    void MakeTopologyOnly() { coordinates.DeleteAll(); }
 
 protected: // implementation
@@ -485,23 +508,15 @@ protected: // implementation
    int Geoms; ///< bit mask of element geometries present, see InitGeomFlags()
    bool Legacy; ///< true if the mesh was loaded from the legacy v1.1 format
 
-   static const int MaxElemNodes =
-      8;       ///< Number of nodes of an element can have
-   static const int MaxElemEdges =
-      12;      ///< Number of edges of an element can have
-   static const int MaxElemFaces =
-      6;       ///< Number of faces of an element can have
-   static const int MaxElemChildren =
-      10;      ///< Number of children of an element can have
 
    /** A Node can hold a vertex, an edge, or both. Elements directly point to
-       their corner nodes, but edge nodes also exist and can be accessed using
-       a hash-table given their two end-point node IDs. All nodes can be
-       accessed in this way, with the exception of top-level vertex nodes.
-       When an element is being refined, the mid-edge nodes are readily
-       available with this mechanism. The new elements "sign in" to the nodes
-       by increasing the reference counts of their vertices and edges. The
-       parent element "signs off" its nodes by decrementing the ref counts. */
+       their corner nodes, but edge nodes also exist and can be accessed using a
+       hash-table given their two end-point node IDs. All nodes can be accessed
+       in this way, with the exception of top-level vertex nodes. When an
+       element is being refined, the mid-edge nodes are readily available with
+       this mechanism. The new elements "sign in" to the nodes by increasing the
+       reference counts of their vertices and edges. The parent element "signs
+       off" its nodes by decrementing the ref counts. */
    struct Node : public Hashed2
    {
       char vert_refc, edge_refc;
@@ -519,9 +534,9 @@ protected: // implementation
    };
 
    /** Similarly to nodes, faces can be accessed by hashing their four vertex
-       node IDs. A face knows about the one or two elements that are using it.
-       A face that is not on the boundary and only has one element referencing
-       it is either a master or a slave face. */
+       node IDs. A face knows about the one or two elements that are using it. A
+       face that is not on the boundary and only has one element referencing it
+       is either a master or a slave face. */
    struct Face : public Hashed4
    {
       int attribute; ///< boundary element attribute, -1 if internal face
@@ -539,11 +554,12 @@ protected: // implementation
 
       /// Return one of elem[0] or elem[1] and make sure the other is -1.
       int GetSingleElement() const;
+      int GetAttribute() const { return attribute; }
    };
 
-   /** This is an element in the refinement hierarchy. Each element has
-       either been refined and points to its children, or is a leaf and points
-       to its vertex nodes. */
+   /** This is an element in the refinement hierarchy. Each element has either
+       been refined and points to its children, or is a leaf and points to its
+       vertex nodes. */
    struct Element
    {
       char geom;     ///< Geometry::Type of the element (char for storage only)
@@ -559,46 +575,114 @@ protected: // implementation
          int child[MaxElemChildren]; ///< 2-10 children (if ref_type != 0)
       };
       int parent; ///< parent element, -1 if this is a root element, -2 if free'd
-
       Element(Geometry::Type geom, int attr);
 
       Geometry::Type Geom() const { return Geometry::Type(geom); }
       bool IsLeaf() const { return !ref_type && (parent != -2); }
+      int GetAttribute() const { return attribute; }
    };
 
 
    // primary data
-
    HashTable<Node> nodes; // associative container holding all Nodes
    HashTable<Face> faces; // associative container holding all Faces
-
    BlockArray<Element> elements; // storage for all Elements
    Array<int> free_element_ids;  // unused element ids - indices into 'elements'
+public:
+   /**
+    * @brief The number of Nodes.
+    *
+    * @return int
+    */
+   int GetNumNodes() const { return nodes.Size(); }
+   /**
+    * @brief Access a Node
+    *
+    * @param i Index of the node
+    * @return const Node&
+    */
+   const Node& GetNode(int i) const {return nodes[i]; }
+   /**
+    * @brief The number of faces
+    *
+    * @return int
+    */
+   int GetNumFaces() const { return faces.Size(); }
+   /**
+    * @brief Access a Face
+    *
+    * @param i Index of the face
+    * @return const Face&
+    */
+   const Face& GetFace(int i) const {return faces[i]; }
+   /**
+    * @brief The number of elements
+    *
+    * @return int
+    */
+   int GetNumElements() const { return elements.Size(); }
+   /**
+    * @brief Access an Element
+    *
+    * @param i Index of the element
+    * @return const Element&
+    */
+   const Element& GetElement(int i) const { return elements[i]; }
+
+   /**
+    * @brief Given a set of nodes defining a face, traverse the nodes structure
+    * to find the nodes that make up the parent face and replace the input nodes
+    * with the parent nodes. Additionally return the child index that the child
+    * face would be, relative to the discovered parent face.
+    * @details This method is concerned with the construction of an NCMesh
+    * structure for a d-1 manifold of an existing NCMesh. It forms a key element
+    * in a leaf -> root traversal of the parent ncmesh elements structure.
+    *
+    * @param[out] nodes The collection of nodes whose parent we are searching
+    * for
+    * @return int The child index corresponding to placing the face for the
+    * original nodes within the face defined by the returned parent nodes. If
+    * child index is -1, then the face is made up of root nodes, and nodes is
+    * unchanged.
+    */
+   int ParentFaceNodes(std::array<int, 4> &nodes) const;
+
+   /**
+    * @brief Method for finding the nodes associated to a @a face
+    * @return Nodes making up the face
+    */
+   std::array<int, 4> FindFaceNodes(int face) const;
+   std::array<int, 4> FindFaceNodes(const Face &fa) const;
+   /**
+    * @brief Backwards compatible method for finding the @a node associated to a
+    * @a face
+    */
+   MFEM_DEPRECATED void FindFaceNodes(int face, int node[4]) const;
+protected:
 
    /** Initial traversal state (~ element orientation) for each root element
-       NOTE: M = root_state.Size() is the number of root elements.
-       NOTE: the first M items of 'elements' is the coarse mesh. */
+       NOTE: M = root_state.Size() is the number of root elements. NOTE: the
+       first M items of 'elements' is the coarse mesh. */
    Array<int> root_state;
 
-   /** Coordinates of top-level vertices (organized as triples). If empty,
-       the Mesh is curved (Nodes != NULL) and NCMesh is topology-only. */
+   /** Coordinates of top-level vertices (organized as triples). If empty, the
+       Mesh is curved (Nodes != NULL) and NCMesh is topology-only. */
    Array<real_t> coordinates;
 
-
    // secondary data
-
    /** Apart from the primary data structure, which is the element/node/face
-       hierarchy, there is secondary data that is derived from the primary
-       data and needs to be updated when the primary data changes. Update()
-       takes care of that and needs to be called after each refinement and
+       hierarchy, there is secondary data that is derived from the primary data
+       and needs to be updated when the primary data changes. Update() takes
+       care of that and needs to be called after each refinement and
        derefinement. */
    virtual void Update();
 
    // set by UpdateLeafElements, UpdateVertices and OnMeshUpdated
    int NElements, NVertices, NEdges, NFaces;
 
-   // NOTE: the serial code understands the bare minimum about ghost elements and
-   // other ghost entities in order to be able to load parallel partial meshes
+   // NOTE: the serial code understands the bare minimum about ghost elements
+   // and other ghost entities in order to be able to load parallel partial
+   // meshes
    int NGhostElements, NGhostVertices, NGhostEdges, NGhostFaces;
 
    Array<int> leaf_elements; ///< finest elements, in Mesh ordering (+ ghosts)
@@ -623,19 +707,19 @@ protected: // implementation
        We must be careful to:
        1. Stay compatible with the conforming code, which expects top-level
           (original) vertices to be indexed first, otherwise GridFunctions
-          defined on a conforming mesh would no longer be valid when the
-          mesh is converted to an NC mesh.
+          defined on a conforming mesh would no longer be valid when the mesh is
+          converted to an NC mesh.
 
-       2. Make sure serial NCMesh is compatible with the parallel ParNCMesh,
-          so it is possible to read parallel partial solutions in serial code
+       2. Make sure serial NCMesh is compatible with the parallel ParNCMesh, so
+          it is possible to read parallel partial solutions in serial code
           (e.g., serial GLVis). This means handling ghost elements, if present.
 
-       3. Assign vertices in a globally consistent order for parallel meshes:
-          if two vertices i,j are shared by two ranks r1,r2, and i<j on r1,
-          then i<j on r2 as well. This is true for top-level vertices but also
-          for the remaining shared vertices thanks to the globally consistent
-          SFC ordering of the leaf elements. This property reduces communication
-          and simplifies ParNCMesh. */
+       3. Assign vertices in a globally consistent order for parallel meshes: if
+          two vertices i,j are shared by two ranks r1,r2, and i<j on r1, then
+          i<j on r2 as well. This is true for top-level vertices but also for
+          the remaining shared vertices thanks to the globally consistent SFC
+          ordering of the leaf elements. This property reduces communication and
+          simplifies ParNCMesh. */
    void UpdateVertices(); ///< update Vertex::index and vertex_nodeId
 
    /** Collect the leaf elements in leaf_elements, and the ghost elements in
@@ -646,8 +730,8 @@ protected: // implementation
                             int &counter);
 
    /** Try to find a space-filling curve friendly orientation of the root
-       elements: set 'root_state' based on the ordering of coarse elements.
-       Note that the coarse mesh itself must be ordered as an SFC by e.g.
+       elements: set 'root_state' based on the ordering of coarse elements. Note
+       that the coarse mesh itself must be ordered as an SFC by e.g.
        Mesh::GetGeckoElementOrdering. */
    void InitRootState(int root_count);
 
@@ -667,7 +751,6 @@ protected: // implementation
    /// Return true if the Element @a el is a ghost element.
    bool IsGhost(const Element &el) const { return el.rank != MyRank; }
 
-
    // refinement/derefinement
 
    Array<Refinement> ref_stack; ///< stack of scheduled refinements (temporary)
@@ -676,8 +759,8 @@ protected: // implementation
 
    Table derefinements; ///< possible derefinements, see GetDerefinementTable
 
-   /** Refine the element @a elem with the refinement @a ref_type
-       (c.f. Refinement::enum) */
+   /** Refine the element @a elem with the refinement @a ref_type (c.f.
+       Refinement::enum) */
    void RefineElement(int elem, char ref_type);
 
    /// Derefine the element @a elem, does nothing on leaf elements.
@@ -695,6 +778,7 @@ protected: // implementation
       }
       return elements.Append(el);
    }
+   int AddElement(Geometry::Type geom, int attr) { return AddElement(Element(geom,attr)); }
 
    // Free the element with index @a id.
    void FreeElement(int id)
@@ -826,6 +910,11 @@ protected: // implementation
 
    int GetMidFaceNode(int en1, int en2, int en3, int en4);
 
+   /**
+    * @brief Add references to all nodes, edges and faces of the element
+    *
+    * @param elem index into elements
+    */
    void ReferenceElement(int elem);
    void UnreferenceElement(int elem, Array<int> &elemFaces);
 
@@ -882,28 +971,28 @@ protected: // implementation
 
    // neighbors / element_vertex table
 
-   /** Return all vertex-, edge- and face-neighbors of a set of elements.
-       The neighbors are returned as a list (neighbors != NULL), as a set
+   /** Return all vertex-, edge- and face-neighbors of a set of elements. The
+       neighbors are returned as a list (neighbors != NULL), as a set
        (neighbor_set != NULL), or both. The sizes of the set arrays must match
-       that of leaf_elements. The function is intended to be used for large
-       sets of elements and its complexity is linear in the number of leaf
-       elements in the mesh. */
+       that of leaf_elements. The function is intended to be used for large sets
+       of elements and its complexity is linear in the number of leaf elements
+       in the mesh. */
    void FindSetNeighbors(const Array<char> &elem_set,
                          Array<int> *neighbors, /* append */
                          Array<char> *neighbor_set = NULL);
 
-   /** Return all vertex-, edge- and face-neighbors of a single element.
-       You can limit the number of elements being checked using 'search_set'.
-       The complexity of the function is linear in the size of the search set.*/
+   /** Return all vertex-, edge- and face-neighbors of a single element. You can
+       limit the number of elements being checked using 'search_set'. The
+       complexity of the function is linear in the size of the search set.*/
    void FindNeighbors(int elem,
                       Array<int> &neighbors, /* append */
                       const Array<int> *search_set = NULL);
 
-   /** Expand a set of elements by all vertex-, edge- and face-neighbors.
-       The output array 'expanded' will contain all items from 'elems'
-       (provided they are in 'search_set') plus their neighbors. The neighbor
-       search can be limited to the optional search set. The complexity is
-       linear in the sum of the sizes of 'elems' and 'search_set'. */
+   /** Expand a set of elements by all vertex-, edge- and face-neighbors. The
+       output array 'expanded' will contain all items from 'elems' (provided
+       they are in 'search_set') plus their neighbors. The neighbor search can
+       be limited to the optional search set. The complexity is linear in the
+       sum of the sizes of 'elems' and 'search_set'. */
    void NeighborExpand(const Array<int> &elems,
                        Array<int> &expanded,
                        const Array<int> *search_set = NULL);
@@ -981,18 +1070,17 @@ protected: // implementation
    /** @brief The PointMatrix stores the coordinates of the slave face using the
        master face coordinate as reference.
 
-       In 2D, the point matrix has the orientation of the parent
-       edge, so its columns need to be flipped when applying it, see
+       In 2D, the point matrix has the orientation of the parent edge, so its
+       columns need to be flipped when applying it, see
        ApplyLocalSlaveTransformation.
 
-       In 3D, the orientation part of Elem2Inf is encoded in the point
-       matrix.
+       In 3D, the orientation part of Elem2Inf is encoded in the point matrix.
 
-       The following transformation gives the relation between the
-       reference quad face coordinates (xi, eta) in [0,1]^2, and the fine quad
-       face coordinates (x, y):
-       x = a0*(1-xi)*(1-eta) + a1*xi*(1-eta) + a2*xi*eta + a3*(1-xi)*eta
-       y = b0*(1-xi)*(1-eta) + b1*xi*(1-eta) + b2*xi*eta + b3*(1-xi)*eta
+       The following transformation gives the relation between the reference
+       quad face coordinates (xi, eta) in [0,1]^2, and the fine quad face
+       coordinates (x, y):
+         x = a0*(1-xi)*(1-eta) + a1*xi*(1-eta) + a2*xi*eta + a3*(1-xi)*eta
+         y = b0*(1-xi)*(1-eta) + b1*xi*(1-eta) + b2*xi*eta + b3*(1-xi)*eta
    */
    struct PointMatrix
    {
@@ -1054,7 +1142,7 @@ protected: // implementation
    void GetPointMatrix(Geometry::Type geom, const char* ref_path,
                        DenseMatrix& matrix) const;
 
-   typedef std::map<std::string, int> RefPathMap;
+   using RefPathMap = std::map<std::string, int>;
 
    void TraverseRefinements(int elem, int coarse_index,
                             std::string &ref_path, RefPathMap &map) const;
@@ -1085,15 +1173,15 @@ protected: // implementation
 
    int GetEdgeMaster(int node) const;
 
-   void FindFaceNodes(int face, int node[4]) const;
-
    /**
     * @brief Return the number of splits of this edge that have occurred in the
-    * NCMesh. If zero, this means the segment is not the master of any other segments.
+    * NCMesh. If zero, this means the segment is not the master of any other
+    * segments.
     *
     * @param vn1 The first vertex making up the segment
     * @param vn2 The second vertex making up the segment
-    * @return int The depth of splits of this segment that are present in the mesh.
+    * @return int The depth of splits of this segment that are present in the
+    * mesh.
     */
    int EdgeSplitLevel(int vn1, int vn2) const;
    /**
@@ -1104,13 +1192,14 @@ protected: // implementation
     * @param vn1 The first vertex making up the triangle
     * @param vn2 The second vertex making up the triangle
     * @param vn3 The third vertex making up the triangle
-    * @return int The depth of splits of this triangle that are present in the mesh.
+    * @return int The depth of splits of this triangle that are present in the
+    * mesh.
     */
    int TriFaceSplitLevel(int vn1, int vn2, int vn3) const;
    /**
     * @brief Computes the number of horizontal and vertical splits of this quad
-    * that have occurred in the NCMesh. If zero, this means the quad is not
-    * the master of any other quad.
+    * that have occurred in the NCMesh. If zero, this means the quad is not the
+    * master of any other quad.
     *
     * @param vn1 The first vertex making up the quad
     * @param vn2 The second vertex making up the quad
@@ -1123,8 +1212,8 @@ protected: // implementation
                            int& h_level, int& v_level) const;
    /**
     * @brief Returns the total number of splits of this quad that have occurred
-    * in the NCMesh. If zero, this means the quad is not
-    * the master of any other quad.
+    * in the NCMesh. If zero, this means the quad is not the master of any other
+    * quad.
     * @details This is a convenience wrapper that sums the horizontal and
     * vertical levels from the full method.
     *
@@ -1141,6 +1230,17 @@ protected: // implementation
    void CountSplits(int elem, int splits[3]) const;
    void GetLimitRefinements(Array<Refinement> &refinements, int max_level);
 
+   // Checker helpers
+
+   static void CheckSupportedGeom(Geometry::Type geom)
+   {
+      MFEM_VERIFY(geom == Geometry::SEGMENT ||
+                  geom == Geometry::TRIANGLE || geom == Geometry::SQUARE ||
+                  geom == Geometry::CUBE || geom == Geometry::PRISM ||
+                  geom == Geometry::PYRAMID || geom == Geometry::TETRAHEDRON,
+                  "Element type " << geom << " is not supported by NCMesh.");
+   }
+
 
    // I/O
 
@@ -1149,8 +1249,8 @@ protected: // implementation
    /// Load the vertex parent hierarchy from a mesh file.
    void LoadVertexParents(std::istream &input);
 
-   /** Print the "boundary" section of the mesh file.
-       If out == NULL, only return the number of boundary elements. */
+   /** Print the "boundary" section of the mesh file. If out == NULL, only
+       return the number of boundary elements. */
    int PrintBoundary(std::ostream *out) const;
    /// Load the "boundary" section of the mesh file.
    void LoadBoundary(std::istream &input);
@@ -1185,6 +1285,7 @@ protected: // implementation
 
       bool initialized;
       GeomInfo() : initialized(false) {}
+      GeomInfo(Geometry::Type geom) : GeomInfo() { InitGeom(geom); }
       void InitGeom(Geometry::Type geom);
    };
 
@@ -1199,6 +1300,8 @@ public:
    friend class ParNCMesh; // for ParNCMesh::ElementSet
    friend struct MatrixMap;
    friend struct PointMatrixHash;
+   friend class NCSubMesh; // for faces, nodes
+   friend class ParNCSubMesh; // for faces, nodes
 };
 
 }

mesh/ncmesh_tables.hpp

@@ -9,14 +9,13 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#ifndef MFEM_NCMESH_TABLES
+#define MFEM_NCMESH_TABLES
+
 namespace mfem
 {
 
-namespace // make everything static
-{
-
-const int ref_type_num_children[8] = { 0, 2, 2, 4, 2, 4, 4, 8 };
-
+static constexpr int ref_type_num_children[8] = { 0, 2, 2, 4, 2, 4, 4, 8 };
 
 // derefinement tables
 // The first n numbers in each line are the refined elements that contain
@@ -24,14 +23,14 @@ const int ref_type_num_children[8] = { 0, 2, 2, 4, 2, 4, 4, 8 };
 // are the refined elements that contain the faces attributes of the parent
 // element.
 
-const int quad_deref_table[3][4 + 4] =
+static constexpr int quad_deref_table[3][4 + 4] =
 {
    { 0, 1, 1, 0, /**/ 1, 1, 0, 0 }, // 1 - X
    { 0, 0, 1, 1, /**/ 0, 0, 1, 1 }, // 2 - Y
    { 0, 1, 2, 3, /**/ 1, 1, 3, 3 }  // 3 - iso
 };
 
-const int hex_deref_table[7][8 + 6] =
+static constexpr int hex_deref_table[7][8 + 6] =
 {
    { 0, 1, 1, 0, 0, 1, 1, 0, /**/ 1, 1, 1, 0, 0, 0 }, // 1 - X
    { 0, 0, 1, 1, 0, 0, 1, 1, /**/ 0, 0, 0, 1, 1, 1 }, // 2 - Y
@@ -42,7 +41,7 @@ const int hex_deref_table[7][8 + 6] =
    { 0, 1, 2, 3, 4, 5, 6, 7, /**/ 1, 1, 1, 7, 7, 7 }  // 7 - iso
 };
 
-const int prism_deref_table[7][6 + 5] =
+static constexpr int prism_deref_table[7][6 + 5] =
 {
    {-1,-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 1
    {-1,-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 2
@@ -53,7 +52,7 @@ const int prism_deref_table[7][6 + 5] =
    { 0, 1, 2, 4, 5, 6, /**/  0, 5, 0, 5, 0 }  // 7 - iso
 };
 
-const int pyramid_deref_table[7][5 + 5] =
+static constexpr int pyramid_deref_table[7][5 + 5] =
 {
    {-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 1
    {-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 2
@@ -66,19 +65,19 @@ const int pyramid_deref_table[7][5 + 5] =
 
 // child ordering tables
 
-const char quad_hilbert_child_order[8][4] =
+static constexpr char quad_hilbert_child_order[8][4] =
 {
    {0,1,2,3}, {0,3,2,1}, {1,2,3,0}, {1,0,3,2},
    {2,3,0,1}, {2,1,0,3}, {3,0,1,2}, {3,2,1,0}
 };
 
-const char quad_hilbert_child_state[8][4] =
+static constexpr char quad_hilbert_child_state[8][4] =
 {
    {1,0,0,5}, {0,1,1,4}, {3,2,2,7}, {2,3,3,6},
    {5,4,4,1}, {4,5,5,0}, {7,6,6,3}, {6,7,7,2}
 };
 
-const char hex_hilbert_child_order[24][8] =
+static constexpr char hex_hilbert_child_order[24][8] =
 {
    {0,1,2,3,7,6,5,4}, {0,3,7,4,5,6,2,1}, {0,4,5,1,2,6,7,3},
    {1,0,3,2,6,7,4,5}, {1,2,6,5,4,7,3,0}, {1,5,4,0,3,7,6,2},
@@ -90,7 +89,7 @@ const char hex_hilbert_child_order[24][8] =
    {7,3,2,6,5,1,0,4}, {7,4,0,3,2,1,5,6}, {7,6,5,4,0,1,2,3}
 };
 
-const char hex_hilbert_child_state[24][8] =
+static constexpr char hex_hilbert_child_state[24][8] =
 {
    {1,2,2,7,7,21,21,17},     {2,0,0,22,22,16,16,8},    {0,1,1,15,15,6,6,23},
    {4,5,5,10,10,18,18,14},   {5,3,3,19,19,13,13,11},   {3,4,4,12,12,9,9,20},
@@ -104,27 +103,26 @@ const char hex_hilbert_child_state[24][8] =
 
 
 // child/parent reference domain transforms
-
-typedef NCMesh::RefCoord RefCoord;
+using RefCoord = NCMesh::RefCoord;
 
 // reference domain coordinates as fixed point numbers
-const RefCoord T_HALF = (1ll << 59);
-const RefCoord T_ONE = (1ll << 60);
-const RefCoord T_TWO = (1ll << 61);
+static constexpr RefCoord T_HALF = (1ll << 59);
+static constexpr RefCoord T_ONE = (1ll << 60);
+static constexpr RefCoord T_TWO = (1ll << 61);
 
 // (scaling factors have a different fixed point multiplier)
-const RefCoord S_HALF = 1;
-const RefCoord S_ONE = 2;
-const RefCoord S_TWO = 4;
+static constexpr RefCoord S_HALF = 1;
+static constexpr RefCoord S_ONE = 2;
+static constexpr RefCoord S_TWO = 4;
 
-const RefCoord tri_corners[3][3] =
+static constexpr RefCoord tri_corners[3][3] =
 {
    {    0,     0, 0},
    {T_ONE,     0, 0},
    {    0, T_ONE, 0}
 };
 
-const RefCoord quad_corners[4][3] =
+static constexpr RefCoord quad_corners[4][3] =
 {
    {    0,     0, 0},
    {T_ONE,     0, 0},
@@ -132,7 +130,7 @@ const RefCoord quad_corners[4][3] =
    {    0, T_ONE, 0}
 };
 
-const RefCoord hex_corners[8][3] =
+static constexpr RefCoord hex_corners[8][3] =
 {
    {    0,     0,     0},
    {T_ONE,     0,     0},
@@ -144,7 +142,7 @@ const RefCoord hex_corners[8][3] =
    {    0, T_ONE, T_ONE}
 };
 
-const RefCoord prism_corners[6][3] =
+static constexpr RefCoord prism_corners[6][3] =
 {
    {    0,     0,     0},
    {T_ONE,     0,     0},
@@ -154,7 +152,7 @@ const RefCoord prism_corners[6][3] =
    {    0, T_ONE, T_ONE}
 };
 
-const RefCoord pyramid_corners[5][3] =
+static constexpr RefCoord pyramid_corners[5][3] =
 {
    {    0,     0,     0},
    {T_ONE,     0,     0},
@@ -164,7 +162,7 @@ const RefCoord pyramid_corners[5][3] =
 };
 
 typedef RefCoord RefPoint[3];
-const RefPoint* geom_corners[8] =
+static const RefPoint* geom_corners[8] =
 {
    NULL, // point
    NULL, // segment
@@ -190,31 +188,31 @@ struct RefTrf
    }
 };
 
-const RefTrf quad_parent_rt1[2] =
+static constexpr RefTrf quad_parent_rt1[2] =
 {
    { {S_HALF, S_ONE, 0}, {     0, 0, 0} },
    { {S_HALF, S_ONE, 0}, {T_HALF, 0, 0} }
 };
 
-const RefTrf quad_child_rt1[2] =
+static constexpr RefTrf quad_child_rt1[2] =
 {
    { {S_TWO, S_ONE, 0}, {     0, 0, 0} },
    { {S_TWO, S_ONE, 0}, {-T_ONE, 0, 0} }
 };
 
-const RefTrf quad_parent_rt2[2] =
+static constexpr RefTrf quad_parent_rt2[2] =
 {
    { {S_ONE, S_HALF, 0}, {0,      0, 0} },
    { {S_ONE, S_HALF, 0}, {0, T_HALF, 0} }
 };
 
-const RefTrf quad_child_rt2[2] =
+static constexpr RefTrf quad_child_rt2[2] =
 {
    { {S_ONE, S_TWO, 0}, {0,      0, 0} },
    { {S_ONE, S_TWO, 0}, {0, -T_ONE, 0} }
 };
 
-const RefTrf quad_parent_rt3[4] =
+static constexpr RefTrf quad_parent_rt3[4] =
 {
    { {S_HALF, S_HALF, 0}, {     0,      0, 0} },
    { {S_HALF, S_HALF, 0}, {T_HALF,      0, 0} },
@@ -222,7 +220,7 @@ const RefTrf quad_parent_rt3[4] =
    { {S_HALF, S_HALF, 0}, {     0, T_HALF, 0} }
 };
 
-const RefTrf quad_child_rt3[4] =
+static constexpr RefTrf quad_child_rt3[4] =
 {
    { {S_TWO, S_TWO, 0}, {     0,      0, 0} },
    { {S_TWO, S_TWO, 0}, {-T_ONE,      0, 0} },
@@ -230,7 +228,7 @@ const RefTrf quad_child_rt3[4] =
    { {S_TWO, S_TWO, 0}, {     0, -T_ONE, 0} }
 };
 
-const RefTrf* quad_parent[4] =
+static const RefTrf* quad_parent[4] =
 {
    NULL,
    quad_parent_rt1,
@@ -238,7 +236,7 @@ const RefTrf* quad_parent[4] =
    quad_parent_rt3
 };
 
-const RefTrf* quad_child[4] =
+static const RefTrf* quad_child[4] =
 {
    NULL,
    quad_child_rt1,
@@ -246,31 +244,31 @@ const RefTrf* quad_child[4] =
    quad_child_rt3
 };
 
-const RefTrf hex_parent_rt1[2] =
+static constexpr RefTrf hex_parent_rt1[2] =
 {
    { {S_HALF, S_ONE, S_ONE}, {     0, 0, 0} },
    { {S_HALF, S_ONE, S_ONE}, {T_HALF, 0, 0} }
 };
 
-const RefTrf hex_child_rt1[2] =
+static constexpr RefTrf hex_child_rt1[2] =
 {
    { {S_TWO, S_ONE, S_ONE}, {     0, 0, 0} },
    { {S_TWO, S_ONE, S_ONE}, {-T_ONE, 0, 0} }
 };
 
-const RefTrf hex_parent_rt2[2] =
+static constexpr RefTrf hex_parent_rt2[2] =
 {
    { {S_ONE, S_HALF, S_ONE}, {0,      0, 0} },
    { {S_ONE, S_HALF, S_ONE}, {0, T_HALF, 0} }
 };
 
-const RefTrf hex_child_rt2[2] =
+static constexpr RefTrf hex_child_rt2[2] =
 {
    { {S_ONE, S_TWO, S_ONE}, {0,      0, 0} },
    { {S_ONE, S_TWO, S_ONE}, {0, -T_ONE, 0} }
 };
 
-const RefTrf hex_parent_rt3[4] =
+static constexpr RefTrf hex_parent_rt3[4] =
 {
    { {S_HALF, S_HALF, S_ONE}, {     0,      0, 0} },
    { {S_HALF, S_HALF, S_ONE}, {T_HALF,      0, 0} },
@@ -278,7 +276,7 @@ const RefTrf hex_parent_rt3[4] =
    { {S_HALF, S_HALF, S_ONE}, {     0, T_HALF, 0} }
 };
 
-const RefTrf hex_child_rt3[4] =
+static constexpr RefTrf hex_child_rt3[4] =
 {
    { {S_TWO, S_TWO, S_ONE}, {     0,      0, 0} },
    { {S_TWO, S_TWO, S_ONE}, {-T_ONE,      0, 0} },
@@ -286,19 +284,19 @@ const RefTrf hex_child_rt3[4] =
    { {S_TWO, S_TWO, S_ONE}, {     0, -T_ONE, 0} }
 };
 
-const RefTrf hex_parent_rt4[2] =
+static constexpr RefTrf hex_parent_rt4[2] =
 {
    { {S_ONE, S_ONE, S_HALF}, {0, 0,      0} },
    { {S_ONE, S_ONE, S_HALF}, {0, 0, T_HALF} }
 };
 
-const RefTrf hex_child_rt4[2] =
+static constexpr RefTrf hex_child_rt4[2] =
 {
    { {S_ONE, S_ONE, S_TWO}, {0, 0,      0} },
    { {S_ONE, S_ONE, S_TWO}, {0, 0, -T_ONE} }
 };
 
-const RefTrf hex_parent_rt5[4] =
+static constexpr RefTrf hex_parent_rt5[4] =
 {
    { {S_HALF, S_ONE, S_HALF}, {     0, 0,      0} },
    { {S_HALF, S_ONE, S_HALF}, {T_HALF, 0,      0} },
@@ -306,7 +304,7 @@ const RefTrf hex_parent_rt5[4] =
    { {S_HALF, S_ONE, S_HALF}, {     0, 0, T_HALF} }
 };
 
-const RefTrf hex_child_rt5[4] =
+static constexpr RefTrf hex_child_rt5[4] =
 {
    { {S_TWO, S_ONE, S_TWO}, {     0, 0,      0} },
    { {S_TWO, S_ONE, S_TWO}, {-T_ONE, 0,      0} },
@@ -314,7 +312,7 @@ const RefTrf hex_child_rt5[4] =
    { {S_TWO, S_ONE, S_TWO}, {     0, 0, -T_ONE} }
 };
 
-const RefTrf hex_parent_rt6[4] =
+static constexpr RefTrf hex_parent_rt6[4] =
 {
    { {S_ONE, S_HALF, S_HALF}, {0,      0,      0} },
    { {S_ONE, S_HALF, S_HALF}, {0, T_HALF,      0} },
@@ -322,7 +320,7 @@ const RefTrf hex_parent_rt6[4] =
    { {S_ONE, S_HALF, S_HALF}, {0, T_HALF, T_HALF} }
 };
 
-const RefTrf hex_child_rt6[4] =
+static constexpr RefTrf hex_child_rt6[4] =
 {
    { {S_ONE, S_TWO, S_TWO}, {0,      0,      0} },
    { {S_ONE, S_TWO, S_TWO}, {0, -T_ONE,      0} },
@@ -330,7 +328,7 @@ const RefTrf hex_child_rt6[4] =
    { {S_ONE, S_TWO, S_TWO}, {0, -T_ONE, -T_ONE} }
 };
 
-const RefTrf hex_parent_rt7[8] =
+static constexpr RefTrf hex_parent_rt7[8] =
 {
    { {S_HALF, S_HALF, S_HALF}, {     0,      0,      0} },
    { {S_HALF, S_HALF, S_HALF}, {T_HALF,      0,      0} },
@@ -342,7 +340,7 @@ const RefTrf hex_parent_rt7[8] =
    { {S_HALF, S_HALF, S_HALF}, {     0, T_HALF, T_HALF} }
 };
 
-const RefTrf hex_child_rt7[8] =
+static constexpr RefTrf hex_child_rt7[8] =
 {
    { {S_TWO, S_TWO, S_TWO}, {     0,      0,      0} },
    { {S_TWO, S_TWO, S_TWO}, {-T_ONE,      0,      0} },
@@ -354,7 +352,7 @@ const RefTrf hex_child_rt7[8] =
    { {S_TWO, S_TWO, S_TWO}, {     0, -T_ONE, -T_ONE} }
 };
 
-const RefTrf* hex_parent[8] =
+static const RefTrf* hex_parent[8] =
 {
    NULL,
    hex_parent_rt1,
@@ -366,7 +364,7 @@ const RefTrf* hex_parent[8] =
    hex_parent_rt7
 };
 
-const RefTrf* hex_child[8] =
+static const RefTrf* hex_child[8] =
 {
    NULL,
    hex_child_rt1,
@@ -378,7 +376,7 @@ const RefTrf* hex_child[8] =
    hex_child_rt7
 };
 
-const RefTrf tri_parent_rt3[4] =
+static constexpr RefTrf tri_parent_rt3[4] =
 {
    { { S_HALF,  S_HALF, 0}, {     0,      0, 0} },
    { { S_HALF,  S_HALF, 0}, {T_HALF,      0, 0} },
@@ -386,7 +384,7 @@ const RefTrf tri_parent_rt3[4] =
    { {-S_HALF, -S_HALF, 0}, {T_HALF, T_HALF, 0} }
 };
 
-const RefTrf tri_child_rt3[4] =
+static constexpr RefTrf tri_child_rt3[4] =
 {
    { { S_TWO,  S_TWO, 0}, {     0,      0, 0} },
    { { S_TWO,  S_TWO, 0}, {-T_ONE,      0, 0} },
@@ -394,19 +392,19 @@ const RefTrf tri_child_rt3[4] =
    { {-S_TWO, -S_TWO, 0}, { T_ONE,  T_ONE, 0} }
 };
 
-const RefTrf* tri_parent[4] =
+static const RefTrf* tri_parent[4] =
 {
    NULL, NULL, NULL,
    tri_parent_rt3
 };
 
-const RefTrf* tri_child[4] =
+static const RefTrf* tri_child[4] =
 {
    NULL, NULL, NULL,
    tri_child_rt3
 };
 
-const RefTrf prism_parent_rt3[4] =
+static constexpr RefTrf prism_parent_rt3[4] =
 {
    { { S_HALF,  S_HALF, S_ONE}, {     0,      0, 0} },
    { { S_HALF,  S_HALF, S_ONE}, {T_HALF,      0, 0} },
@@ -414,7 +412,7 @@ const RefTrf prism_parent_rt3[4] =
    { {-S_HALF, -S_HALF, S_ONE}, {T_HALF, T_HALF, 0} }
 };
 
-const RefTrf prism_child_rt3[4] =
+static constexpr RefTrf prism_child_rt3[4] =
 {
    { { S_TWO,  S_TWO, S_ONE}, {     0,      0, 0} },
    { { S_TWO,  S_TWO, S_ONE}, {-T_ONE,      0, 0} },
@@ -422,19 +420,19 @@ const RefTrf prism_child_rt3[4] =
    { {-S_TWO, -S_TWO, S_ONE}, { T_ONE,  T_ONE, 0} }
 };
 
-const RefTrf prism_parent_rt4[2] =
+static constexpr RefTrf prism_parent_rt4[2] =
 {
    { {S_ONE, S_ONE, S_HALF}, {0, 0,      0} },
    { {S_ONE, S_ONE, S_HALF}, {0, 0, T_HALF} }
 };
 
-const RefTrf prism_child_rt4[2] =
+static constexpr RefTrf prism_child_rt4[2] =
 {
    { {S_ONE, S_ONE, S_TWO}, {0, 0,      0} },
    { {S_ONE, S_ONE, S_TWO}, {0, 0, -T_ONE} }
 };
 
-const RefTrf prism_parent_rt7[8] =
+static constexpr RefTrf prism_parent_rt7[8] =
 {
    { { S_HALF,  S_HALF, S_HALF}, {     0,      0,      0} },
    { { S_HALF,  S_HALF, S_HALF}, {T_HALF,      0,      0} },
@@ -446,7 +444,7 @@ const RefTrf prism_parent_rt7[8] =
    { {-S_HALF, -S_HALF, S_HALF}, {T_HALF, T_HALF, T_HALF} }
 };
 
-const RefTrf prism_child_rt7[8] =
+static constexpr RefTrf prism_child_rt7[8] =
 {
    { { S_TWO,  S_TWO, S_TWO}, {     0,      0,      0} },
    { { S_TWO,  S_TWO, S_TWO}, {-T_ONE,      0,      0} },
@@ -458,7 +456,7 @@ const RefTrf prism_child_rt7[8] =
    { {-S_TWO, -S_TWO, S_TWO}, { T_ONE,  T_ONE, -T_ONE} }
 };
 
-const RefTrf* prism_parent[8] =
+static const RefTrf* prism_parent[8] =
 {
    NULL, NULL, NULL,
    prism_parent_rt3,
@@ -467,7 +465,7 @@ const RefTrf* prism_parent[8] =
    prism_parent_rt7
 };
 
-const RefTrf* prism_child[8] =
+static const RefTrf* prism_child[8] =
 {
    NULL, NULL, NULL,
    prism_child_rt3,
@@ -476,7 +474,7 @@ const RefTrf* prism_child[8] =
    prism_child_rt7
 };
 
-const RefTrf** geom_parent[7] =
+static const RefTrf** geom_parent[7] =
 {
    NULL,
    NULL,
@@ -487,7 +485,7 @@ const RefTrf** geom_parent[7] =
    prism_parent
 };
 
-const RefTrf** geom_child[7] =
+static const RefTrf** geom_child[7] =
 {
    NULL,
    NULL,
@@ -498,7 +496,6 @@ const RefTrf** geom_child[7] =
    prism_child
 };
 
-
-} // namespace
-
 } // namespace mfem
+
+#endif // MFEM_NCMESH_TABLES

mesh/pmesh.hpp

@@ -109,8 +109,8 @@ protected:
    // Determine sedge_ledge and sface_lface.
    void FinalizeParTopo();
 
-   // Mark all tets to ensure consistency across MPI tasks; also mark the
-   // shared and boundary triangle faces using the consistently marked tets.
+   // Mark all tets to ensure consistency across MPI tasks; also mark the shared
+   // and boundary triangle faces using the consistently marked tets.
    void MarkTetMeshForRefinement(const DSTable &v_to_v) override;
 
    /// Return a number(0-1) identifying how the given edge has been split
@@ -337,12 +337,12 @@ public:
       have_face_nbr_data(false), pncmesh(NULL) { }
 
    /// Create a parallel mesh by partitioning a serial Mesh.
-   /** The mesh is partitioned automatically or using external partitioning
-       data (the optional parameter 'partitioning_[i]' contains the desired MPI
-       rank for element 'i'). Automatic partitioning uses METIS for conforming
-       meshes and quick space-filling curve equipartitioning for nonconforming
-       meshes (elements of nonconforming meshes should ideally be ordered as a
-       sequence of face-neighbors). */
+   /** The mesh is partitioned automatically or using external partitioning data
+       (the optional parameter 'partitioning_[i]' contains the desired MPI rank
+       for element 'i'). Automatic partitioning uses METIS for conforming meshes
+       and quick space-filling curve equipartitioning for nonconforming meshes
+       (elements of nonconforming meshes should ideally be ordered as a sequence
+       of face-neighbors). */
    ParMesh(MPI_Comm comm, Mesh &mesh, const int *partitioning_ = nullptr,
            int part_method = 1);
 
@@ -446,11 +446,42 @@ public:
    int GroupNTriangles(int group) const { return group_stria.RowSize(group-1); }
    int GroupNQuadrilaterals(int group) const { return group_squad.RowSize(group-1); }
 
+   /**
+    * @brief Accessors for entities within a shared group structure.
+    * @details For all vertex/edge/face the two argument version returns the
+    * local index, for those entities with an orientation. The two out parameter
+    * version additionally returns an orientation to use in manipulating the
+    * entity.
+    *
+    * @param group The communicator group's indices
+    * @param i the index within the group
+    * @return int The local index of the entity
+    */
    int GroupVertex(int group, int i) const
    { return svert_lvert[group_svert.GetRow(group-1)[i]]; }
    void GroupEdge(int group, int i, int &edge, int &o) const;
    void GroupTriangle(int group, int i, int &face, int &o) const;
    void GroupQuadrilateral(int group, int i, int &face, int &o) const;
+   int GroupEdge(int group, int i) const
+   {
+      int e, o;
+      GroupEdge(group, i, e, o);
+      return e;
+   }
+   int GroupTriangle(int group, int i) const
+   {
+      int f, o;
+      GroupTriangle(group, i, f, o);
+      return f;
+   }
+   int GroupQuadrilateral(int group, int i) const
+   {
+      int f, o;
+      GroupQuadrilateral(group, i, f, o);
+      return f;
+   }
+
+
    ///@}
 
    /**
@@ -496,18 +527,15 @@ public:
    void GenerateOffsets(int N, HYPRE_BigInt loc_sizes[],
                         Array<HYPRE_BigInt> *offsets[]) const;
 
-   /** Return true if the face is interior or shared. In parallel, this
-       method only works if the face neighbor data is exchanged. */
-   inline bool FaceIsTrueInterior(int FaceNo) const { return Mesh::FaceIsTrueInterior(FaceNo); }
-
+   using Mesh::FaceIsTrueInterior;
    void ExchangeFaceNbrData();
    void ExchangeFaceNbrNodes();
 
    void SetCurvature(int order, bool discont = false, int space_dim = -1,
                      int ordering = 1) override;
 
-   /** Replace the internal node GridFunction with a new GridFunction defined
-       on the given FiniteElementSpace. The new node coordinates are projected
+   /** Replace the internal node GridFunction with a new GridFunction defined on
+       the given FiniteElementSpace. The new node coordinates are projected
        (derived) from the current nodes/vertices. */
    void SetNodalFESpace(FiniteElementSpace *nfes) override;
    void SetNodalFESpace(ParFiniteElementSpace *npfes);
@@ -571,15 +599,15 @@ public:
                                       IsoparametricTransformation &ElTr2,
                                       int mask = 31) const override;
 
-   /// @brief Get the FaceElementTransformations for the given shared face
-   /// (edge 2D) using the shared face index @a sf. @a fill2 specify if the
-   /// information for elem2 of the face should be computed or not.
-   /// In the returned object, 1 and 2 refer to the local and the neighbor
-   /// elements, respectively.
+   /// @brief Get the FaceElementTransformations for the given shared face (edge
+   /// 2D) using the shared face index @a sf. @a fill2 specify if the
+   /// information for elem2 of the face should be computed or not. In the
+   /// returned object, 1 and 2 refer to the local and the neighbor elements,
+   /// respectively.
    ///
    /// @note The returned object is owned by the class and is shared, i.e.,
-   /// calling this function resets pointers obtained from previous calls.
-   /// Also, the returned object should NOT be deleted by the caller.
+   /// calling this function resets pointers obtained from previous calls. Also,
+   /// the returned object should NOT be deleted by the caller.
    FaceElementTransformations *
    GetSharedFaceTransformations(int sf, bool fill2 = true);
 
@@ -591,15 +619,14 @@ public:
                                      IsoparametricTransformation &ElTr2,
                                      bool fill2 = true) const;
 
-   /// @brief Get the FaceElementTransformations for the given shared face
-   /// (edge 2D) using the face index @a FaceNo. @a fill2 specify if the
-   /// information for elem2 of the face should be computed or not.
-   /// In the returned object, 1 and 2 refer to the local and the neighbor
-   /// elements, respectively.
+   /// @brief Get the FaceElementTransformations for the given shared face (edge
+   /// 2D) using the face index @a FaceNo. @a fill2 specify if the information
+   /// for elem2 of the face should be computed or not. In the returned object,
+   /// 1 and 2 refer to the local and the neighbor elements, respectively.
    ///
    /// @note The returned object is owned by the class and is shared, i.e.,
-   /// calling this function resets pointers obtained from previous calls.
-   /// Also, the returned object should NOT be deleted by the caller.
+   /// calling this function resets pointers obtained from previous calls. Also,
+   /// the returned object should NOT be deleted by the caller.
    FaceElementTransformations *
    GetSharedFaceTransformationsByLocalIndex(int FaceNo, bool fill2 = true);
 
@@ -615,8 +642,8 @@ public:
    /// neighbor.
    ///
    /// @note The returned object is owned by the class and is shared, i.e.,
-   /// calling this function resets pointers obtained from previous calls.
-   /// Also, the returned object should NOT be deleted by the caller.
+   /// calling this function resets pointers obtained from previous calls. Also,
+   /// the returned object should NOT be deleted by the caller.
    ElementTransformation *GetFaceNbrElementTransformation(int FaceNo);
 
    /// @brief Variant of GetFaceNbrElementTransformation using a user allocated
@@ -637,11 +664,11 @@ public:
    /** @brief Returns the number of local faces according to the requested type,
        does not count master non-conforming faces.
 
-       If type==Boundary returns only the number of true boundary faces
-       contrary to GetNBE() that returns all "boundary" elements which may
-       include actual interior faces.
-       Similarly, if type==Interior, only the true interior faces (including
-       shared faces) are counted excluding all master non-conforming faces. */
+       If type==Boundary returns only the number of true boundary faces contrary
+       to GetNBE() that returns all "boundary" elements which may include actual
+       interior faces. Similarly, if type==Interior, only the true interior
+       faces (including shared faces) are counted excluding all master
+       non-conforming faces. */
    int GetNFbyType(FaceType type) const override;
 
    void GenerateBoundaryElements() override
@@ -657,9 +684,9 @@ public:
        sequence of elements. Works for nonconforming meshes only. */
    void Rebalance();
 
-   /** Load balance a nonconforming mesh using a user-defined partition.
-       Each local element 'i' is migrated to processor rank 'partition[i]',
-       for 0 <= i < GetNE(). */
+   /** Load balance a nonconforming mesh using a user-defined partition. Each
+       local element 'i' is migrated to processor rank 'partition[i]', for 0 <=
+       i < GetNE(). */
    void Rebalance(const Array<int> &partition);
 
    /** Save the mesh in a parallel mesh format. If @a comments is non-empty, it

mesh/pncmesh.hpp

@@ -63,6 +63,8 @@ class FiniteElementSpace;
  */
 class ParNCMesh : public NCMesh
 {
+protected:
+   ParNCMesh() = default;
 public:
    /// Construct by partitioning a serial NCMesh.
    /** SFC partitioning is used by default. A user-specified partition can be
@@ -252,6 +254,7 @@ public:
 protected: // interface for ParMesh
 
    friend class ParMesh;
+   friend class ParSubMesh;
 
    /** For compatibility with conforming code in ParMesh and ParFESpace.
        Initializes shared structures in ParMesh: gtopo, shared_*, group_s*,

mesh/submesh/ncsubmesh.cpp

@@ -0,0 +1,133 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "ncsubmesh.hpp"
+
+#include <unordered_map>
+#include "submesh_utils.hpp"
+#include "submesh.hpp"
+
+namespace mfem
+{
+
+using namespace SubMeshUtils;
+
+NCSubMesh::NCSubMesh(SubMesh& submesh, const NCMesh &parent, From from,
+                     const Array<int> &attributes)
+   : NCMesh(), parent_(&parent)
+{
+   Dim = submesh.Dimension();
+   spaceDim = submesh.SpaceDimension();
+   MyRank = 0;
+   Iso = true;
+   Legacy = false;
+
+   if (from == From::Domain)
+   {
+      SubMeshUtils::ConstructVolumeTree(*this, attributes);
+   }
+   else if (from == From::Boundary)
+   {
+      SubMeshUtils::ConstructFaceTree(*this, attributes);
+   }
+
+   // Loop over all nodes, and reparent based on the node relations of the
+   // parent
+   for (int i = 0; i < parent_node_ids_.Size(); i++)
+   {
+      const auto &parent_node = parent.nodes[parent_node_ids_[i]];
+      const int submesh_p1 = parent_to_submesh_node_ids_[parent_node.p1];
+      const int submesh_p2 = parent_to_submesh_node_ids_[parent_node.p2];
+      nodes.Reparent(i, submesh_p1, submesh_p2);
+   }
+
+   nodes.UpdateUnused();
+   for (int i = 0; i < elements.Size(); i++)
+   {
+      if (elements[i].IsLeaf())
+      {
+         // Register all faces
+         RegisterFaces(i);
+      }
+   }
+
+   InitRootElements();
+   InitRootState(root_state.Size());
+   InitGeomFlags();
+   Update(); // Fills in secondary information based off of elements, nodes and faces.
+
+   // If parent has coordinates defined, copy the relevant portion
+   if (parent.coordinates.Size() > 0)
+   {
+      coordinates.SetSize(3*parent_node_ids_.Size());
+      parent.tmp_vertex = new TmpVertex[parent.nodes.NumIds()];
+      for (int n = 0; n < parent_node_ids_.Size(); n++)
+      {
+         std::memcpy(&coordinates[3*n], parent.CalcVertexPos(parent_node_ids_[n]),
+                     3*sizeof(real_t));
+      }
+      delete [] parent.tmp_vertex;
+   }
+
+   // The element indexing was changed as part of generation of leaf elements.
+   // We need to update the map.
+   if (from == From::Domain)
+   {
+      // The element indexing was changed as part of generation of leaf
+      // elements. We need to update the map.
+      submesh.parent_to_submesh_element_ids_ = -1;
+      for (int i = 0; i < submesh.parent_element_ids_.Size(); i++)
+      {
+         submesh.parent_element_ids_[i] =
+            parent.elements[parent_element_ids_[leaf_elements[i]]].index;
+         submesh.parent_to_submesh_element_ids_[submesh.parent_element_ids_[i]] = i;
+      }
+   }
+   else
+   {
+      submesh.parent_to_submesh_element_ids_ = -1;
+      // parent elements are BOUNDARY elements, need to map face index to be.
+      const auto &parent_face_to_be = submesh.GetParent()->GetFaceToBdrElMap();
+      MFEM_ASSERT(NElements == submesh.GetNE(), "!");
+      auto new_parent_to_submesh_element_ids = submesh.parent_to_submesh_element_ids_;
+      Array<int> new_parent_element_ids;
+      new_parent_element_ids.Reserve(submesh.parent_element_ids_.Size());
+      for (int i = 0; i < submesh.parent_element_ids_.Size(); i++)
+      {
+         new_parent_element_ids.Append(
+            parent_face_to_be[parent.faces[parent_element_ids_[leaf_elements[i]]].index]);
+         new_parent_to_submesh_element_ids[new_parent_element_ids[i]] = i;
+      }
+
+      MFEM_ASSERT(new_parent_element_ids.Size() == submesh.parent_element_ids_.Size(),
+                  "!");
+#ifdef MFEM_DEBUG
+      for (auto x : new_parent_element_ids)
+      {
+         MFEM_ASSERT(std::find(submesh.parent_element_ids_.begin(),
+                               submesh.parent_element_ids_.end(), x)
+                     != submesh.parent_element_ids_.end(),
+                     x << " not found in submesh.parent_element_ids_");
+      }
+      for (auto x : submesh.parent_element_ids_)
+      {
+         MFEM_ASSERT(std::find(new_parent_element_ids.begin(),
+                               new_parent_element_ids.end(), x)
+                     != new_parent_element_ids.end(), x << " not found in new_parent_element_ids_");
+      }
+#endif
+      submesh.parent_element_ids_ = std::move(new_parent_element_ids);
+      submesh.parent_to_submesh_element_ids_ =
+         std::move(new_parent_to_submesh_element_ids);
+   }
+}
+
+} // namespace mfem

mesh/submesh/ncsubmesh.hpp

@@ -0,0 +1,97 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_NCSUBMESH
+#define MFEM_NCSUBMESH
+
+#include "../ncmesh.hpp"
+#include "submesh.hpp"
+#include "submesh_utils.hpp"
+#include <unordered_map>
+
+namespace mfem
+{
+
+/**
+ * @brief Class representing a Nonconformal SubMesh. This is only used by
+ * SubMesh.
+ */
+class NCSubMesh : public NCMesh
+{
+   friend class SubMesh; ///< Only SubMesh can use methods in this class
+public:
+   using From = SubMesh::From; ///< Convenience type alias
+   /// Get the parent NCMesh object
+   const NCMesh* GetParent() const
+   {
+      return parent_;
+   }
+
+   /**
+   * @brief Check if NCMesh @a m is a NCSubMesh.
+   *
+   * @param m The input NCMesh
+   */
+   static bool IsNCSubMesh(const NCMesh *m)
+   {
+      return dynamic_cast<const NCSubMesh *>(m) != nullptr;
+   }
+private:
+
+   /// Private constructor
+   NCSubMesh(SubMesh& submesh, const NCMesh &parent, From from,
+             const Array<int> &attributes);
+
+   /// The parent NCMesh. Not owned.
+   const NCMesh *parent_;
+
+   /// Mapping from submesh element nc ids (index of the array), to the parent
+   /// element ids. If from a boundary, these map to faces in the parent.
+   Array<int> parent_element_ids_;
+
+   /// Mapping from NCSubMesh node ids (index of the array), to the parent
+   /// NCMesh node ids.
+   Array<int> parent_node_ids_;
+
+   /// Mapping from parent NCMesh node ids to submesh NCMesh node ids.
+   // Inverse map of parent_node_ids_.
+   std::unordered_map<int, int> parent_to_submesh_node_ids_;
+
+   /// Mapping from parent NCMesh element ids to submesh NCMesh element ids.
+   // Inverse map of parent_element_ids_.
+   std::unordered_map<int, int> parent_to_submesh_element_ids_;
+
+   // Helper friend methods for construction.
+   friend void SubMeshUtils::ConstructFaceTree<NCSubMesh>(NCSubMesh &submesh,
+                                                          const Array<int> &attributes);
+   friend void SubMeshUtils::ConstructVolumeTree<NCSubMesh>(NCSubMesh &submesh,
+                                                            const Array<int> &attributes);
+
+   /**
+    * @brief Accessor for parent nodes
+    * @details Required to bypass access protection in parent class.
+    *
+    * @return const HashTable<Node>&
+    */
+   const HashTable<Node> &ParentNodes() const { return parent_->nodes; }
+
+   /**
+    * @brief Accessor for parent faces
+    * @details Required to bypass access protection in parent class.
+    *
+    * @return const HashTable<Face>&
+    */
+   const HashTable<Face> &ParentFaces() const { return parent_->faces; }
+};
+
+} // namespace mfem
+
+#endif // MFEM_NCSUBMESH

mesh/submesh/pncsubmesh.cpp

@@ -0,0 +1,157 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "pncsubmesh.hpp"
+
+#include <numeric>
+#include <unordered_map>
+#include "submesh_utils.hpp"
+#include "psubmesh.hpp"
+namespace mfem
+{
+
+using namespace SubMeshUtils;
+
+
+ParNCSubMesh::ParNCSubMesh(ParSubMesh& submesh, const ParNCMesh &parent,
+                           From from, const Array<int> &attributes)
+   : ParNCMesh(), parent_(&parent)
+{
+   MyComm = submesh.GetComm();
+   NRanks = submesh.GetNRanks();
+   MyRank = submesh.GetMyRank();
+
+   Dim = submesh.Dimension();
+   spaceDim = submesh.SpaceDimension();
+   Iso = true;
+   Legacy = false;
+
+   // Loop over parent leaf elements and add nodes for all vertices. Register as
+   // top level nodes, will reparent when looping over edges. Cannot add edge
+   // nodes at same time because top level vertex nodes must be contiguous and
+   // first in node list (see coordinates).
+   if (from == From::Domain)
+   {
+      SubMeshUtils::ConstructVolumeTree(*this, attributes);
+   }
+   else if (from == From::Boundary)
+   {
+      SubMeshUtils::ConstructFaceTree(*this, attributes);
+   }
+
+   // Loop over all nodes, and reparent based on the node relations of the
+   // parent
+   for (int i = 0; i < parent_node_ids_.Size(); i++)
+   {
+      const auto &parent_node = parent.nodes[parent_node_ids_[i]];
+      const int submesh_p1 = parent_to_submesh_node_ids_[parent_node.p1];
+      const int submesh_p2 = parent_to_submesh_node_ids_[parent_node.p2];
+      nodes.Reparent(i, submesh_p1, submesh_p2);
+   }
+
+   nodes.UpdateUnused();
+   for (int i = 0; i < elements.Size(); i++)
+   {
+      if (elements[i].IsLeaf())
+      {
+         // Register all faces
+         RegisterFaces(i);
+      }
+   }
+
+   InitRootElements();
+   InitRootState(root_state.Size());
+   InitGeomFlags();
+   Update(); // Fills in secondary information based off of elements, nodes and faces.
+#ifdef MFEM_DEBUG
+   // Check all processors have the same number of roots
+   {
+      int p[2] = {root_state.Size(), -root_state.Size()};
+      MPI_Allreduce(MPI_IN_PLACE, p, 2, MPI_INT, MPI_MIN, submesh.GetComm());
+      MFEM_ASSERT(p[0] == -p[1], "Ranks must agree on number of root elements: min "
+                  << p[0] << " max " << -p[1] << " local " << root_state.Size() << " MyRank " <<
+                  submesh.GetMyRank());
+   }
+#endif
+
+   // If parent has coordinates defined, copy the relevant portion
+   if (parent.coordinates.Size() > 0)
+   {
+      // Loop over new_nodes -> coordinates is indexed by node.
+      coordinates.SetSize(3*parent_node_ids_.Size());
+      parent.tmp_vertex = new TmpVertex[parent.nodes.NumIds()];
+      for (int n = 0; n < parent_node_ids_.Size(); n++)
+      {
+         std::memcpy(&coordinates[3*n], parent.CalcVertexPos(parent_node_ids_[n]),
+                     3*sizeof(real_t));
+      }
+      delete [] parent.tmp_vertex;
+   }
+
+   // The element indexing was changed as part of generation of leaf elements.
+   // We need to update the map.
+   if (from == From::Domain)
+   {
+      // The element indexing was changed as part of generation of leaf
+      // elements. We need to update the map.
+      submesh.parent_to_submesh_element_ids_ = -1;
+      for (int i = 0; i < submesh.parent_element_ids_.Size(); i++)
+      {
+         submesh.parent_element_ids_[i] =
+            parent.elements[parent_element_ids_[leaf_elements[i]]].index;
+         submesh.parent_to_submesh_element_ids_[submesh.parent_element_ids_[i]] = i;
+      }
+   }
+   else
+   {
+      submesh.parent_to_submesh_element_ids_ = -1;
+      // parent elements are BOUNDARY elements, need to map face index to be.
+      const auto &parent_face_to_be = submesh.GetParent()->GetFaceToBdrElMap();
+      MFEM_ASSERT(NElements == submesh.GetNE(), NElements << ' ' << submesh.GetNE());
+      auto new_parent_to_submesh_element_ids = submesh.parent_to_submesh_element_ids_;
+      Array<int> new_parent_element_ids;
+      new_parent_element_ids.Reserve(submesh.parent_element_ids_.Size());
+      for (int i = 0; i < submesh.parent_element_ids_.Size(); i++)
+      {
+         new_parent_element_ids.Append(
+            parent_face_to_be[parent.faces[parent_element_ids_[leaf_elements[i]]].index]);
+         new_parent_to_submesh_element_ids[new_parent_element_ids[i]] = i;
+      }
+
+      MFEM_ASSERT(new_parent_element_ids.Size() == submesh.parent_element_ids_.Size(),
+                  new_parent_element_ids.Size() << ' ' << submesh.parent_element_ids_.Size());
+#ifdef MFEM_DEBUG
+      for (auto x : new_parent_element_ids)
+      {
+         MFEM_ASSERT(std::find(submesh.parent_element_ids_.begin(),
+                               submesh.parent_element_ids_.end(), x)
+                     != submesh.parent_element_ids_.end(),
+                     x << " not found in submesh.parent_element_ids_");
+      }
+      for (auto x : submesh.parent_element_ids_)
+      {
+         MFEM_ASSERT(std::find(new_parent_element_ids.begin(),
+                               new_parent_element_ids.end(), x)
+                     != new_parent_element_ids.end(), x << " not found in new_parent_element_ids_");
+      }
+#endif
+      submesh.parent_element_ids_ = new_parent_element_ids;
+      submesh.parent_to_submesh_element_ids_ = new_parent_to_submesh_element_ids;
+   }
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_MPI

mesh/submesh/pncsubmesh.hpp

@@ -0,0 +1,102 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PNCSUBMESH
+#define MFEM_PNCSUBMESH
+
+#include "../../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "../pncmesh.hpp"
+#include "psubmesh.hpp"
+#include "submesh_utils.hpp"
+#include <unordered_map>
+
+namespace mfem
+{
+
+/**
+ * @brief Class representing a Parallel Nonconformal SubMesh. This is only used
+ * by ParSubMesh.
+ */
+class ParNCSubMesh : public ParNCMesh
+{
+   friend class ParSubMesh; ///< Only ParSubMesh can use methods in this class
+public:
+   using From = SubMesh::From; ///< Convenience type alias
+   /**
+   * @brief Check if NCMesh @a m is a ParNCSubMesh.
+   *
+   * @param m The input Mesh
+   */
+   static bool IsParNCSubMesh(const NCMesh *m)
+   {
+      return dynamic_cast<const ParNCSubMesh *>(m) != nullptr;
+   }
+   /// Get the parent ParNCMesh object
+   const ParNCMesh* GetParent() const
+   {
+      return parent_;
+   }
+
+protected:
+   /// protected constructor
+   ParNCSubMesh(ParSubMesh& submesh, const ParNCMesh &parent, From from,
+                const Array<int> &attributes);
+
+   /// The parent ParNCMesh. Not owned.
+   const ParNCMesh *parent_;
+
+   /// Mapping from submesh element nc ids (index of the array), to the parent
+   /// element ids. If from a boundary, these map to faces in the parent.
+   Array<int> parent_element_ids_;
+
+   /// Mapping from ParNCSubMesh node ids (index of the array), to the parent
+   /// NCMesh node ids.
+   Array<int> parent_node_ids_;
+
+   /// Mapping from parent NCMesh node ids to submesh NCMesh node ids.
+   // Inverse map of parent_node_ids_.
+   std::unordered_map<int, int> parent_to_submesh_node_ids_;
+
+   /// Mapping from parent NCMesh element ids to submesh NCMesh element ids.
+   // Inverse map of parent_element_ids_.
+   std::unordered_map<int, int> parent_to_submesh_element_ids_;
+
+   // Helper friend methods for construction.
+   friend void SubMeshUtils::ConstructFaceTree<ParNCSubMesh>
+   (ParNCSubMesh &submesh, const Array<int> &attributes);
+   friend void SubMeshUtils::ConstructVolumeTree<ParNCSubMesh>
+   (ParNCSubMesh &submesh, const Array<int> &attributes);
+
+   /**
+    * @brief Accessor for parent nodes
+    * @details Required to bypass access protection in parent class.
+    *
+    * @return const HashTable<Node>&
+    */
+   const HashTable<Node> &ParentNodes() const { return parent_->nodes; }
+
+   /**
+    * @brief Accessor for parent faces
+    * @details Required to bypass access protection in parent class.
+    *
+    * @return const HashTable<Face>&
+    */
+   const HashTable<Face> &ParentFaces() const { return parent_->faces; }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_MPI
+
+#endif // MFEM_PNCSUBMESH

mesh/submesh/psubmesh.cpp

@@ -17,6 +17,7 @@
 #include <unordered_set>
 #include <algorithm>
 #include "psubmesh.hpp"
+#include "pncsubmesh.hpp"
 #include "submesh_utils.hpp"
 #include "../segment.hpp"
 
@@ -24,33 +25,29 @@ namespace mfem
 {
 
 ParSubMesh ParSubMesh::CreateFromDomain(const ParMesh &parent,
-                                        Array<int> &domain_attributes)
+                                        const Array<int> &domain_attributes)
 {
    return ParSubMesh(parent, SubMesh::From::Domain, domain_attributes);
 }
 
 ParSubMesh ParSubMesh::CreateFromBoundary(const ParMesh &parent,
-                                          Array<int> &boundary_attributes)
+                                          const Array<int> &boundary_attributes)
 {
    return ParSubMesh(parent, SubMesh::From::Boundary, boundary_attributes);
 }
 
 ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
-                       Array<int> &attributes) : parent_(parent), from_(from), attributes_(attributes)
+                       const Array<int> &attributes) : parent_(parent), from_(from),
+   attributes_(attributes)
 {
-   if (Nonconforming())
-   {
-      MFEM_ABORT("SubMesh does not support non-conforming meshes");
-   }
-
    MyComm = parent.GetComm();
    NRanks = parent.GetNRanks();
    MyRank = parent.GetMyRank();
 
-   // This violation of const-ness may be justified in this instance because
-   // the exchange of face neighbor information only establishes or updates
-   // derived information without altering the primary mesh information,
-   // i.e., the topology, geometry, or region attributes.
+   // This violation of const-ness may be justified in this instance because the
+   // exchange of face neighbor information only establishes or updates derived
+   // information without altering the primary mesh information, i.e., the
+   // topology, geometry, or region attributes.
    const_cast<ParMesh&>(parent).ExchangeFaceNbrData();
 
    if (from == SubMesh::From::Domain)
@@ -70,11 +67,6 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
                                                                       attributes_, true);
    }
 
-   // Don't let boundary elements get generated automatically. This would
-   // generate boundary elements on each rank locally, which is topologically
-   // wrong for the distributed SubMesh.
-   FinalizeTopology(false);
-
    parent_to_submesh_vertex_ids_.SetSize(parent_.GetNV());
    parent_to_submesh_vertex_ids_ = -1;
    for (int i = 0; i < parent_vertex_ids_.Size(); i++)
@@ -82,6 +74,43 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       parent_to_submesh_vertex_ids_[parent_vertex_ids_[i]] = i;
    }
 
+   parent_to_submesh_element_ids_.SetSize(from == From::Boundary ? parent.GetNBE()
+                                          : parent.GetNE());
+   parent_to_submesh_element_ids_ = -1;
+   for (int i = 0; i < parent_element_ids_.Size(); i++)
+   {
+      parent_to_submesh_element_ids_[parent_element_ids_[i]] = i;
+   }
+
+   // Don't let boundary elements get generated automatically. This would
+   // generate boundary elements on each rank locally, which is topologically
+   // wrong for the distributed SubMesh.
+   FinalizeTopology(false);
+
+   if (parent.Nonconforming())
+   {
+      pncmesh = new ParNCSubMesh(*this, *parent.pncmesh, from, attributes);
+      pncsubmesh_ = dynamic_cast<ParNCSubMesh*>(pncmesh);
+      ncmesh = pncmesh;
+      InitFromNCMesh(*pncmesh);
+      pncmesh->OnMeshUpdated(this);
+
+      // Update the submesh to parent vertex mapping, NCSubMesh reordered the
+      // vertices so the map to parent is no longer valid.
+      parent_to_submesh_vertex_ids_ = -1;
+      for (int i = 0; i < parent_vertex_ids_.Size(); i++)
+      {
+         // vertex -> node -> parent node -> parent vertex
+         auto node = pncsubmesh_->vertex_nodeId[i];
+         auto parent_node = pncsubmesh_->parent_node_ids_[node];
+         auto parent_vertex = parent.pncmesh->GetNodeVertex(parent_node);
+         parent_vertex_ids_[i] = parent_vertex;
+         parent_to_submesh_vertex_ids_[parent_vertex] = i;
+      }
+      GenerateNCFaceInfo();
+      SetAttributes();
+   }
+
    DSTable v2v(parent_.GetNV());
    parent_.GetVertexToVertexTable(v2v);
    for (int i = 0; i < NumOfEdges; i++)
@@ -115,7 +144,6 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       }
 
       parent_face_ori_.SetSize(NumOfFaces);
-
       for (int i = 0; i < NumOfFaces; i++)
       {
          Array<int> sub_vert;
@@ -191,7 +219,6 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
    // Every rank containing elements of the ParSubMesh attributes now has a
    // local ParSubMesh. We have to connect the local meshes and assign global
    // boundaries correctly.
-
    Array<int> rhvtx;
    FindSharedVerticesRanks(rhvtx);
    AppendSharedVerticesGroups(groups, rhvtx);
@@ -207,6 +234,7 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       AppendSharedFacesGroups(groups, rht, rhq);
    }
 
+
    // Build the group communication topology
    gtopo.SetComm(MyComm);
    gtopo.Create(groups, 822);
@@ -239,113 +267,17 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
 
    ExchangeFaceNbrData();
 
-   // Add boundaries
+   SubMeshUtils::AddBoundaryElements(*this,
+                                     (from == SubMesh::From::Domain)
+                                     ? FindGhostBoundaryElementAttributes()
+                                     : std::unordered_map<int,int> {});
+
+   if (Dim > 1)
    {
-      const int num_codim_1 = [this]()
-      {
-         if (Dim == 1) { return NumOfVertices; }
-         else if (Dim == 2) { return NumOfEdges; }
-         else if (Dim == 3) { return NumOfFaces; }
-         else { MFEM_ABORT("Invalid dimension."); return -1; }
-      }();
-
-      if (Dim == 3)
-      {
-         // In 3D we check for `bel_to_edge`. It shouldn't have been set
-         // previously.
-         delete bel_to_edge;
-         bel_to_edge = nullptr;
-      }
-
-      NumOfBdrElements = 0;
-      for (int i = 0; i < num_codim_1; i++)
-      {
-         if (GetFaceInformation(i).IsBoundary())
-         {
-            NumOfBdrElements++;
-         }
-      }
-
-      boundary.SetSize(NumOfBdrElements);
-      be_to_face.SetSize(NumOfBdrElements);
-      Array<int> parent_face_to_be = parent.GetFaceToBdrElMap();
-      int max_bdr_attr = parent.bdr_attributes.Max();
-
-      for (int i = 0, j = 0; i < num_codim_1; i++)
-      {
-         if (GetFaceInformation(i).IsBoundary())
-         {
-            boundary[j] = faces[i]->Duplicate(this);
-            be_to_face[j] = i;
-
-            if (from == SubMesh::From::Domain && Dim >= 2)
-            {
-               int pbeid = Dim == 3 ? parent_face_to_be[parent_face_ids_[i]] :
-                           parent_face_to_be[parent_edge_ids_[i]];
-               if (pbeid != -1)
-               {
-                  boundary[j]->SetAttribute(parent.GetBdrAttribute(pbeid));
-               }
-               else
-               {
-                  boundary[j]->SetAttribute(max_bdr_attr + 1);
-               }
-            }
-            else
-            {
-               boundary[j]->SetAttribute(SubMesh::GENERATED_ATTRIBUTE);
-            }
-            ++j;
-         }
-      }
-
-      if (from == SubMesh::From::Domain && Dim >= 2)
-      {
-         // Search for and count interior boundary elements
-         int InteriorBdrElems = 0;
-         for (int i=0; i<parent.GetNBE(); i++)
-         {
-            const int parentFaceIdx = parent.GetBdrElementFaceIndex(i);
-            const int submeshFaceIdx =
-               Dim == 3 ?
-               parent_to_submesh_face_ids_[parentFaceIdx] :
-               parent_to_submesh_edge_ids_[parentFaceIdx];
-
-            if (submeshFaceIdx == -1) { continue; }
-            if (GetFaceInformation(submeshFaceIdx).IsBoundary()) { continue; }
-
-            InteriorBdrElems++;
-         }
-
-         if (InteriorBdrElems > 0)
-         {
-            const int OldNumOfBdrElements = NumOfBdrElements;
-            NumOfBdrElements += InteriorBdrElems;
-            boundary.SetSize(NumOfBdrElements);
-            be_to_face.SetSize(NumOfBdrElements);
-
-            // Search for and transfer interior boundary elements
-            for (int i=0, j = OldNumOfBdrElements; i<parent.GetNBE(); i++)
-            {
-               const int parentFaceIdx = parent.GetBdrElementFaceIndex(i);
-               const int submeshFaceIdx =
-                  parent_to_submesh_face_ids_[parentFaceIdx];
-
-               if (submeshFaceIdx == -1) { continue; }
-               if (GetFaceInformation(submeshFaceIdx).IsBoundary())
-               { continue; }
-
-               boundary[j] = faces[submeshFaceIdx]->Duplicate(this);
-               be_to_face[j] = submeshFaceIdx;
-               boundary[j]->SetAttribute(parent.GetBdrAttribute(i));
-
-               ++j;
-            }
-         }
-      }
+      if (!el_to_edge) { el_to_edge = new Table; }
+      NumOfEdges = GetElementToEdgeTable(*el_to_edge);
    }
-
-   if (Dim == 3)
+   if (Dim > 2)
    {
       GetElementToFaceTable();
    }
@@ -376,84 +308,6 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
 
       Transfer(*pn, *n);
    }
-
-   if (Dim > 1)
-   {
-      if (!el_to_edge) { el_to_edge = new Table; }
-      NumOfEdges = GetElementToEdgeTable(*el_to_edge);
-   }
-
-   if (Dim > 1 && from == SubMesh::From::Domain)
-   {
-      // Order 0 Raviart-Thomas space will have precisely 1 DoF per face.
-      // We can use this DoF to communicate boundary attribute numbers.
-      RT_FECollection fec_rt(0, Dim);
-      ParFiniteElementSpace parent_fes_rt(const_cast<ParMesh*>(&parent),
-                                          &fec_rt);
-
-      ParGridFunction parent_bdr_attr_gf(&parent_fes_rt);
-      parent_bdr_attr_gf = 0.0;
-
-      Array<int> vdofs;
-      DofTransformation doftrans;
-      int dof, faceIdx;
-      real_t sign, w;
-
-      // Copy boundary attribute numbers into local portion of a parallel
-      // grid function
-      parent_bdr_attr_gf.HostReadWrite(); // not modifying all entries
-      for (int i=0; i<parent.GetNBE(); i++)
-      {
-         faceIdx = parent.GetBdrElementFaceIndex(i);
-         const FaceInformation &faceInfo = parent.GetFaceInformation(faceIdx);
-         parent_fes_rt.GetBdrElementDofs(i, vdofs, doftrans);
-         dof = ParFiniteElementSpace::DecodeDof(vdofs[0], sign);
-
-         // Shared interior boundary elements are not duplicated across
-         // processor boundaries but ParGridFunction::ParallelAverage will
-         // assume both processors contribute to the averaged DoF value. So,
-         // we multiply shared boundary values by 2 so that the average
-         // produces the desired value.
-         w = faceInfo.IsShared() ? 2.0 : 1.0;
-
-         // The DoF sign is needed to ensure that non-shared interior
-         // boundary values sum properly rather than canceling.
-         parent_bdr_attr_gf[dof] = sign * w * parent.GetBdrAttribute(i);
-      }
-
-      Vector parent_bdr_attr(parent_fes_rt.GetTrueVSize());
-
-      // Compute the average of the attribute numbers
-      parent_bdr_attr_gf.ParallelAverage(parent_bdr_attr);
-      // Distribute boundary attributes to neighboring processors
-      parent_bdr_attr_gf.Distribute(parent_bdr_attr);
-
-      ParFiniteElementSpace submesh_fes_rt(this,
-                                           &fec_rt);
-
-      ParGridFunction submesh_bdr_attr_gf(&submesh_fes_rt);
-
-      // Transfer the averaged boundary attribute values to the submesh
-      auto transfer_map = ParSubMesh::CreateTransferMap(parent_bdr_attr_gf,
-                                                        submesh_bdr_attr_gf);
-      transfer_map.Transfer(parent_bdr_attr_gf, submesh_bdr_attr_gf);
-
-      // Extract the boundary attribute numbers from the local portion
-      // of the ParGridFunction and set the corresponding boundary element
-      // attributes.
-      int attr;
-      for (int i=0; i<NumOfBdrElements; i++)
-      {
-         submesh_fes_rt.GetBdrElementDofs(i, vdofs, doftrans);
-         dof = ParFiniteElementSpace::DecodeDof(vdofs[0], sign);
-         attr = (int)std::round(std::abs(submesh_bdr_attr_gf[dof]));
-         if (attr != 0)
-         {
-            SetBdrAttribute(i, attr);
-         }
-      }
-   }
-
    SetAttributes();
    Finalize();
 }
@@ -494,6 +348,7 @@ void ParSubMesh::FindSharedVerticesRanks(Array<int> &rhvtx)
       }
    }
 
+
    // Compute the sum on the root rank and broadcast the result to all ranks.
    svert_comm.Reduce(rhvtx, GroupCommunicator::Sum);
    svert_comm.Bcast<int>(rhvtx, 0);
@@ -511,8 +366,8 @@ void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
    rhe.SetSize(nsedges);
    rhe = 0;
 
-   // On each rank of the group, locally determine if the shared edge is in
-   // the SubMesh.
+   // On each rank of the group, locally determine if the shared edge is in the
+   // SubMesh.
    for (int g = 1, se = 0; g < parent_.GetNGroups(); g++)
    {
       const int group_sz = parent_.gtopo.GetGroupSize(g);
@@ -528,8 +383,7 @@ void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
 
       for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
       {
-         int ple, o;
-         parent_.GroupEdge(g, ge, ple, o);
+         int ple = parent_.GroupEdge(g, ge);
          int submesh_edge_id = parent_to_submesh_edge_ids_[ple];
          if (submesh_edge_id != -1)
          {
@@ -538,6 +392,7 @@ void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
       }
    }
 
+
    // Compute the sum on the root rank and broadcast the result to all ranks.
    sedge_comm.Reduce(rhe, GroupCommunicator::Sum);
    sedge_comm.Bcast<int>(rhe, 0);
@@ -545,50 +400,21 @@ void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
 
 void ParSubMesh::FindSharedFacesRanks(Array<int>& rht, Array<int> &rhq)
 {
-   GroupCommunicator squad_comm(parent_.gtopo);
-   parent_.GetSharedQuadCommunicator(squad_comm);
-
-   int nsquad = squad_comm.GroupLDofTable().Size_of_connections();
-
-   rhq.SetSize(nsquad);
-   rhq = 0;
-
-   for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
-   {
-      for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
-      {
-         // Group size of a shared face is always 2
-
-         int plq, o;
-         parent_.GroupQuadrilateral(g, gq, plq, o);
-         int submesh_face_id = parent_to_submesh_face_ids_[plq];
-         if (submesh_face_id != -1)
-         {
-            rhq[sq] = 1;
-         }
-      }
-   }
-
-   // Compute the sum on the root rank and broadcast the result to all ranks.
-   squad_comm.Reduce(rhq, GroupCommunicator::Sum);
-   squad_comm.Bcast<int>(rhq, 0);
-
    GroupCommunicator stria_comm(parent_.gtopo);
    parent_.GetSharedTriCommunicator(stria_comm);
-
    int nstria = stria_comm.GroupLDofTable().Size_of_connections();
-
    rht.SetSize(nstria);
    rht = 0;
 
    for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
    {
+      MFEM_ASSERT(parent_.gtopo.GetGroupSize(g) == 2
+                  || parent_.GroupNTriangles(g) == 0,
+                  parent_.gtopo.GetGroupSize(g) << ' ' << parent_.GroupNTriangles(g));
       for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
       {
          // Group size of a shared face is always 2
-
-         int plt, o;
-         parent_.GroupTriangle(g, gt, plt, o);
+         int plt = parent_.GroupTriangle(g, gt);
          int submesh_face_id = parent_to_submesh_face_ids_[plt];
          if (submesh_face_id != -1)
          {
@@ -600,6 +426,33 @@ void ParSubMesh::FindSharedFacesRanks(Array<int>& rht, Array<int> &rhq)
    // Compute the sum on the root rank and broadcast the result to all ranks.
    stria_comm.Reduce(rht, GroupCommunicator::Sum);
    stria_comm.Bcast<int>(rht, 0);
+
+   GroupCommunicator squad_comm(parent_.gtopo);
+   parent_.GetSharedQuadCommunicator(squad_comm);
+   int nsquad = squad_comm.GroupLDofTable().Size_of_connections();
+   rhq.SetSize(nsquad);
+   rhq = 0;
+
+   for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
+   {
+      MFEM_ASSERT(parent_.gtopo.GetGroupSize(g) == 2
+                  || parent_.GroupNQuadrilaterals(g) == 0,
+                  parent_.gtopo.GetGroupSize(g) << ' ' << parent_.GroupNQuadrilaterals(g));
+      for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
+      {
+         // Group size of a shared face is always 2
+         int plq = parent_.GroupQuadrilateral(g, gq);
+         int submesh_face_id = parent_to_submesh_face_ids_[plq];
+         if (submesh_face_id != -1)
+         {
+            rhq[sq] = 1;
+         }
+      }
+   }
+
+   // Compute the sum on the root rank and broadcast the result to all ranks.
+   squad_comm.Reduce(rhq, GroupCommunicator::Sum);
+   squad_comm.Bcast<int>(rhq, 0);
 }
 
 
@@ -608,6 +461,7 @@ void ParSubMesh::AppendSharedVerticesGroups(ListOfIntegerSets &groups,
 {
    IntegerSet group;
 
+   // g = 0 corresponds to the singleton group of each rank alone.
    for (int g = 1, sv = 0; g < parent_.GetNGroups(); g++)
    {
       const int group_sz = parent_.gtopo.GetGroupSize(g);
@@ -679,8 +533,7 @@ void ParSubMesh::AppendSharedEdgesGroups(ListOfIntegerSets &groups,
 
       for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
       {
-         int ple, o;
-         parent_.GroupEdge(g, ge, ple, o);
+         int ple = parent_.GroupEdge(g, ge);
          int submesh_edge = parent_to_submesh_edge_ids_[ple];
 
          // Reusing the `rhe` array as shared edge to group array.
@@ -729,8 +582,7 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
          const int group_sz = parent_.gtopo.GetGroupSize(g);
          MFEM_ASSERT(group_sz == 2, "internal error");
 
-         int plq, o;
-         parent_.GroupQuadrilateral(g, gq, plq, o);
+         int plq = parent_.GroupQuadrilateral(g, gq);
          int submesh_face_id = parent_to_submesh_face_ids_[plq];
 
          // Reusing the `rhq` array as shared face to group array.
@@ -743,8 +595,8 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
          {
             // shared face is present on this rank and others
 
-            // There can only be two ranks in this group sharing faces. Add
-            // all ranks to a new communication group.
+            // There can only be two ranks in this group sharing faces. Add all
+            // ranks to a new communication group.
             Array<int> &ranks = quad_group;
             ranks.SetSize(0);
             ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[0]));
@@ -770,8 +622,7 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
          const int group_sz = parent_.gtopo.GetGroupSize(g);
          MFEM_ASSERT(group_sz == 2, "internal error");
 
-         int plt, o;
-         parent_.GroupTriangle(g, gt, plt, o);
+         int plt = parent_.GroupTriangle(g, gt);
          int submesh_face_id = parent_to_submesh_face_ids_[plt];
 
          // Reusing the `rht` array as shared face to group array.
@@ -784,8 +635,8 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
          {
             // shared face is present on this rank and others
 
-            // There can only be two ranks in this group sharing faces. Add
-            // all ranks to a new communication group.
+            // There can only be two ranks in this group sharing faces. Add all
+            // ranks to a new communication group.
             Array<int> &ranks = tria_group;
             ranks.SetSize(0);
             ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[0]));
@@ -802,96 +653,46 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
    }
 }
 
-void ParSubMesh::BuildVertexGroup(int ngroups, const Array<int>& rhvtx,
-                                  int& nsverts)
+void BuildGroup(Table &group, int ngroups, const Array<int>& rh, int &ns)
 {
-   group_svert.MakeI(ngroups);
-   for (int i = 0; i < rhvtx.Size(); i++)
+   group.MakeI(ngroups);
+   for (int i = 0; i < rh.Size(); i++)
    {
-      if (rhvtx[i] >= 0)
+      if (rh[i] >= 0)
       {
-         group_svert.AddAColumnInRow(rhvtx[i]);
+         group.AddAColumnInRow(rh[i]);
       }
    }
 
-   group_svert.MakeJ();
-   nsverts = 0;
-   for (int i = 0; i < rhvtx.Size(); i++)
+   group.MakeJ();
+   ns = 0;
+   for (int i = 0; i < rh.Size(); i++)
    {
-      if (rhvtx[i] >= 0)
+      if (rh[i] >= 0)
       {
-         group_svert.AddConnection(rhvtx[i], nsverts++);
+         group.AddConnection(rh[i], ns++);
       }
    }
-   group_svert.ShiftUpI();
+   group.ShiftUpI();
+}
+
+void ParSubMesh::BuildVertexGroup(int ngroups, const Array<int>& rhvtx,
+                                  int& nsverts)
+{
+   BuildGroup(group_svert, ngroups, rhvtx, nsverts);
 }
 
 void ParSubMesh::BuildEdgeGroup(int ngroups, const Array<int>& rhe,
                                 int& nsedges)
 {
-   group_sedge.MakeI(ngroups);
-   for (int i = 0; i < rhe.Size(); i++)
-   {
-      if (rhe[i] >= 0)
-      {
-         group_sedge.AddAColumnInRow(rhe[i]);
-      }
-   }
-
-   group_sedge.MakeJ();
-   nsedges = 0;
-   for (int i = 0; i < rhe.Size(); i++)
-   {
-      if (rhe[i] >= 0)
-      {
-         group_sedge.AddConnection(rhe[i], nsedges++);
-      }
-   }
-   group_sedge.ShiftUpI();
+   BuildGroup(group_sedge, ngroups, rhe, nsedges);
 }
 
 void ParSubMesh::BuildFaceGroup(int ngroups, const Array<int>& rht,
                                 int& nstrias, const Array<int>& rhq, int& nsquads)
 {
-   group_squad.MakeI(ngroups);
-   for (int i = 0; i < rhq.Size(); i++)
-   {
-      if (rhq[i] >= 0)
-      {
-         group_squad.AddAColumnInRow(rhq[i]);
-      }
-   }
-
-   group_squad.MakeJ();
-   nsquads = 0;
-   for (int i = 0; i < rhq.Size(); i++)
-   {
-      if (rhq[i] >= 0)
-      {
-         group_squad.AddConnection(rhq[i], nsquads++);
-      }
-   }
-   group_squad.ShiftUpI();
-
-   group_stria.MakeI(ngroups);
-   for (int i = 0; i < rht.Size(); i++)
-   {
-      if (rht[i] >= 0)
-      {
-         group_stria.AddAColumnInRow(rht[i]);
-      }
-   }
-
-   group_stria.MakeJ();
-   nstrias = 0;
-   for (int i = 0; i < rht.Size(); i++)
-   {
-      if (rht[i] >= 0)
-      {
-         group_stria.AddConnection(rht[i], nstrias++);
-      }
-   }
-   group_stria.ShiftUpI();
+   BuildGroup(group_squad, ngroups, rhq, nsquads);
+   BuildGroup(group_stria, ngroups, rht, nstrias);
 }
 
 void ParSubMesh::BuildSharedVerticesMapping(const int nsverts,
@@ -943,8 +744,8 @@ void ParSubMesh::BuildSharedEdgesMapping(const int sedges_ct,
             int v0 = parent_to_submesh_vertex_ids_[vert[(1-o)/2]];
             int v1 = parent_to_submesh_vertex_ids_[vert[(1+o)/2]];
 
-            // The orienation of the shared edge relative to the local edge
-            // will be determined by whether v0 < v1 or v1 < v0
+            // The orienation of the shared edge relative to the local edge will
+            // be determined by whether v0 < v1 or v1 < v0
             shared_edges.Append(new Segment(v0, v1, 1));
             sedge_ledge.Append(submesh_edge_id);
          }
@@ -960,9 +761,8 @@ void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
    shared_quads.Reserve(nsquads);
    sface_lface.Reserve(nstrias + nsquads);
 
-   // sface_lface should list the triangular shared faces first
-   // followed by the quadrilateral shared faces.
-
+   // sface_lface should list the triangular shared faces first followed by the
+   // quadrilateral shared faces.
    for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
    {
       for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
@@ -1028,7 +828,7 @@ void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
             int v2 = vert[2];
             int v3 = vert[3];
 
-            // See Mesh::GetQuadOrientation for info on interpretting "o"
+            // See Mesh::GetQuadOrientation for info on interpreting "o"
             switch (o)
             {
                case 1:
@@ -1057,10 +857,254 @@ void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
    }
 }
 
+std::unordered_map<int, int>
+ParSubMesh::FindGhostBoundaryElementAttributes() const
+{
+   // Loop over shared faces in the parent mesh, find their attributes if they
+   // exist, and map to local faces in the submesh.
+   std::unordered_map<int,int> lface_boundary_attribute;
+   const auto &face_to_be = parent_.GetFaceToBdrElMap();
+   if (Dim == 3)
+   {
+      GroupCommunicator squad_comm(parent_.gtopo);
+      parent_.GetSharedQuadCommunicator(squad_comm);
+      int nsquad = squad_comm.GroupLDofTable().Size_of_connections();
+
+      GroupCommunicator stria_comm(parent_.gtopo);
+      parent_.GetSharedTriCommunicator(stria_comm);
+      int nstria = stria_comm.GroupLDofTable().Size_of_connections();
+
+      Array<int> stba(nstria), sqba(nsquad);
+      Array<int> parent_ltface(nstria), parent_lqface(nsquad);
+      stba = 0; sqba = 0;
+      parent_ltface = -1; parent_lqface = -1;
+      for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
+      {
+         for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
+         {
+            // Group size of a shared face is always 2
+            int plt = parent_.GroupTriangle(g, gt);
+            auto pbe = face_to_be[plt];
+            if (pbe >= 0)
+            {
+               stba[st] = parent_.GetBdrAttribute(pbe);
+            }
+            parent_ltface[st] = plt;
+         }
+      }
+      for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
+      {
+         for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
+         {
+            // Group size of a shared face is always 2
+            int plq = parent_.GroupQuadrilateral(g, gq);
+            auto pbe = face_to_be[plq];
+            if (pbe >= 0)
+            {
+               sqba[sq] = parent_.GetBdrAttribute(pbe);
+            }
+            parent_lqface[sq] = plq;
+         }
+      }
+#ifdef MFEM_DEBUG
+      auto pre_stba = stba;
+      auto pre_sqba = sqba;
+#endif
+      stria_comm.Reduce(stba, GroupCommunicator::Sum);
+      stria_comm.Bcast<int>(stba, 0);
+      squad_comm.Reduce(sqba, GroupCommunicator::Sum);
+      squad_comm.Bcast<int>(sqba, 0);
+#ifdef MFEM_DEBUG
+      {
+         Array<int> fail_indices;
+         fail_indices.Reserve(stba.Size());
+         for (int i = 0; i < stba.Size(); i++)
+            if (pre_stba[i] != 0 && pre_stba[i] != stba[i])
+            {
+               fail_indices.Append(i);
+            }
+         MFEM_ASSERT(fail_indices.Size() == 0, [&]()
+         {
+            std::stringstream msg;
+            msg << "More than one rank found attribute on shared tri face: ";
+            for (auto x : fail_indices)
+            {
+               msg << x << ' ';
+            }
+            return msg.str();
+         }());
+      }
+
+      {
+         Array<int> fail_indices;
+         fail_indices.Reserve(sqba.Size());
+         for (int i = 0; i < sqba.Size(); i++)
+            if (pre_sqba[i] != 0 && pre_sqba[i] != sqba[i])
+            {
+               fail_indices.Append(i);
+            }
+         MFEM_ASSERT(fail_indices.Size() == 0, [&]()
+         {
+            std::stringstream msg;
+            msg << "More than one rank found attribute on shared quad face: ";
+            for (auto x : fail_indices)
+            {
+               msg << x << ' ';
+            }
+            return msg.str();
+         }());
+      }
+#endif
+      int nghost = 0;
+      for (auto x : stba)
+         if (x > 0) { ++nghost; }
+
+      for (auto x : sqba)
+         if (x > 0) { ++nghost; }
+
+      lface_boundary_attribute.reserve(nghost);
+      for (int i = 0; i < stba.Size(); i++)
+         if (stba[i] > 0)
+         {
+            MFEM_ASSERT(parent_ltface[i] > -1, i);
+            lface_boundary_attribute[parent_ltface[i]] = stba[i];
+         }
+      for (int i = 0; i < sqba.Size(); i++)
+         if (sqba[i] > 0)
+         {
+            MFEM_ASSERT(parent_lqface[i] > -1, i);
+            lface_boundary_attribute[parent_lqface[i]] = sqba[i];
+         }
+   }
+   else if (Dim == 2)
+   {
+      GroupCommunicator sedge_comm(parent_.gtopo);
+      parent_.GetSharedEdgeCommunicator(sedge_comm);
+      int nsedge = sedge_comm.GroupLDofTable().Size_of_connections();
+
+      Array<int> seba(nsedge), parent_ledge(nsedge);
+      seba = 0; parent_ledge = -1;
+      for (int g = 1, se = 0; g < parent_.GetNGroups(); g++)
+      {
+         for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
+         {
+            // Group size of a shared edge is always 2
+            int ple = parent_.GroupEdge(g, ge);
+            auto pbe = face_to_be[ple];
+            if (pbe >= 0)
+            {
+               seba[se] = parent_.GetBdrAttribute(pbe);
+            }
+            parent_ledge[se] = ple;
+         }
+      }
+
+#ifdef MFEM_DEBUG
+      auto pre_seba = seba;
+#endif
+      sedge_comm.Reduce(seba, GroupCommunicator::Sum);
+      sedge_comm.Bcast<int>(seba, 0);
+#ifdef MFEM_DEBUG
+      {
+         Array<int> fail_indices;
+         fail_indices.Reserve(seba.Size());
+         for (int i = 0; i < seba.Size(); i++)
+            if (pre_seba[i] != 0 && pre_seba[i] != seba[i])
+            {
+               fail_indices.Append(i);
+            }
+         MFEM_ASSERT(fail_indices.Size() == 0, [&]()
+         {
+            std::stringstream msg;
+            msg << "More than one rank found attribute on shared edge: ";
+            for (auto x : fail_indices)
+            {
+               msg << x << ' ';
+            }
+            return msg.str();
+         }());
+      }
+#endif
+      int nghost = 0;
+      for (auto x : seba)
+         if (x > 0) { ++nghost; }
+
+      lface_boundary_attribute.reserve(nghost);
+      for (int i = 0; i < seba.Size(); i++)
+         if (seba[i] > 0)
+         {
+            MFEM_ASSERT(parent_ledge[i] > -1, i);
+            lface_boundary_attribute[parent_ledge[i]] = seba[i];
+         }
+   }
+   else if (Dim == 1)
+   {
+      GroupCommunicator svert_comm(parent_.gtopo);
+      parent_.GetSharedVertexCommunicator(svert_comm);
+      int nsvtx = svert_comm.GroupLDofTable().Size_of_connections();
+
+      Array<int> svba(nsvtx), parent_lvtx(nsvtx);
+      svba = 0; parent_lvtx = -1;
+      for (int g = 1, sv = 0; g < parent_.GetNGroups(); g++)
+      {
+         for (int gv = 0; gv < parent_.GroupNVertices(g); gv++, sv++)
+         {
+            // Group size of a shared vertex is always 2
+            int plv = parent_.GroupVertex(g, gv);
+            auto pbe = face_to_be[plv];
+            if (pbe >= 0)
+            {
+               svba[sv] = parent_.GetBdrAttribute(pbe);
+            }
+            parent_lvtx[sv] = plv;
+         }
+      }
+
+#ifdef MFEM_DEBUG
+      auto pre_svba = svba;
+#endif
+      svert_comm.Reduce(svba, GroupCommunicator::Sum);
+      svert_comm.Bcast<int>(svba, 0);
+#ifdef MFEM_DEBUG
+      {
+         Array<int> fail_indices;
+         fail_indices.Reserve(svba.Size());
+         for (int i = 0; i < svba.Size(); i++)
+            if (pre_svba[i] != 0 && pre_svba[i] != svba[i])
+            {
+               fail_indices.Append(i);
+            }
+         MFEM_ASSERT(fail_indices.Size() == 0, [&]()
+         {
+            std::stringstream msg;
+            msg << "More than one rank found attribute on shared vertex: ";
+            for (auto x : fail_indices)
+            {
+               msg << x << ' ';
+            }
+            return msg.str();
+         }());
+      }
+#endif
+      int nghost = 0;
+      for (auto x : svba)
+         if (x > 0) { ++nghost; }
+
+      lface_boundary_attribute.reserve(nghost);
+      for (int i = 0; i < svba.Size(); i++)
+         if (svba[i] > 0)
+         {
+            MFEM_ASSERT(parent_lvtx[i] > -1, i);
+            lface_boundary_attribute[parent_lvtx[i]] = svba[i];
+         }
+   }
+   return lface_boundary_attribute;
+}
+
+
 void ParSubMesh::Transfer(const ParGridFunction &src, ParGridFunction &dst)
 {
-   ParTransferMap map(src, dst);
-   map.Transfer(src, dst);
+   CreateTransferMap(src, dst).Transfer(src, dst);
 }
 
 ParTransferMap ParSubMesh::CreateTransferMap(const ParGridFunction &src,