Merge branch 'master' into mdspan

Update to stale.yml to enable cache, on-demand dispatch, and higher rate limit

.github/stale.yml

@@ -1,61 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 30
-
-# Number of days of inactivity before an Issue or Pull Request with the stale
-# label is closed.  Set to false to disable. If disabled, issues still need to
-# be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale.
-# Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set
-# to `[]` to disable
-exemptLabels:
-  - bug
-  - WIP
-  - ready-for-review
-  - in-review
-  - in-next
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking an issue as stale
-staleLabel: stale
-
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  :warning: This issue or PR has been automatically marked as stale because it has not
-  had any activity in the last month. *If no activity occurs in the next week, it will
-  be automatically closed.* Thank you for your contributions.
-
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-# only: issues
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed

.github/workflows/stale.yml

@@ -0,0 +1,31 @@
+# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
+# For more information, see: https://github.com/actions/stale
+name: Mark stale issues and pull requests
+
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: '0 0 * * *'
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+      actions: write
+
+    steps:
+    - uses: actions/stale@v9
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: ':warning: This issue has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
+        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last month. *If no activity occurs in the next week, it will be automatically closed.* Thank you for your contributions.'
+        days-before-stale: 30
+        days-before-close: 7
+        stale-issue-label: 'stale'
+        stale-pr-label: 'stale'
+        operations-per-run: 500
+        exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
+        exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"

CHANGELOG

@@ -20,6 +20,8 @@ Discretization improvements
 
 - Added support for boundary constraints to the hybridization class.
 
+- Added support for external boundary submeshes with nonconformal mesh adaptation.
+
 Meshing improvements
 --------------------
 - The ExodusII reader now handles pyramid and wedge element types. Mixed meshes
@@ -58,6 +60,9 @@ GPU computing
 
 Miscellaneous
 -------------
+- Added support for SUNDIALS v7. See the section "API changes" for some small
+  changes related to this new version.
+
 - Refactored the `ARKStepSolver` class (ARKODE interface) to use
   `TimeDependentOperator::Mult` only when the associated ODE operator is
   expressed in explicit form (i.e., `TimeDependentOperator::isExplicit()`),
@@ -74,6 +79,18 @@ API changes
 -----------
 - API change: in class GridFunction, 'fec' was renamed to 'fec_owned'.
 
+- API change: support for SUNDIALS v7:
+  * the SUNDIALS types `realtype` and `booleantype` are no longer defined by v7
+    and therefore MFEM now uses the new type names `sunrealtype` and
+    `sunbooleantype`, respectively, which MFEM defines when using SUNDIALS < v6
+    where these types were not defined.
+  * The SUNDIALS macro `SUNLS_SUCCESS` and some other `*_SUCCESS` macros were
+    removed and replaced by `SUN_SUCCESS` in v7, so to avoid tedious checks for
+    SUNDIALS versions, MFEM now defines and uses the constant `SUN_SUCCESS` when
+    using SUNDIALS < v7.
+  * The constants `SUN_PREC_*`, introduced by SUNDIALS v6 are now introduced by
+    MFEM when using SUNDIALS < v6 to avoid tedious version checks.
+
 
 Version 4.7, released on May 7, 2024
 ====================================

CMakeLists.txt

@@ -340,7 +340,10 @@ if (MFEM_USE_SUNDIALS)
   if (MFEM_USE_HIP)
     list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
   endif()
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
+  # The Core component was added in SUNDIALS v7, so we treat it as optional in
+  # order to support older versions.
+  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS}
+    OPTIONAL_COMPONENTS Core)
 endif()
 
 # SuperLU_DIST can only be enabled in parallel

config/cmake/modules/FindSUNDIALS.cmake

@@ -31,4 +31,5 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
   ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
   ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
   ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
-  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)
+  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol
+  ADD_COMPONENT Core "include" sundials/sundials_core.h "lib" sundials_core)

config/defaults.mk

@@ -289,6 +289,13 @@ endif
 ifeq ($(MFEM_USE_HIP),YES)
    SUNDIALS_LIB += -lsundials_nvechip
 endif
+SUNDIALS_CORE_PAT = $(subst\
+ @MFEM_DIR@,$(MFEM_DIR),$(SUNDIALS_DIR))/lib*/libsundials_core.*
+ifeq ($(MFEM_USE_SUNDIALS),YES)
+   ifneq ($(wildcard $(SUNDIALS_CORE_PAT)),)
+      SUNDIALS_LIB += -lsundials_core
+   endif
+endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 

examples/ex38.cpp

@@ -71,7 +71,7 @@ real_t integrand(const Vector& X)
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return 1.;
+         return pow(X(0), 2.);
       case IntegrationType::Surface2D:
          return 3. * pow(X(0), 2.) - pow(X(1), 2.);
       case IntegrationType::Volumetric2D:
@@ -91,7 +91,7 @@ real_t Surface()
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return 1.;
+         return .3025;
       case IntegrationType::Surface2D:
          return 2. * M_PI;
       case IntegrationType::Volumetric2D:
@@ -111,7 +111,7 @@ real_t Volume()
    switch (itype)
    {
       case IntegrationType::Volumetric1D:
-         return .55;
+         return pow(.55, 3.) / 3.;
       case IntegrationType::Surface2D:
          return NAN;
       case IntegrationType::Volumetric2D:
@@ -455,6 +455,8 @@ public:
          add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
       }
    }
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
 /**
@@ -524,6 +526,8 @@ public:
          add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
       }
    }
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
 int main(int argc, char *argv[])
@@ -674,7 +678,7 @@ int main(int argc, char *argv[])
       cout << "Number of div free basis functions: " << nbasis << endl;
       cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
    }
-   cout << scientific << setprecision(2);
+   cout << scientific << setprecision(10);
    cout << "============================================" << endl;
    cout << "Computed value of surface integral: " << surface.Sum() << endl;
    cout << "True value of surface integral:     " << Surface() << endl;

examples/sundials/ex10.cpp

@@ -486,7 +486,11 @@ int main(int argc, char *argv[])
          arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
          arkode->Init(*oper);
          arkode->SetSStolerances(reltol, abstol);
+#if MFEM_SUNDIALS_VERSION < 70100
          ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
+#else
+         ARKodeSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
+#endif
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 15)
          {

examples/sundials/ex10p.cpp

@@ -541,7 +541,11 @@ int main(int argc, char *argv[])
          arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
          arkode->Init(*oper);
          arkode->SetSStolerances(reltol, abstol);
+#if MFEM_SUNDIALS_VERSION < 70100
          ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
+#else
+         ARKodeSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
+#endif
          arkode->SetMaxStep(dt);
          if (ode_solver_type == 15)
          {

examples/sundials/ex16.cpp

@@ -447,7 +447,7 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &fes,
                                        const Vector &u,
                                        const Type &ode_expression_type)
    : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
-     fespace(fes), alpha(alpha), kappa(kappa), M(&fespace), z(height)
+     fespace(fes), M(&fespace), alpha(alpha), kappa(kappa), z(height)
 {
    // specify a relative tolerance for all solves with MFEM integrators
    const real_t rel_tol = 1e-8;
@@ -522,7 +522,7 @@ int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
    T = std::unique_ptr<SparseMatrix>(Add(1.0, Mmat, gam, Kmat));
    T_solver.SetOperator(*T);
    *jcur = SUNTRUE; // this should eventually only be set true if K(u) is used
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 
 int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
@@ -544,7 +544,7 @@ int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
    }
    if (T_solver.GetConverged())
    {
-      return SUNLS_SUCCESS;
+      return SUN_SUCCESS;
    }
    else
    {
@@ -555,7 +555,7 @@ int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
 int ConductionOperator::SUNMassSetup()
 {
    // Do nothing b/c mass solver was setup in constructor.
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 
 int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
@@ -565,7 +565,7 @@ int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
    M_solver.Mult(b, x);
    if (M_solver.GetConverged())
    {
-      return SUNLS_SUCCESS;
+      return SUN_SUCCESS;
    }
    else
    {
@@ -577,6 +577,6 @@ int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
 {
    // Compute M x.
    Mmat.Mult(x, v);
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 

examples/sundials/ex16p.cpp

@@ -499,7 +499,7 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &fes,
                                        const Vector &u,
                                        const Type &ode_expression_type)
    : TimeDependentOperator(fes.GetTrueVSize(), 0.0, ode_expression_type),
-     fespace(fes), alpha(alpha), kappa(kappa), M(&fespace),
+     fespace(fes), M(&fespace), alpha(alpha), kappa(kappa),
      M_solver(fes.GetComm()), T_solver(fes.GetComm()), z(height)
 {
    // specify a relative tolerance for all solves with MFEM integrators
@@ -576,7 +576,7 @@ int ConductionOperator::SUNImplicitSetup(const Vector &u, const Vector &fu,
    T = std::unique_ptr<HypreParMatrix>(Add(1.0, Mmat, gam, Kmat));
    T_solver.SetOperator(*T);
    *jcur = SUNTRUE; // this should eventually only be set true if K(u) is used
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 
 int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
@@ -598,7 +598,7 @@ int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
    }
    if (T_solver.GetConverged())
    {
-      return SUNLS_SUCCESS;
+      return SUN_SUCCESS;
    }
    else
    {
@@ -609,7 +609,7 @@ int ConductionOperator::SUNImplicitSolve(const Vector &r, Vector &dk,
 int ConductionOperator::SUNMassSetup()
 {
    // Do nothing b/c mass solver was setup in constructor.
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }
 
 int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
@@ -619,7 +619,7 @@ int ConductionOperator::SUNMassSolve(const Vector &b, Vector &x, real_t tol)
    M_solver.Mult(b, x);
    if (M_solver.GetConverged())
    {
-      return SUNLS_SUCCESS;
+      return SUN_SUCCESS;
    }
    else
    {
@@ -631,5 +631,5 @@ int ConductionOperator::SUNMassMult(const Vector &x, Vector &v)
 {
    // Compute M x.
    Mmat.Mult(x, v);
-   return SUNLS_SUCCESS;
+   return SUN_SUCCESS;
 }

fem/CMakeLists.txt

@@ -101,6 +101,7 @@ set(SRCS
   lor/lor_ads.cpp
   lor/lor_ams.cpp
   lor/lor_batched.cpp
+  mdgridfunc.hpp
   multigrid.cpp
   nonlinearform.cpp
   nonlinearform_ext.cpp

fem/coefficient.hpp

@@ -798,6 +798,12 @@ public:
    /// Sets coefficient in the vector.
    void Set(int i, Coefficient *c, bool own=true);
 
+   /// Set ownership of the i'th coefficient
+   void SetOwnership(int i, bool own) { ownCoeff[i] = own; }
+
+   /// Get ownership of the i'th coefficient
+   bool GetOwnership(int i) const { return ownCoeff[i]; }
+
    /// Evaluates i'th component of the vector of coefficients and returns the
    /// value.
    real_t Eval(int i, ElementTransformation &T, const IntegrationPoint &ip)
@@ -1320,6 +1326,12 @@ public:
        can be overridden with the @a own parameter. */
    void Set(int i, int j, Coefficient * c, bool own=true);
 
+   /// Set ownership of the coefficient at (i,j) in the matrix
+   void SetOwnership(int i, int j, bool own) { ownCoeff[i*width+j] = own; }
+
+   /// Get ownership of the coefficient at (i,j) in the matrix
+   bool GetOwnership(int i, int j) const { return ownCoeff[i*width+j]; }
+
    using MatrixCoefficient::Eval;
 
    /// Evaluate coefficient located at (i,j) in the matrix using integration
@@ -1360,6 +1372,12 @@ public:
        can be overridden with the @a own parameter. */
    void Set(int i, VectorCoefficient * c, bool own=true);
 
+   /// Set ownership of the i'th coefficient
+   void SetOwnership(int i, bool own) { ownCoeff[i] = own; }
+
+   /// Get ownership of the i'th coefficient
+   bool GetOwnership(int i) const { return ownCoeff[i]; }
+
    using MatrixCoefficient::Eval;
 
    /// Evaluate coefficient located at the i-th row of the matrix using integration

fem/intrules_cut.cpp

@@ -341,6 +341,7 @@ void MomentFittingIntRules::ComputeFaceWeights(ElementTransformation& Tr)
          local_mesh.GetElementTransformation(0, &faceTrafo);
 
          // The 3D face integrals are computed as 2D volumetric integrals.
+         // The 2D face integrals are computed as 1D volumetric integrals.
          MomentFittingIntRules FaceRules(Order, *LvlSet, lsOrder);
          IntegrationRule FaceRule;
          FaceRules.GetVolumeIntegrationRule(faceTrafo, FaceRule);
@@ -420,8 +421,56 @@ void MomentFittingIntRules::ComputeSurfaceWeights1D(ElementTransformation& Tr)
    }
 }
 
-void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr,
-                                                   const IntegrationRule* sir)
+double bisect(ElementTransformation &Tr, Coefficient *LvlSet)
+{
+   IntegrationPoint intp;
+
+   IntegrationPoint ip0;
+   ip0.x = 0.;
+   IntegrationPoint ip1;
+   ip1.x = 1.;
+   Tr.SetIntPoint(&ip0);
+   if (LvlSet->Eval(Tr, ip0) * LvlSet->Eval(Tr, ip1) < 0.)
+   {
+      IntegrationPoint ip2;
+      ip2.x = .5;
+      while (LvlSet->Eval(Tr, ip2) > 1e-12
+             || LvlSet->Eval(Tr, ip2) < -1e-12)
+      {
+         if (LvlSet->Eval(Tr, ip0) * LvlSet->Eval(Tr, ip2) < 0.)
+         {
+            ip1.x = ip2.x;
+         }
+         else
+         {
+            ip0.x = ip2.x;
+         }
+
+         ip2.x = (ip1.x + ip0.x) / 2.;
+      }
+      intp.x = ip2.x;
+      intp.weight = 1. / Tr.Weight();
+   }
+   else if (LvlSet->Eval(Tr, ip0) > 0. && LvlSet->Eval(Tr, ip1) <= 1e-12)
+   {
+      intp.x = 1.;
+      intp.weight = 1. / Tr.Weight();
+   }
+   else if (LvlSet->Eval(Tr, ip1) > 0. && LvlSet->Eval(Tr, ip0) <= 1e-12)
+   {
+      intp.x = 0.;
+      intp.weight = 1. / Tr.Weight();
+   }
+   else
+   {
+      intp.x = .5;
+      intp.weight = 0.;
+   }
+
+   return intp.x;
+}
+
+void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr)
 {
    IntegrationRules irs(0, Quadrature1D::GaussLegendre);
    IntegrationRule ir2 = irs.Get(Geometry::SEGMENT, ir.GetOrder());
@@ -437,7 +486,7 @@ void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr,
       real_t length;
       if (LvlSet->Eval(Tr, ip0) > 0.)
       {
-         length = sir->IntPoint(0).x;
+         length = bisect(Tr, LvlSet);
          for (int ip = 0; ip < ir.GetNPoints(); ip++)
          {
             IntegrationPoint &intp = ir.IntPoint(ip);
@@ -447,11 +496,11 @@ void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr,
       }
       else
       {
-         length = 1. - sir->IntPoint(0).x;
+         length = 1. - bisect(Tr, LvlSet);
          for (int ip = 0; ip < ir.GetNPoints(); ip++)
          {
             IntegrationPoint &intp = ir.IntPoint(ip);
-            intp.x = sir->IntPoint(ip).x + ir2.IntPoint(ip).x * length;
+            intp.x = bisect(Tr, LvlSet) + ir2.IntPoint(ip).x * length;
             intp.weight = ir2.IntPoint(ip).weight * length;
          }
       }
@@ -1657,26 +1706,29 @@ void MomentFittingIntRules::GetVolumeIntegrationRule(ElementTransformation& Tr,
    }
 
    IntegrationRule SIR;
-   if (sir == NULL)
-   {
-      Order++;
-      GetSurfaceIntegrationRule(Tr, SIR);
-      Order--;
-   }
-   else if ((sir->GetOrder() - 1) != ir.GetOrder())
-   {
-      Order++;
-      GetSurfaceIntegrationRule(Tr, SIR);
-      Order--;
-   }
-   else
-   {
-      SIR = *sir;
-   }
 
    if (Tr.GetDimension() == 1)
    {
-      ComputeVolumeWeights1D(Tr, &SIR);
+      Clear();
+      InitVolume(Order, *LvlSet, lsOrder, Tr);
+   }
+   else if (sir == NULL)
+   {
+      Order++;
+      GetSurfaceIntegrationRule(Tr, SIR);
+      Order--;
+   }
+   else if (sir->GetOrder() - 1 != ir.GetOrder())
+   {
+      Order++;
+      GetSurfaceIntegrationRule(Tr, SIR);
+      Order--;
+   }
+   else { SIR = *sir; }
+
+   if (Tr.GetDimension() == 1)
+   {
+      ComputeVolumeWeights1D(Tr);
    }
    else if (Tr.GetDimension() == 2)
    {

fem/intrules_cut.hpp

@@ -565,10 +565,8 @@ protected:
     rule.
 
     @param [in] Tr ElementTransformation of the current element
-    @param [in] sir corresponding IntegrationRule on surface
     */
-   void ComputeVolumeWeights1D(ElementTransformation& Tr,
-                               const IntegrationRule* sir);
+   void ComputeVolumeWeights1D(ElementTransformation& Tr);
 
    /**
     @brief Compute 2D quadrature weights

fem/mdgridfunc.hpp

@@ -0,0 +1,154 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_MDGRIDFUNC
+#define MFEM_MDGRIDFUNC
+
+#include "../config/config.hpp"
+
+#include "fem/gridfunc.hpp"
+#include "general/mdspan.hpp"
+
+namespace mfem
+{
+
+template<int N, class Layout = MDLayoutLeft<N>>
+class MDGridFunction : public MDSpan<GridFunction, N, Layout>
+{
+   using base_t = MDSpan<GridFunction, N, Layout>;
+   using base_t::Nd;
+   using base_t::Sd;
+   using GridFunction::data;
+
+public:
+
+   /**
+    * @brief MDGridFunction default constructor (recursion)
+    */
+   MDGridFunction(): base_t() { }
+
+   /**
+    * @brief MDGridFunction recursion constructor
+    * @param[in] fes Finite element space to use
+    * @param[in] args Rest of dimension indices
+    */
+   template <typename... Ts>
+   MDGridFunction(FiniteElementSpace *fes, Ts... args): MDGridFunction(args...)
+   {
+      SetSpace(fes);
+      MFEM_VERIFY(fes->GetVDim() == 1,
+                  "Only FiniteElementSpace with vdim of 1 are supported");
+      base_t::Setup(fes->GetNDofs(), args...);
+   }
+
+   /**
+    * @brief MDGridFunction recursion constructor
+    * @param[in] dim Dimension indice
+    * @param[in] args Rest of dimension indices or finite element space to use
+    */
+   template <typename... Ts>
+   MDGridFunction(int dim, Ts... args): MDGridFunction(args...)
+   {
+      base_t::Setup(dim, args...);
+   }
+
+   /// Move constructor not supported
+   MDGridFunction(MDGridFunction&&) = delete;
+
+   /// Copy constructor not supported
+   MDGridFunction(const MDGridFunction&) = delete;
+
+   /// Move assignment not supported
+   MDGridFunction& operator=(MDGridFunction&&) = delete;
+
+   /// Copy assignment not supported
+   MDGridFunction& operator=(const MDGridFunction&) = delete;
+
+   /**
+    * @brief Returns the specific GridFunction from dimension indices
+    * @param[out] gf Returned GridFunction
+    * @param[in] args Rest of dimension indices
+    */
+   template <int n = 1, typename... Ts>
+   void GetScalarGridFunction(GridFunction &gf, Ts... args) const
+   {
+      FiniteElementSpace *fes = GridFunction::fes;
+      MFEM_VERIFY(fes->GetNDofs() == Nd[n-1], "Error in dofs size!");
+      gf.SetSpace(fes);
+      for (int s = 0; s < Nd[n-1]; s++)
+      {
+         gf[s] = data[get_vdofs_offset +
+                      MDOffset<n,N,int,Ts...>::offset(Sd, s, args...)];
+      }
+      get_vdofs_offset = 0; // re-init for next calls
+   }
+
+   /**
+    * @brief Returns the specific GridFunction from dimension indices
+    * @param[in] dim Dimension indice
+    * @param args Rest of dimension indices or GridFunction to be returned
+    */
+   template <int n = 1, typename... Ts>
+   void GetScalarGridFunction(int dim, Ts&&... args) const
+   {
+      get_vdofs_offset += dim * Sd[n-1];
+      MDGridFunction::GetScalarGridFunction<n+1>(std::forward<Ts>(args)...);
+   }
+
+   /**
+    * @brief Sets the given GridFunction at the specific dimension indices
+    * @param[in] gf GridFunction to set
+    * @param[in] args Rest of dimension indices
+    */
+   template <int n = 1, typename... Ts>
+   void SetScalarGridFunction(const GridFunction &gf, Ts... args)
+   {
+      MFEM_VERIFY(GridFunction::fes->GetNDofs() == Nd[n-1], "Error in dofs size!");
+      for (int s = 0; s < Nd[n-1]; s++)
+      {
+         data[get_vdofs_offset +
+              MDOffset<n,N,int,Ts...>::offset(Sd, s, args...)] = gf[s];
+      }
+      get_vdofs_offset = 0; // re-init for next calls
+   }
+
+   /**
+    * @brief Sets the given GridFunction at the specific dimension indices
+    * @param[in] dim Dimension indice
+    * @param args Rest of dimension indices or given GridFunction to be used
+    */
+   template <int n = 1, typename... Ts>
+   void SetScalarGridFunction(int dim, Ts... args)
+   {
+      get_vdofs_offset += dim * Sd[n-1];
+      MDGridFunction::SetScalarGridFunction<n+1>(args...);
+   }
+
+   using GridFunction::Read;
+   using GridFunction::Write;
+   using GridFunction::ReadWrite;
+   using GridFunction::HostRead;
+   using GridFunction::HostWrite;
+   using GridFunction::HostReadWrite;
+
+   using GridFunction::GetData;
+   using GridFunction::SetData;
+   using GridFunction::SetSpace;
+
+   using Vector::operator=;
+
+private:
+   mutable int get_vdofs_offset = 0;
+};
+
+} // namespace mfem
+
+#endif // MFEM_MDGRIDFUNC

general/CMakeLists.txt

@@ -47,6 +47,8 @@ list(APPEND HDRS
   zstr.hpp
   hash.hpp
   isockstream.hpp
+  mdarray.hpp
+  mdspan.hpp
   kdtree.hpp
   mem_alloc.hpp
   mem_manager.hpp

general/array.hpp

@@ -45,6 +45,8 @@ template <class T>
 class Array
 {
 protected:
+   template<typename mfem_type, int N, typename L> friend class MDSpan;
+
    /// Pointer to data
    Memory<T> data;
    /// Size of the array
@@ -52,10 +54,7 @@ protected:
 
    inline void GrowSize(int minsize);
 
-   static inline void TypeAssert()
-   {
-      static_assert(std::is_trivial<T>::value, "type T must be trivial");
-   }
+   static_assert(std::is_trivial<T>::value, "type T must be trivial");
 
 public:
    friend void Swap<T>(Array<T> &, Array<T> &);
@@ -95,11 +94,26 @@ public:
    template <typename CT, int N>
    explicit inline Array(const CT (&values)[N]);
 
+   /**
+    * @brief Construct a new Array object from an initializer list.
+    *
+    * @param init_list List of entities to construct from.
+    */
+   Array(const std::initializer_list<T> &init_list)
+      : Array(static_cast<int>(init_list.size()))
+   {
+      auto * it = GetData();
+      for (auto value : init_list)
+      {
+         *it++ = value;
+      }
+   }
+
    /// Move constructor ("steals" data from 'src')
    inline Array(Array<T> &&src) { Swap(src, *this); }
 
    /// Destructor
-   inline ~Array() { TypeAssert(); data.Delete(); }
+   inline ~Array() { data.Delete(); }
 
    /// Assignment operator: deep copy from 'src'.
    Array<T> &operator=(const Array<T> &src) { src.Copy(*this); return *this; }
@@ -204,6 +218,8 @@ public:
    /// Delete the whole array.
    inline void DeleteAll();
 
+   /// Reduces the capacity of the array to exactly match the current size.
+   inline void ShrinkToFit();
 
    ///  Create a copy of the internal array to the provided @a copy.
    inline void Copy(Array &copy) const;
@@ -221,6 +237,18 @@ public:
    /// Make this Array a reference to 'master'.
    inline void MakeRef(const Array &master);
 
+   /**
+    * @brief Permute the array using the provided indices. Sorts the indices
+    * variable in the process, thereby destroying the permutation. The rvalue
+    * reference is to be used when this destruction is allowed, whilst the const
+    * reference preserves at the cost of duplication.
+    *
+    * @param indices The indices of the ordering. data[i] = data[indices[i]].
+    */
+   template <typename I>
+   inline void Permute(I &&indices);
+   template <typename I>
+   inline void Permute(const I &indices) { Permute(I(indices)); }
 
    /// Copy sub array starting from @a offset out to the provided @a sa.
    inline void GetSubArray(int offset, int sa_size, Array<T> &sa) const;
@@ -275,6 +303,9 @@ public:
    /// Return 1 if the array is sorted from lowest to highest.  Otherwise return 0.
    int IsSorted() const;
 
+   /// Does the Array have Size zero.
+   bool IsEmpty() const { return Size() == 0; }
+
    /// Fill the entries of the array with the cumulative sum of the entries.
    void PartialSum();
 
@@ -492,6 +523,8 @@ public:
    BlockArray(int block_size = 16*1024);
    BlockArray(const BlockArray<T> &other); // deep copy
    BlockArray& operator=(const BlockArray&) = delete; // not supported
+   BlockArray(BlockArray<T> &&other) = default;
+   BlockArray& operator=(BlockArray<T> &&other) = default;
    ~BlockArray() { Destroy(); }
 
    /// Allocate and construct a new item in the array, return its index.
@@ -613,6 +646,8 @@ public:
 
    iterator begin() { return size ? iterator(this) : iterator(true); }
    iterator end() { return iterator(); }
+   const_iterator begin() const { return cbegin(); }
+   const_iterator end() const { return cend(); }
 
    const_iterator cbegin() const
    { return size ? const_iterator(this) : const_iterator(true); }
@@ -685,6 +720,35 @@ inline void Array<T>::GrowSize(int minsize)
    data = p;
 }
 
+template <typename T>
+inline void Array<T>::ShrinkToFit()
+{
+   if (Capacity() == size) { return; }
+   Memory<T> p(size, data.GetMemoryType());
+   p.CopyFrom(data, size);
+   p.UseDevice(data.UseDevice());
+   data.Delete();
+   data = p;
+}
+
+template <typename T>
+template <typename I>
+inline void Array<T>::Permute(I &&indices)
+{
+   for (int i = 0; i < size; i++)
+   {
+      auto current = i;
+      while (i != indices[current])
+      {
+         auto next = indices[current];
+         std::swap(data[current], data[next]);
+         indices[current] = current;
+         current = next;
+      }
+      indices[current] = current;
+   }
+}
+
 template <typename T> template <typename CT>
 inline Array<T> &Array<T>::operator=(const Array<CT> &src)
 {

general/mdarray.hpp

@@ -0,0 +1,70 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_MDARRAY
+#define MFEM_MDARRAY
+
+#include "../config/config.hpp"
+
+#include "array.hpp"
+#include "mdspan.hpp"
+
+namespace mfem
+{
+
+template<typename T, int N, typename Layout = MDLayoutLeft<N>>
+struct MDArray : public MDSpan<Array<T>, N, Layout>
+{
+   using base_t = MDSpan<Array<T>, N, Layout>;
+
+   /**
+    * @brief MDArray default constructor (recursion)
+    */
+   MDArray(): base_t() { }
+
+   /**
+    * @brief MDArray recursion constructor
+    * @param[in] n Dimension indice
+    * @param[in] args Rest of dimension indices
+    */
+   template <typename... Ts>
+   MDArray(int n, Ts... args): MDArray(args...) { base_t::Setup(n, args...); }
+
+   /// Move constructor not supported
+   MDArray(MDArray&&) = delete;
+
+   /// Copy constructor not supported
+   MDArray(const MDArray&) = delete;
+
+   /// Move assignment not supported
+   MDArray& operator=(MDArray&&) = delete;
+
+   /// Copy assignment not supported
+   MDArray& operator=(const MDArray&) = delete;
+
+   using Array<T>::Read;
+   using Array<T>::Write;
+   using Array<T>::ReadWrite;
+   using Array<T>::HostRead;
+   using Array<T>::HostWrite;
+   using Array<T>::HostReadWrite;
+
+   using Array<T>::Assign;
+   using Array<T>::Print;
+
+   using Array<T>::GetData;
+
+   using Array<T>::operator=;
+};
+
+} // namespace mfem
+
+#endif // MFEM_MDARRAY

general/mdspan.hpp

@@ -0,0 +1,417 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_MDSPAN_HPP
+#define MFEM_MDSPAN_HPP
+
+#include <list>
+#include <array>
+#include <vector>
+#include <utility>
+#include <type_traits>
+
+#include "device.hpp"
+#include "backends.hpp"
+
+namespace mfem
+{
+
+namespace internal // experimental helper functions for mfem::MDLayout
+{
+
+// md_sequence represents a compile-time sequence of integers
+template <typename T, T... args> struct md_sequence { };
+
+template <typename T, int N, bool left> struct make_md_sequence;
+
+// make_sequence, specialized for left (default) and right layout
+template <typename T, int N, bool left = true>
+using make_sequence = typename make_md_sequence<T, N, left>::type;
+
+} // namespace internal
+
+/// @brief The MDOffset class computes the multi-dimensional offsets
+template <int n, int N, typename T, typename... Ts>
+struct MDOffset
+{
+   static MFEM_HOST_DEVICE inline
+   T offset(const int (&Sd)[N], T nd, Ts... args)
+   { return nd * Sd[n-1] + MDOffset<n+1, N, Ts...>::offset(Sd, args...); }
+};
+
+template <int N, typename T, typename... Ts>
+struct MDOffset<N, N, T, Ts...>
+{
+   static MFEM_HOST_DEVICE inline
+   T offset(const int (&Sd)[N], T nd) { return nd * Sd[N-1]; }
+};
+
+/// @brief The MDTensor class holds the pointer and strides for each dimension
+template<int N, typename T> class MDTensor
+{
+   T *ptr;
+   int Sd[N];
+
+public:
+   /// Default constructor
+   MDTensor() = delete;
+
+   /// Copy constructor (default)
+   MDTensor(const MDTensor&) = default;
+
+   /// Copy assignment (default)
+   MDTensor& operator=(const MDTensor&) = default;
+
+   /// Constructor to initialize a tensor from a pointer and strides
+   template <typename... Args> MFEM_HOST_DEVICE
+   MDTensor(T *ptr, const int (&sd)[N]): ptr(ptr)
+   { for (int i = 0; i < N; ++i) { Sd[i] = sd[i]; } }
+
+   /// Accessor for the data
+   template <typename... Ts> MFEM_HOST_DEVICE inline
+   T& operator()(Ts... args) { return ptr[Offset(args...)]; }
+
+   /// Const accessor for the data
+   template <typename... Ts> MFEM_HOST_DEVICE inline
+   T& operator()(Ts... args) const { return ptr[Offset(args...)]; }
+
+   /// Offset computation
+   template <typename... Ts> MFEM_HOST_DEVICE inline
+   int Offset(Ts... args) const
+   {
+      static_assert(sizeof...(args) == N, "Wrong number of dimensions");
+      return MDOffset<1, N, Ts...>::offset(Sd, args...);
+   }
+};
+
+/// \brief The MDLayout class, defaulted to a column-major (left) ordering
+template<int N, bool left = true> struct MDLayout
+{
+   /// Create a layout with the internal::md_sequence
+   template <int... args>
+   static constexpr auto Make(internal::md_sequence<int, args...>)
+   -> std::array<int, sizeof...(args)> { return {(static_cast<int>(args))...}; }
+
+   /// Array holding the layout permutation
+   using perm_type = std::array<int, N>;
+   perm_type perm = Make(internal::make_sequence<int, N, left> {});
+
+   /// Default constructor
+   MDLayout() = default;
+
+   /// Copy constructor (default)
+   MDLayout(const MDLayout&) = default;
+
+   /// Copy assignment (default)
+   MDLayout& operator=(const MDLayout&) = default;
+
+   /// Constructor to initialize a layout from an array of indices
+   template <typename... Ts>
+   MDLayout(int n, Ts... args) noexcept: MDLayout(args...)
+   {
+      constexpr int k = N - sizeof...(args) - 1;
+      static_assert(0 <= k && k < N, "Index out of bounds!");
+      perm[k] = n;
+   }
+
+   /// Access layout entries using operator()
+   inline int operator()(int i) const
+   { return Assert(i), perm[static_cast<typename perm_type::size_type>(i)]; }
+
+   /// Access layout entries using operator[]
+   inline int operator[](int i) const
+   { return Assert(i), perm[static_cast<typename perm_type::size_type>(i)]; }
+
+   /// Asserts the given index is valid (only in MFEM_DEBUG)
+   inline void Assert(const int k) const
+   {
+      MFEM_CONTRACT_VAR(k);
+      MFEM_ASSERT(0 <= k && k < N, "Index should be in [0," << (N-1) << "]");
+   }
+};
+
+/// Left (Column-major (Fortran)) and Right (Row-major (C/C++)) layouts
+template<int N> using MDLayoutLeft = MDLayout<N, true>;
+template<int N> using MDLayoutRight = MDLayout<N, false>;
+
+/// \brief The MDSpan base class is a generic non-owning mfem_type's view
+/// that reinterprets it as a multidimensional type.
+template<typename mfem_type, int N, class layout_type = MDLayoutLeft<N>>
+class MDSpan : protected mfem_type
+{
+protected:
+   using T = typename std::remove_pointer<decltype(mfem_type::data.h_ptr)>::type;
+
+   int Nd[N], Sd[N]; // dimension sizes and strides, once the layout is set
+   layout_type layout; // stored layout, useful for reshapes
+
+   /// Set the dimensions (Nd) and strides (Sd) during contruction.
+   /// When all the arguments have been processed, SetSize is called on the
+   /// mfem_type with Device::GetMemoryType() as memory type and SetLayout is
+   /// called using the layout.
+   template <typename... Ts> void Setup(int dim, Ts... args)
+   {
+      constexpr int k = N - sizeof...(args) - 1;
+      Sd[k] = Nd[k] = dim;
+      if (k > 0) { return; }
+      int psize = 1;
+      for (int i = 0; i < N; i++) { psize *= Nd[i]; }
+      mfem_type::SetSize(static_cast<int>(psize), Device::GetMemoryType());
+      SetLayout(layout);
+   }
+
+public:
+
+   /// Default constructor (recursion)
+   MDSpan() noexcept: mfem_type() { }
+
+   /// Recursion constructor
+   template <typename... Ts>
+   MDSpan(int n, Ts... args): MDSpan(args...) { Setup(n, args...); }
+
+   /// Move constructor (delete)
+   MDSpan(MDSpan&&) = delete;
+
+   /// Copy constructor (delete)
+   MDSpan(const MDSpan&) = delete;
+
+   /// Move assignment (delete)
+   MDSpan& operator=(MDSpan&&) = delete;
+
+   /// Copy assignment (delete)
+   MDSpan& operator=(const MDSpan&) = delete;
+
+   /// Return the ith dimension
+   int Extent(int i) const { return Nd[i]; }
+
+   /// Return the size of the span.
+   int Size() const { return mfem_type::Size(); }
+
+   /// Store and use the given layout to update the strides
+   template<typename Layout> void SetLayout(const Layout &l)
+   {
+      layout = l;
+      Sd[l[0]] = 1;
+      for (int i = 1; i < N; i++) { Sd[l[i]] = Nd[l[i-1]] * Sd[l[i-1]]; }
+   }
+
+   /// Variadic resize the mfem_type
+   template <typename... Ts> inline void SetSize(int size, Ts... args)
+   {
+      constexpr int k = N - sizeof...(args) - 1;
+      Sd[k] = Nd[k] = size;
+      const int msize = mfem_type::Size();
+      MFEM_VERIFY(size > 0, "Size should be positive!");
+      mfem_type::SetSize(msize > 0 ? msize*size : size, Device::GetMemoryType());
+      MDSpan::SetSize(args...);
+   }
+
+   /// Variadic terminal case of the mfem_type resize
+   inline void SetSize(int size)
+   {
+      Sd[N-1] = Nd[N-1] = size;
+      const int msize = mfem_type::Size();
+      MFEM_VERIFY(size > 0, "Size should be positive!");
+      mfem_type::SetSize(msize > 0 ? msize*size : size, Device::GetMemoryType());
+      SetLayout(layout);
+   }
+
+   /// Access mfem_type data entries using operator()
+   template <typename... Ts> inline
+   T& operator()(Ts... args) { return mfem_type::data[Offset(args...)]; }
+
+   /// Const access mfem_type data entries using operator()
+   template <typename... Ts> inline const T& operator()(Ts... args) const
+   {
+      return mfem_type::data[Offset(args...)];
+   }
+
+   /// Offset computation
+   template <typename... Ts> inline int Offset(Ts... args) const
+   {
+      static_assert(sizeof...(args) == N, "Wrong number of dimensions");
+      return MDOffset<1,N,Ts...>::offset(Sd, args...);
+   }
+
+   /// Shortcut for mfem::Read(mfem_type::data, mfem_type::size, on_dev)
+   /// and return an MDTensor with the MDSpan's pointer and strides
+   const MDTensor<N,const T> MDRead(bool on_dev = true) const
+   {
+      const T *ptr = mfem::Read(mfem_type::data, mfem_type::size, on_dev);
+      return MDTensor<N,const T>(ptr, Sd);
+   }
+
+   /// Shortcut for mfem::Read(mfem_type::data, mfem_type::size, false)
+   /// and return an MDTensor with the MDSpan's pointer and strides
+   const MDTensor<N,const T> MDHostRead() const
+   {
+      const T *ptr = mfem::Read(mfem_type::data, mfem_type::size, false);
+      return MDTensor<N,const T>(ptr, Sd);
+   }
+
+   /// Shortcut for mfem::Write(mfem_type::data, mfem_type::size, on_dev)
+   /// and return an MDTensor with the MDSpan's pointer and strides
+   MDTensor<N,T> MDWrite(bool on_dev = true)
+   {
+      T *ptr = mfem::Write(mfem_type::data, mfem_type::size, on_dev);
+      return MDTensor<N,T>(ptr, Sd);
+   }
+
+   /// Shortcut for mfem::Write(mfem_type::data, mfem_type::size, false)
+   /// and return an MDTensor with the MDSpan's pointer and strides
+   MDTensor<N,T> MDHostWrite()
+   {
+      T *ptr = mfem::Write(mfem_type::data, mfem_type::size, false);
+      return MDTensor<N,T>(ptr, Sd);
+   }
+
+   /// Shortcut for mfem::ReadWrite(mfem_type::data, mfem_type::size, on_dev)
+   /// and return an MDTensor with the MDSpan's pointer and strides
+   MDTensor<N,T> MDReadWrite(bool on_dev = true)
+   {
+      T *ptr = mfem::ReadWrite(mfem_type::data, mfem_type::size, on_dev);
+      return MDTensor<N,T>(ptr, Sd);
+   }
+
+   /// Shortcut for mfem::ReadWrite(mfem_type::data, mfem_type::size, false)
+   /// and return an MDTensor with the MDSpan's pointer and strides
+   MDTensor<N,T> MDHostReadWrite()
+   {
+      T *ptr = mfem::ReadWrite(mfem_type::data, mfem_type::size, false);
+      return MDTensor<N,T>(ptr, Sd);
+   }
+
+   /// The MDReshape function allows to reshape the multi-dimentional view
+   /// into a new multi-dimentional one, by the use of std::array blocks.
+   /// For example, if 'this' has three dimensions {N1, N2, N3}, it could handle
+   /// this->MDReshape<4>(ptr, N1, std::array<int,2> {2, N2/2}, N3);
+
+   // Parameter R could be omitted with c++14 standard's deduced return types
+
+   // first method with given data pointer and rest of arguments
+   template <int R, int m = 0, int M = 0, typename... Ts>
+   inline auto MDReshape(T *ptr, Ts&&... args) -> MDTensor<R,T>
+   {
+      rNd.clear();
+      reshape_ptr = ptr;
+      reshape_offset = 1, reshape_shifts[0] = reshape_shifts[1] = 0;
+      return MDReshape<R,m,M>(std::forward<Ts>(args)...);
+   }
+
+   // variadic method, where a new block of reshape is given in argument
+   template <int R, int m = 0, int M = 0, size_t P, typename... Ts>
+   inline auto MDReshape(std::array<int,P> list, Ts&&... args) -> MDTensor<R,T>
+   {
+      reshape_shifts[0] = layout.perm[m]; // store layout shift begin
+      int shifted_layout = reshape_shifts[1] + layout.perm[m];
+      for (int dim: list)
+      {
+         rNd.push_back(dim);
+         rLt[m].push_back(sub_layout_pair{shifted_layout,-1});
+         shifted_layout += 1; // default left layout
+      }
+      reshape_shifts[1] += P-1; // update end
+      return MDReshape<R,m+1,M+P>(std::forward<Ts>(args)...);
+   }
+
+   // variadic method, where a new dimension of reshape is given
+   template <int R, int m = 0, int M = 0, typename... Ts>
+   inline auto MDReshape(int dim, Ts&&... args) -> MDTensor<R,T>
+   {
+      rNd.push_back(dim);
+      const int shift =
+         reshape_shifts[0] < layout.perm[m] ? reshape_shifts[1] : 0;
+      rLt[m].push_back(sub_layout_pair{layout.perm[m] + shift,-1});
+      return MDReshape<R,m+1,M+1>(std::forward<Ts>(args)...);
+   }
+
+   // terminal case which returns the resulting MDTensor
+   template <int R, int m = 0, int M = 0>
+   inline MDTensor<R,T> MDReshape()
+   {
+      int k = 0, rLt_idx[M], rSd[M];
+      // initialize sub_layout_pair's second
+      for (sub_layout_type &sub: rLt)
+      {
+         for (sub_layout_pair &p: sub) { p.second = k++; }
+      }
+      // scan with the previous layout (N) order the reshaped layout (M)
+      for (int i = 0, j = 0; i < N; i++)
+      {
+         for (sub_layout_pair &p: rLt[layout[i]])
+         {
+            rLt_idx[j++] = p.second;
+         }
+      }
+      // apply the reshaped layout (M)
+      rSd[rLt_idx[0]] = 1;
+      for (int i = 1; i < M; i++)
+      {
+         rSd[rLt_idx[i]] = rNd[rLt_idx[i-1]] * rSd[rLt_idx[i-1]];
+      }
+      // construct the MDTensor with the given pointer and reshaped sizes
+      static_assert(R == M, "R != M");
+      return MDTensor<R,T>(reshape_ptr, rSd);
+   }
+
+private:
+   T *reshape_ptr;
+   std::vector<int> rNd; // reshape sizes
+   int reshape_offset, reshape_shifts[2];// shift begin & end
+   using sub_layout_pair = std::pair<int,int>;
+   using sub_layout_type = std::list<sub_layout_pair>;
+   std::array<sub_layout_type,N> rLt; // layout
+};
+
+// md_sequence, md_extend and make_md_sequence implementation
+namespace internal
+{
+
+template <typename T, int N, int mod, bool left> struct md_extend;
+
+template <typename T, T... args, int N>
+struct md_extend<md_sequence<T, args...>, N, 0, true>
+{
+   using type = md_sequence<T, args..., (args + N)...>;
+};
+
+template <typename T, T... args, int N>
+struct md_extend<md_sequence<T, args...>, N, 1, true>
+{
+   using type = md_sequence<T, args..., (args + N)..., 2*N>;
+};
+
+template <typename T, T... args, int N>
+struct md_extend<md_sequence<T, args...>, N, 0, false>
+{
+   using type = md_sequence<T, (args + N)..., args...>;
+};
+
+template <typename T, T... args, int N>
+struct md_extend<md_sequence<T, args...>, N, 1, false>
+{
+   using type = md_sequence<T, 2*N, (args + N)..., args...>;
+};
+
+template <typename T, int N, bool L> struct make_md_sequence
+{
+   using sequence_type = typename make_md_sequence<T,N/2,L>::type;
+   using type = typename md_extend<sequence_type, N/2, N%2, L>::type;
+};
+
+template <typename T, bool L>
+struct make_md_sequence<T,0,L> { using type = md_sequence<T>; };
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif // MFEM_MDSPAN_HPP

general/mem_manager.hpp

@@ -169,6 +169,7 @@ class Memory
 protected:
    friend class MemoryManager;
    friend void MemoryPrintFlags(unsigned flags);
+   template<typename mfem_type, int N, typename L> friend class MDSpan;
 
    enum FlagMask: unsigned
    {

general/sets.cpp

@@ -15,80 +15,45 @@
 namespace mfem
 {
 
-IntegerSet::IntegerSet(IntegerSet &s)
-   : me(s.me.Size())
+int IntegerSet::PickRandomElement() const
 {
-   for (int i = 0; i < me.Size(); i++)
-   {
-      me[i] = s.me[i];
-   }
-}
-
-IntegerSet& IntegerSet::operator=(const IntegerSet &s)
-{
-   me.SetSize(s.me.Size());
-   for (int i = 0; i < me.Size(); i++)
-   {
-      me[i] = s.me[i];
-   }
-   return *this;
-}
-
-int IntegerSet::operator== (IntegerSet &s)
-{
-   if (me.Size() != s.me.Size())
-   {
-      return 0;
-   }
-
-   for (int i = 0; i < me.Size(); i++)
-      if (me[i] != s.me[i])
-      {
-         return 0;
-      }
-
-   return 1;
-}
-
-int IntegerSet::PickRandomElement()
-{
-   int i, size = me.Size();
+   int i, size = Size();
    unsigned int seed = 0;
 
    for (i = 0; i < size; i++)
    {
-      seed += me[i];
+      seed += data[i];
    }
 
    srand(seed);
 
-   return me[rand()/(RAND_MAX/size)];
+   return data[rand()/(RAND_MAX/size)];
 }
 
 void IntegerSet::Recreate(const int n, const int *p)
 {
    int i, j;
 
-   me.SetSize(n);
+   SetSize(n);
 
    for (i = 0; i < n; i++)
    {
-      me[i] = p[i];
+      data[i] = p[i];
    }
 
-   me.Sort();
+   Sort();
 
    for (j = 0, i = 1; i < n; i++)
-      if (me[i] != me[j])
+      if (data[i] != data[j])
       {
-         me[++j] = me[i];
+         data[++j] = data[i];
       }
 
-   me.SetSize(j+1);
+   SetSize(j+1);
 }
 
 
-int ListOfIntegerSets::Insert(IntegerSet &s)
+int ListOfIntegerSets::Insert(const IntegerSet &s)
 {
    for (int i = 0; i < TheList.Size(); i++)
       if (*TheList[i] == s)
@@ -101,7 +66,7 @@ int ListOfIntegerSets::Insert(IntegerSet &s)
    return TheList.Size()-1;
 }
 
-int ListOfIntegerSets::Lookup(IntegerSet &s)
+int ListOfIntegerSets::Lookup(const IntegerSet &s) const
 {
    for (int i = 0; i < TheList.Size(); i++)
       if (*TheList[i] == s)
@@ -113,7 +78,7 @@ int ListOfIntegerSets::Lookup(IntegerSet &s)
    return -1;
 }
 
-void ListOfIntegerSets::AsTable(Table & t)
+void ListOfIntegerSets::AsTable(Table & t) const
 {
    int i;
 

general/sets.hpp

@@ -20,38 +20,26 @@ namespace mfem
 {
 
 /// A set of integers
-class IntegerSet
+class IntegerSet : public Array<int>
 {
-private:
-   Array<int> me;
-
 public:
-   /// Create an empty set.
-   IntegerSet() { }
-
-   /// Create a copy of set 's'.
-   IntegerSet(IntegerSet &s);
+   using Array<int>::Array; ///< Inherit all Array constructors.
+   // MSVC fails to recognize that rule of zero applies after using base class
+   // constructors.
+   IntegerSet() = default; ///< Default construct and empty set.
+   IntegerSet(const IntegerSet &) = default; ///< Copy constructor.
+   IntegerSet(IntegerSet &&) = default; ///< Move constructor.
+   IntegerSet& operator=(const IntegerSet &) = default; ///< Copy assignment.
+   IntegerSet& operator=(IntegerSet &&) = default; ///< Move assignment.
 
    /// Create an integer set from C-array 'p' of 'n' integers.
    IntegerSet(const int n, const int *p) { Recreate(n, p); }
 
-   /// Return the size of the set.
-   int Size() { return me.Size(); }
-
-   /// Return a reference to the sorted array of all the set entries.
-   operator Array<int>& () { return me; }
-
    /// Return the value of the lowest element of the set.
-   int PickElement() { return me[0]; }
+   int PickElement() const { return data[0]; }
 
    /// Return the value of a random element of the set.
-   int PickRandomElement();
-
-   /// Create a copy of set 's'.
-   IntegerSet& operator=(const IntegerSet &s);
-
-   /// Return 1 if the sets are equal and 0 otherwise.
-   int operator==(IntegerSet &s);
+   int PickRandomElement() const;
 
    /** @brief Create an integer set from C-array 'p' of 'n' integers.
        Overwrites any existing set data. */
@@ -67,25 +55,25 @@ private:
 public:
 
    /// Return the number of integer sets in the list.
-   int Size() { return TheList.Size(); }
+   int Size() const { return TheList.Size(); }
 
    /// Return the value of the first element of the ith set.
-   int PickElementInSet(int i) { return TheList[i]->PickElement(); }
+   int PickElementInSet(int i) const { return TheList[i]->PickElement(); }
 
    /// Return a random value from the ith set in the list.
-   int PickRandomElementInSet(int i) { return TheList[i]->PickRandomElement(); }
+   int PickRandomElementInSet(int i) const { return TheList[i]->PickRandomElement(); }
 
    /** @brief Check to see if set 's' is in the list. If not append it to the
        end of the list. Returns the index of the list where set 's' can be
        found. */
-   int Insert(IntegerSet &s);
+   int Insert(const IntegerSet &s);
 
    /** Return the index of the list where set 's' can be found. Returns -1 if
        not found. */
-   int Lookup(IntegerSet &s);
+   int Lookup(const IntegerSet &s) const;
 
    /// Write the list of sets into table 't'.
-   void AsTable(Table &t);
+   void AsTable(Table &t) const;
 
    ~ListOfIntegerSets();
 };

linalg/CMakeLists.txt

@@ -57,6 +57,7 @@ list(APPEND HDRS
   lapack.hpp
   linalg.hpp
   matrix.hpp
+  mdvector.hpp
   ode.hpp
   operator.hpp
   solvers.hpp

linalg/batched/native.cpp

@@ -185,7 +185,7 @@ void NativeBatchedLinAlg::LUFactor(DenseTensor &A, Array<int> &P) const
             }
          } // pivot end
 
-         if (abs(data_all(i,i,e)) <= tol)
+         if (std::abs(data_all(i,i,e)) <= tol)
          {
             d_pivot_flag[0] = false;
          }

linalg/hypre.cpp

@@ -2793,6 +2793,33 @@ void HypreParMatrix::PrintHash(std::ostream &os) const
    os << "col map offd hash : " << hf.GetHash() << '\n';
 }
 
+real_t HypreParMatrix::FNorm() const
+{
+   real_t norm_fro = 0.0;
+   if (A != NULL)
+#if MFEM_HYPRE_VERSION >= 21900
+   {
+      const int ierr = hypre_ParCSRMatrixNormFro(A, &norm_fro);
+      MFEM_VERIFY(ierr == 0, "");
+   }
+#else
+   {
+      // HYPRE_USING_GPU is not defined for
+      // MFEM_HYPRE_VERSION < 22100 and so here it is
+      // guaranteed that the matrix is in "host" memory
+      Vector Avec_diag(A->diag->data, A->diag->num_nonzeros);
+      real_t normsqr_fro = InnerProduct(Avec_diag, Avec_diag);
+      Vector Avec_offd(A->offd->data, A->offd->num_nonzeros);
+      normsqr_fro += InnerProduct(Avec_offd, Avec_offd);
+      MPI_Allreduce(MPI_IN_PLACE, &normsqr_fro, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, hypre_ParCSRMatrixComm(A));
+      norm_fro = sqrt(normsqr_fro);
+   }
+#endif
+   return norm_fro;
+}
+
+
 inline void delete_hypre_ParCSRMatrixColMapOffd(hypre_ParCSRMatrix *A)
 {
    HYPRE_BigInt  *A_col_map_offd = hypre_ParCSRMatrixColMapOffd(A);

linalg/hypre.hpp

@@ -945,6 +945,10 @@ public:
        without the need to save the whole matrix. */
    void PrintHash(std::ostream &out) const;
 
+   /// @brief Return the Frobenius norm of the matrix (or 0 if the underlying
+   /// hypre matrix is NULL)
+   real_t FNorm() const;
+
    /// Calls hypre's destroy function
    virtual ~HypreParMatrix() { Destroy(); }
 

linalg/mdvector.hpp

@@ -0,0 +1,68 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_MDVECTOR
+#define MFEM_MDVECTOR
+
+#include "../config/config.hpp"
+
+#include "vector.hpp"
+#include "general/mdspan.hpp"
+
+namespace mfem
+{
+
+template<int N, typename Layout = MDLayoutLeft<N>>
+struct MDVector : public MDSpan<Vector, N, Layout>
+{
+   using base_t = MDSpan<Vector, N, Layout>;
+
+   /**
+    * @brief MDVector default constructor (recursion)
+    */
+   MDVector(): base_t() { }
+
+   /**
+    * @brief MDVector recursion constructor
+    * @param[in] n Dimension indice
+    * @param[in] args Rest of dimension indices
+    */
+   template <typename... Ts>
+   MDVector(int n, Ts... args): MDVector(args...) { base_t::Setup(n, args...); }
+
+   /// Move constructor not supported
+   MDVector(MDVector&&) = delete;
+
+   /// Copy constructor not supported
+   MDVector(const MDVector&) = delete;
+
+   /// Move assignment not supported
+   MDVector& operator=(MDVector&&) = delete;
+
+   /// Copy assignment not supported
+   MDVector& operator=(const MDVector&) = delete;
+
+   using Vector::Read;
+   using Vector::Write;
+   using Vector::ReadWrite;
+   using Vector::HostRead;
+   using Vector::HostWrite;
+   using Vector::HostReadWrite;
+
+   using Vector::GetData;
+   using Vector::SetData;
+
+   using Vector::operator=;
+};
+
+} // namespace mfem
+
+#endif // MFEM_MDVECTOR

linalg/sundials.cpp

@@ -95,7 +95,7 @@ MFEM_DEPRECATED void* CVodeCreate(int lmm, SUNContext)
 
 /// (DEPRECATED) Wrapper function for backwards compatibility with SUNDIALS
 /// version < 6
-MFEM_DEPRECATED void* ARKStepCreate(ARKRhsFn fe, ARKRhsFn fi, realtype t0,
+MFEM_DEPRECATED void* ARKStepCreate(ARKRhsFn fe, ARKRhsFn fi, sunrealtype t0,
                                     N_Vector y0, SUNContext)
 {
    return ARKStepCreate(fe, fi, t0, y0);
@@ -127,7 +127,7 @@ MFEM_DEPRECATED N_Vector N_VNewEmpty_Parallel(MPI_Comm comm,
 /// (DEPRECATED) Wrapper function for backwards compatibility with SUNDIALS
 /// version < 6
 MFEM_DEPRECATED N_Vector SUN_Hip_OR_Cuda(N_VNewWithMemHelp)(sunindextype length,
-                                                            booleantype use_managed_mem,
+                                                            sunbooleantype use_managed_mem,
                                                             SUNMemoryHelper helper,
                                                             SUNContext)
 {
@@ -157,6 +157,16 @@ MFEM_DEPRECATED N_Vector N_VMake_MPIPlusX(MPI_Comm comm, N_Vector local_vector,
 
 #endif // SUNDIALS_VERSION_MAJOR < 6
 
+#if MFEM_SUNDIALS_VERSION < 70100
+#define MFEM_ARKode(FUNC) ARKStep##FUNC
+#else
+#define MFEM_ARKode(FUNC) ARKode##FUNC
+#endif
+
+// Macro STR(): expand the argument and add double quotes
+#define STR1(s) #s
+#define STR(s)  STR1(s)
+
 
 namespace mfem
 {
@@ -187,11 +197,21 @@ SundialsMemHelper &Sundials::GetMemHelper()
 Sundials::Sundials()
 {
 #ifdef MFEM_USE_MPI
-   MPI_Comm communicator = MPI_COMM_WORLD;
+   int mpi_initialized = 0;
+   MPI_Initialized(&mpi_initialized);
+   MPI_Comm communicator = mpi_initialized ? MPI_COMM_WORLD : MPI_COMM_NULL;
+#if SUNDIALS_VERSION_MAJOR < 7
    int return_val = SUNContext_Create((void*) &communicator, &context);
 #else
-   int return_val = SUNContext_Create(nullptr, &context);
+   int return_val = SUNContext_Create(communicator, &context);
 #endif
+#else // #ifdef MFEM_USE_MPI
+#if SUNDIALS_VERSION_MAJOR < 7
+   int return_val = SUNContext_Create(nullptr, &context);
+#else
+   int return_val = SUNContext_Create((SUNComm)(0), &context);
+#endif
+#endif // #ifdef MFEM_USE_MPI
    MFEM_VERIFY(return_val == 0, "Call to SUNContext_Create failed");
    SundialsMemHelper actual_helper(context);
    memHelper = std::move(actual_helper);
@@ -250,7 +270,11 @@ int SundialsMemHelper::SundialsMemHelper_Alloc(SUNMemoryHelper helper,
 #endif
                                               )
 {
+#if (SUNDIALS_VERSION_MAJOR < 7)
    SUNMemory sunmem = SUNMemoryNewEmpty();
+#else
+   SUNMemory sunmem = SUNMemoryNewEmpty(helper->sunctx);
+#endif
 
    sunmem->ptr = NULL;
    sunmem->own = SUNTRUE;
@@ -631,7 +655,7 @@ static int LSFree(SUNLinearSolver LS)
 // ---------------------------------------------------------------------------
 // CVODE interface
 // ---------------------------------------------------------------------------
-int CVODESolver::RHS(realtype t, const N_Vector y, N_Vector ydot,
+int CVODESolver::RHS(sunrealtype t, const N_Vector y, N_Vector ydot,
                      void *user_data)
 {
    // At this point the up-to-date data for N_Vector y and ydot is on the device.
@@ -648,7 +672,8 @@ int CVODESolver::RHS(realtype t, const N_Vector y, N_Vector ydot,
    return (0);
 }
 
-int CVODESolver::root(realtype t, N_Vector y, realtype *gout, void *user_data)
+int CVODESolver::root(sunrealtype t, N_Vector y, sunrealtype *gout,
+                      void *user_data)
 {
    CVODESolver *self = static_cast<CVODESolver*>(user_data);
 
@@ -668,8 +693,9 @@ void CVODESolver::SetRootFinder(int components, RootFunction func)
    MFEM_VERIFY(flag == CV_SUCCESS, "error in SetRootFinder()");
 }
 
-int CVODESolver::LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
-                             booleantype jok, booleantype *jcur, realtype gamma,
+int CVODESolver::LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy,
+                             SUNMatrix A, sunbooleantype jok,
+                             sunbooleantype *jcur, sunrealtype gamma,
                              void*, N_Vector, N_Vector, N_Vector)
 {
    // Get data from N_Vectors
@@ -683,7 +709,7 @@ int CVODESolver::LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
 }
 
 int CVODESolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
-                             N_Vector b, realtype tol)
+                             N_Vector b, sunrealtype tol)
 {
    SundialsNVector mfem_x(x);
    const SundialsNVector mfem_b(b);
@@ -859,7 +885,7 @@ void CVODESolver::UseSundialsLinearSolver()
    if (LSA != NULL) { SUNLinSolFree(LSA); LSA = NULL; }
 
    // Create linear solver
-   LSA = SUNLinSol_SPGMR(*Y, PREC_NONE, 0, Sundials::GetContext());
+   LSA = SUNLinSol_SPGMR(*Y, SUN_PREC_NONE, 0, Sundials::GetContext());
    MFEM_VERIFY(LSA, "error in SUNLinSol_SPGMR()");
 
    // Attach linear solver
@@ -1150,7 +1176,7 @@ void CVODESSolver::UseSundialsLinearSolverB()
    if (LSB != NULL) { SUNLinSolFree(LSB); LSB = NULL; }
 
    // Set default linear solver (Newton is the default Nonlinear Solver)
-   LSB = SUNLinSol_SPGMR(*yB, PREC_NONE, 0, Sundials::GetContext());
+   LSB = SUNLinSol_SPGMR(*yB, SUN_PREC_NONE, 0, Sundials::GetContext());
    MFEM_VERIFY(LSB, "error in SUNLinSol_SPGMR()");
 
    /* Attach the matrix and linear solver */
@@ -1158,11 +1184,11 @@ void CVODESSolver::UseSundialsLinearSolverB()
    MFEM_VERIFY(flag == CV_SUCCESS, "error in CVodeSetLinearSolverB()");
 }
 
-int CVODESSolver::LinSysSetupB(realtype t, N_Vector y, N_Vector yB,
+int CVODESSolver::LinSysSetupB(sunrealtype t, N_Vector y, N_Vector yB,
                                N_Vector fyB, SUNMatrix AB,
-                               booleantype jokB, booleantype *jcurB,
-                               realtype gammaB, void *user_data, N_Vector tmp1,
-                               N_Vector tmp2, N_Vector tmp3)
+                               sunbooleantype jokB, sunbooleantype *jcurB,
+                               sunrealtype gammaB, void *user_data,
+                               N_Vector tmp1, N_Vector tmp2, N_Vector tmp3)
 {
    // Get data from N_Vectors
    const SundialsNVector mfem_y(y);
@@ -1178,7 +1204,7 @@ int CVODESSolver::LinSysSetupB(realtype t, N_Vector y, N_Vector yB,
 }
 
 int CVODESSolver::LinSysSolveB(SUNLinearSolver LS, SUNMatrix AB, N_Vector yB,
-                               N_Vector Rb, realtype tol)
+                               N_Vector Rb, sunrealtype tol)
 {
    SundialsNVector mfem_yB(yB);
    const SundialsNVector mfem_Rb(Rb);
@@ -1216,7 +1242,7 @@ void CVODESSolver::SetWFTolerances(EWTFunction func)
 
 // CVODESSolver static functions
 
-int CVODESSolver::RHSQ(realtype t, const N_Vector y, N_Vector qdot,
+int CVODESSolver::RHSQ(sunrealtype t, const N_Vector y, N_Vector qdot,
                        void *user_data)
 {
    CVODESSolver *self = static_cast<CVODESSolver*>(user_data);
@@ -1229,7 +1255,7 @@ int CVODESSolver::RHSQ(realtype t, const N_Vector y, N_Vector qdot,
    return 0;
 }
 
-int CVODESSolver::RHSQB(realtype t, N_Vector y, N_Vector yB, N_Vector qBdot,
+int CVODESSolver::RHSQB(sunrealtype t, N_Vector y, N_Vector yB, N_Vector qBdot,
                         void *user_dataB)
 {
    CVODESSolver *self = static_cast<CVODESSolver*>(user_dataB);
@@ -1243,7 +1269,7 @@ int CVODESSolver::RHSQB(realtype t, N_Vector y, N_Vector yB, N_Vector qBdot,
    return 0;
 }
 
-int CVODESSolver::RHSB(realtype t, N_Vector y, N_Vector yB, N_Vector yBdot,
+int CVODESSolver::RHSB(sunrealtype t, N_Vector y, N_Vector yB, N_Vector yBdot,
                        void *user_dataB)
 {
    CVODESSolver *self = static_cast<CVODESSolver*>(user_dataB);
@@ -1341,7 +1367,7 @@ CVODESSolver::~CVODESSolver()
 // ARKStep interface
 // ---------------------------------------------------------------------------
 
-int ARKStepSolver::RHS1(realtype t, const N_Vector y, N_Vector result,
+int ARKStepSolver::RHS1(sunrealtype t, const N_Vector y, N_Vector result,
                         void *user_data)
 {
    // Get data from N_Vectors
@@ -1373,7 +1399,7 @@ int ARKStepSolver::RHS1(realtype t, const N_Vector y, N_Vector result,
    return (0);
 }
 
-int ARKStepSolver::RHS2(realtype t, const N_Vector y, N_Vector result,
+int ARKStepSolver::RHS2(sunrealtype t, const N_Vector y, N_Vector result,
                         void *user_data)
 {
    // Get data from N_Vectors
@@ -1399,9 +1425,9 @@ int ARKStepSolver::RHS2(realtype t, const N_Vector y, N_Vector result,
    return (0);
 }
 
-int ARKStepSolver::LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
-                               SUNMatrix, booleantype jok, booleantype *jcur,
-                               realtype gamma,
+int ARKStepSolver::LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy,
+                               SUNMatrix A, SUNMatrix, sunbooleantype jok,
+                               sunbooleantype *jcur, sunrealtype gamma,
                                void*, N_Vector, N_Vector, N_Vector)
 {
    // Get data from N_Vectors
@@ -1419,7 +1445,7 @@ int ARKStepSolver::LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
 }
 
 int ARKStepSolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
-                               N_Vector b, realtype tol)
+                               N_Vector b, sunrealtype tol)
 {
    SundialsNVector mfem_x(x);
    const SundialsNVector mfem_b(b);
@@ -1433,7 +1459,7 @@ int ARKStepSolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
    return (self->f->SUNImplicitSolve(mfem_b, mfem_x, tol));
 }
 
-int ARKStepSolver::MassSysSetup(realtype t, SUNMatrix M,
+int ARKStepSolver::MassSysSetup(sunrealtype t, SUNMatrix M,
                                 void*, N_Vector, N_Vector, N_Vector)
 {
    ARKStepSolver *self = static_cast<ARKStepSolver*>(GET_CONTENT(M));
@@ -1444,7 +1470,7 @@ int ARKStepSolver::MassSysSetup(realtype t, SUNMatrix M,
 }
 
 int ARKStepSolver::MassSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector x,
-                                N_Vector b, realtype tol)
+                                N_Vector b, sunrealtype tol)
 {
    SundialsNVector mfem_x(x);
    const SundialsNVector mfem_b(b);
@@ -1464,7 +1490,7 @@ int ARKStepSolver::MassMult1(SUNMatrix M, N_Vector x, N_Vector v)
    return (self->f->SUNMassMult(mfem_x, mfem_v));
 }
 
-int ARKStepSolver::MassMult2(N_Vector x, N_Vector v, realtype t,
+int ARKStepSolver::MassMult2(N_Vector x, N_Vector v, sunrealtype t,
                              void* mtimes_data)
 {
    const SundialsNVector mfem_x(x);
@@ -1535,7 +1561,7 @@ void ARKStepSolver::Init(TimeDependentOperator &f_)
       // Free existing solver memory and re-create with new vector size
       if (resize)
       {
-         ARKStepFree(&sundials_mem);
+         MFEM_ARKode(Free)(&sundials_mem);
          sundials_mem = NULL;
       }
    }
@@ -1573,12 +1599,15 @@ void ARKStepSolver::Init(TimeDependentOperator &f_)
       MFEM_VERIFY(sundials_mem, "error in ARKStepCreate()");
 
       // Attach the ARKStepSolver as user-defined data
-      flag = ARKStepSetUserData(sundials_mem, this);
-      MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetUserData()");
+      flag = MFEM_ARKode(SetUserData)(sundials_mem, this);
+      MFEM_VERIFY(flag == ARK_SUCCESS,
+                  "error in " STR(MFEM_ARKode(SetUserData)) "()");
 
       // Set default tolerances
-      flag = ARKStepSStolerances(sundials_mem, default_rel_tol, default_abs_tol);
-      MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetSStolerances()");
+      flag = MFEM_ARKode(SStolerances)(sundials_mem, default_rel_tol,
+                                       default_abs_tol);
+      MFEM_VERIFY(flag == ARK_SUCCESS,
+                  "error in " STR(MFEM_ARKode(SStolerances)) "()");
 
       // If implicit, attach MFEM linear solver by default
       if (use_implicit) { UseMFEMLinearSolver(); }
@@ -1617,15 +1646,16 @@ void ARKStepSolver::Step(Vector &x, real_t &t, real_t &dt)
 
    // Integrate the system
    double tout = t + dt;
-   flag = ARKStepEvolve(sundials_mem, tout, *Y, &t, step_mode);
-   MFEM_VERIFY(flag >= 0, "error in ARKStepEvolve()");
+   flag = MFEM_ARKode(Evolve)(sundials_mem, tout, *Y, &t, step_mode);
+   MFEM_VERIFY(flag >= 0, "error in " STR(MFEM_ARKode(Evolve)) "()");
 
    // Make sure host is up to date
    Y->HostRead();
 
    // Return the last incremental step size
-   flag = ARKStepGetLastStep(sundials_mem, &dt);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepGetLastStep()");
+   flag = MFEM_ARKode(GetLastStep)(sundials_mem, &dt);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(GetLastStep)) "()");
 }
 
 void ARKStepSolver::UseMFEMLinearSolver()
@@ -1651,12 +1681,14 @@ void ARKStepSolver::UseMFEMLinearSolver()
    A->ops->destroy = MatDestroy;
 
    // Attach the linear solver and matrix
-   flag = ARKStepSetLinearSolver(sundials_mem, LSA, A);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetLinearSolver()");
+   flag = MFEM_ARKode(SetLinearSolver)(sundials_mem, LSA, A);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetLinearSolver)) "()");
 
    // Set the linear system evaluation function
-   flag = ARKStepSetLinSysFn(sundials_mem, ARKStepSolver::LinSysSetup);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetLinSysFn()");
+   flag = MFEM_ARKode(SetLinSysFn)(sundials_mem, ARKStepSolver::LinSysSetup);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetLinSysFn)) "()");
 }
 
 void ARKStepSolver::UseSundialsLinearSolver()
@@ -1666,12 +1698,13 @@ void ARKStepSolver::UseSundialsLinearSolver()
    if (LSA != NULL) { SUNLinSolFree(LSA); LSA = NULL; }
 
    // Create linear solver
-   LSA = SUNLinSol_SPGMR(*Y, PREC_NONE, 0, Sundials::GetContext());
+   LSA = SUNLinSol_SPGMR(*Y, SUN_PREC_NONE, 0, Sundials::GetContext());
    MFEM_VERIFY(LSA, "error in SUNLinSol_SPGMR()");
 
    // Attach linear solver
-   flag = ARKStepSetLinearSolver(sundials_mem, LSA, NULL);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetLinearSolver()");
+   flag = MFEM_ARKode(SetLinearSolver)(sundials_mem, LSA, NULL);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetLinearSolver)) "()");
 }
 
 void ARKStepSolver::UseMFEMMassLinearSolver(int tdep)
@@ -1698,12 +1731,14 @@ void ARKStepSolver::UseMFEMMassLinearSolver(int tdep)
    M->ops->destroy = MatDestroy;
 
    // Attach the linear solver and matrix
-   flag = ARKStepSetMassLinearSolver(sundials_mem, LSM, M, tdep);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetLinearSolver()");
+   flag = MFEM_ARKode(SetMassLinearSolver)(sundials_mem, LSM, M, tdep);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMassLinearSolver)) "()");
 
    // Set the linear system function
-   flag = ARKStepSetMassFn(sundials_mem, ARKStepSolver::MassSysSetup);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMassFn()");
+   flag = MFEM_ARKode(SetMassFn)(sundials_mem, ARKStepSolver::MassSysSetup);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMassFn)) "()");
 
    // Check that the ODE is not expressed in EXPLICIT form
    MFEM_VERIFY(!f->isExplicit(), "ODE operator is expressed in EXPLICIT form")
@@ -1716,17 +1751,19 @@ void ARKStepSolver::UseSundialsMassLinearSolver(int tdep)
    if (LSM != NULL) { SUNLinSolFree(LSM); LSM = NULL; }
 
    // Create linear solver
-   LSM = SUNLinSol_SPGMR(*Y, PREC_NONE, 0, Sundials::GetContext());
+   LSM = SUNLinSol_SPGMR(*Y, SUN_PREC_NONE, 0, Sundials::GetContext());
    MFEM_VERIFY(LSM, "error in SUNLinSol_SPGMR()");
 
    // Attach linear solver
-   flag = ARKStepSetMassLinearSolver(sundials_mem, LSM, NULL, tdep);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMassLinearSolver()");
+   flag = MFEM_ARKode(SetMassLinearSolver)(sundials_mem, LSM, NULL, tdep);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMassLinearSolver)) "()");
 
    // Attach matrix multiplication function
-   flag = ARKStepSetMassTimes(sundials_mem, NULL, ARKStepSolver::MassMult2,
-                              this);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMassTimes()");
+   flag = MFEM_ARKode(SetMassTimes)(sundials_mem, NULL,
+                                    ARKStepSolver::MassMult2, this);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMassTimes)) "()");
 
    // Check that the ODE is not expressed in EXPLICIT form
    MFEM_VERIFY(!f->isExplicit(), "ODE operator is expressed in EXPLICIT form")
@@ -1739,20 +1776,23 @@ void ARKStepSolver::SetStepMode(int itask)
 
 void ARKStepSolver::SetSStolerances(double reltol, double abstol)
 {
-   flag = ARKStepSStolerances(sundials_mem, reltol, abstol);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSStolerances()");
+   flag = MFEM_ARKode(SStolerances)(sundials_mem, reltol, abstol);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SStolerances)) "()");
 }
 
 void ARKStepSolver::SetMaxStep(double dt_max)
 {
-   flag = ARKStepSetMaxStep(sundials_mem, dt_max);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMaxStep()");
+   flag = MFEM_ARKode(SetMaxStep)(sundials_mem, dt_max);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetMaxStep)) "()");
 }
 
 void ARKStepSolver::SetOrder(int order)
 {
-   flag = ARKStepSetOrder(sundials_mem, order);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetOrder()");
+   flag = MFEM_ARKode(SetOrder)(sundials_mem, order);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetOrder)) "()");
 }
 
 void ARKStepSolver::SetERKTableNum(ARKODE_ERKTableID table_id)
@@ -1776,8 +1816,9 @@ void ARKStepSolver::SetIMEXTableNum(ARKODE_ERKTableID etable_id,
 
 void ARKStepSolver::SetFixedStep(double dt)
 {
-   flag = ARKStepSetFixedStep(sundials_mem, dt);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetFixedStep()");
+   flag = MFEM_ARKode(SetFixedStep)(sundials_mem, dt);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(SetFixedStep)) "()");
 }
 
 void ARKStepSolver::PrintInfo() const
@@ -1799,18 +1840,19 @@ void ARKStepSolver::PrintInfo() const
                                      &netfails);
    MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepGetTimestepperStats()");
 
-   flag = ARKStepGetStepStats(sundials_mem,
-                              &nsteps,
-                              &hinused,
-                              &hlast,
-                              &hcur,
-                              &tcur);
+   flag = MFEM_ARKode(GetStepStats)(sundials_mem,
+                                    &nsteps,
+                                    &hinused,
+                                    &hlast,
+                                    &hcur,
+                                    &tcur);
 
    // Get nonlinear solver stats
-   flag = ARKStepGetNonlinSolvStats(sundials_mem,
-                                    &nniters,
-                                    &nncfails);
-   MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepGetNonlinSolvStats()");
+   flag = MFEM_ARKode(GetNonlinSolvStats)(sundials_mem,
+                                          &nniters,
+                                          &nncfails);
+   MFEM_VERIFY(flag == ARK_SUCCESS,
+               "error in " STR(MFEM_ARKode(GetNonlinSolvStats)) "()");
 
    mfem::out <<
              "ARKStep:\n"
@@ -1838,7 +1880,7 @@ ARKStepSolver::~ARKStepSolver()
    SUNMatDestroy(A);
    SUNLinSolFree(LSA);
    SUNNonlinSolFree(NLS);
-   ARKStepFree(&sundials_mem);
+   MFEM_ARKode(Free)(&sundials_mem);
 }
 
 // ---------------------------------------------------------------------------
@@ -1861,7 +1903,7 @@ int KINSolver::Mult(const N_Vector u, N_Vector fu, void *user_data)
 
 // Wrapper for computing Jacobian-vector products
 int KINSolver::GradientMult(N_Vector v, N_Vector Jv, N_Vector u,
-                            booleantype *new_u, void *user_data)
+                            sunbooleantype *new_u, void *user_data)
 {
    const SundialsNVector mfem_v(v);
    SundialsNVector mfem_Jv(Jv);
@@ -1901,7 +1943,7 @@ int KINSolver::LinSysSetup(N_Vector u, N_Vector, SUNMatrix J,
 
 // Wrapper for solving linear systems J u = b
 int KINSolver::LinSysSolve(SUNLinearSolver LS, SUNMatrix, N_Vector u,
-                           N_Vector b, realtype)
+                           N_Vector b, sunrealtype)
 {
    SundialsNVector mfem_u(u), mfem_b(b);
    KINSolver *self = static_cast<KINSolver*>(GET_CONTENT(LS));
@@ -1960,7 +2002,11 @@ KINSolver::KINSolver(int strategy, bool oper_grad)
    f_scale = new SundialsNVector();
 
    // Default abs_tol and print_level
+#if MFEM_SUNDIALS_VERSION < 70000
    abs_tol     = pow(UNIT_ROUNDOFF, 1.0/3.0);
+#else
+   abs_tol     = pow(SUN_UNIT_ROUNDOFF, 1.0/3.0);
+#endif
    print_level = 0;
 }
 
@@ -1974,7 +2020,11 @@ KINSolver::KINSolver(MPI_Comm comm, int strategy, bool oper_grad)
    f_scale = new SundialsNVector(comm);
 
    // Default abs_tol and print_level
+#if MFEM_SUNDIALS_VERSION < 70000
    abs_tol     = pow(UNIT_ROUNDOFF, 1.0/3.0);
+#else
+   abs_tol     = pow(SUN_UNIT_ROUNDOFF, 1.0/3.0);
+#endif
    print_level = 0;
 }
 #endif
@@ -2086,7 +2136,7 @@ void KINSolver::SetOperator(const Operator &op)
          if (A != NULL) { SUNMatDestroy(A); A = NULL; }
          if (LSA != NULL) { SUNLinSolFree(LSA); LSA = NULL; }
 
-         LSA = SUNLinSol_SPGMR(*Y, PREC_NONE, 0, Sundials::GetContext());
+         LSA = SUNLinSol_SPGMR(*Y, SUN_PREC_NONE, 0, Sundials::GetContext());
          MFEM_VERIFY(LSA, "error in SUNLinSol_SPGMR()");
 
          flag = KINSetLinearSolver(sundials_mem, LSA, NULL);
@@ -2155,12 +2205,12 @@ void KINSolver::SetJFNKSolver(Solver &solver)
    if (LSA != NULL) { SUNLinSolFree(LSA); LSA = NULL; }
 
    // Setup FGMRES
-   LSA = SUNLinSol_SPFGMR(*Y, prec ? PREC_RIGHT : PREC_NONE, maxli,
+   LSA = SUNLinSol_SPFGMR(*Y, prec ? SUN_PREC_RIGHT : SUN_PREC_NONE, maxli,
                           Sundials::GetContext());
    MFEM_VERIFY(LSA, "error in SUNLinSol_SPFGMR()");
 
    flag = SUNLinSol_SPFGMRSetMaxRestarts(LSA, maxlrs);
-   MFEM_VERIFY(flag == SUNLS_SUCCESS, "error in SUNLinSol_SPFGMR()");
+   MFEM_VERIFY(flag == SUN_SUCCESS, "error in SUNLinSol_SPFGMR()");
 
    flag = KINSetLinearSolver(sundials_mem, LSA, NULL);
    MFEM_VERIFY(flag == KIN_SUCCESS, "error in KINSetLinearSolver()");
@@ -2317,18 +2367,21 @@ void KINSolver::Mult(Vector &x,
 
    if (rank == 0)
    {
+#if MFEM_SUNDIALS_VERSION < 70000
       flag = KINSetPrintLevel(sundials_mem, print_level);
       MFEM_VERIFY(flag == KIN_SUCCESS, "KINSetPrintLevel() failed!");
+#endif
+      // NOTE: there is no KINSetPrintLevel in SUNDIALS v7!
 
 #ifdef SUNDIALS_BUILD_WITH_MONITORING
       if (jfnk && print_level)
       {
          flag = SUNLinSolSetInfoFile_SPFGMR(LSA, stdout);
-         MFEM_VERIFY(flag == SUNLS_SUCCESS,
+         MFEM_VERIFY(flag == SUN_SUCCESS,
                      "error in SUNLinSolSetInfoFile_SPFGMR()");
 
          flag = SUNLinSolSetPrintLevel_SPFGMR(LSA, 1);
-         MFEM_VERIFY(flag == SUNLS_SUCCESS,
+         MFEM_VERIFY(flag == SUN_SUCCESS,
                      "error in SUNLinSolSetPrintLevel_SPFGMR()");
       }
 #endif

linalg/sundials.hpp

@@ -54,6 +54,10 @@
 
 #include <functional>
 
+#define MFEM_SUNDIALS_VERSION \
+   (SUNDIALS_VERSION_MAJOR*10000 + SUNDIALS_VERSION_MINOR*100 + \
+    SUNDIALS_VERSION_PATCH)
+
 #if (SUNDIALS_VERSION_MAJOR < 6)
 
 /// (DEPRECATED) Map SUNDIALS version >= 6 datatypes and constants to
@@ -68,13 +72,30 @@ constexpr ARKODE_ERKTableID ARKODE_FEHLBERG_13_7_8 = FEHLBERG_13_7_8;
 /// arbitrary type for more compact backwards compatibility
 using SUNContext = void*;
 
+/// 'sunrealtype' was first introduced in v6.0.0
+typedef realtype sunrealtype;
+/// 'sunbooleantype' was first introduced in v6.0.0
+typedef booleantype sunbooleantype;
+
+/// New constant names introduced in v6.0.0
+enum { SUN_PREC_NONE, SUN_PREC_LEFT, SUN_PREC_RIGHT, SUN_PREC_BOTH };
+
 // KIN_ORTH_MGS was introduced in SUNDIALS v6; here, we define it just so that
 // it can be used as the default option in the second parameter of
 // KINSolver::EnableAndersonAcc -- the actual value of the parameter will be
 // ignored when using SUNDIALS < v6.
 #define KIN_ORTH_MGS 0
 
-#endif // SUNDIALS_VERSION_MAJOR < 6
+#endif // #if SUNDIALS_VERSION_MAJOR < 6
+
+#if (SUNDIALS_VERSION_MAJOR < 7)
+
+/** @brief The enum constant SUN_SUCCESS was added in v7 as a replacement of
+    various *_SUCCESS macros that were removed in v7. */
+enum { SUN_SUCCESS = 0 };
+
+#endif // #if SUNDIALS_VERSION_MAJOR < 7
+
 
 namespace mfem
 {
@@ -244,7 +265,14 @@ public:
 
 #ifdef MFEM_USE_MPI
    /// Returns the MPI communicator for the internal N_Vector x.
-   inline MPI_Comm GetComm() const { return *static_cast<MPI_Comm*>(N_VGetCommunicator(x)); }
+   inline MPI_Comm GetComm() const
+   {
+#if SUNDIALS_VERSION_MAJOR < 7
+      return *static_cast<MPI_Comm*>(N_VGetCommunicator(x));
+#else
+      return N_VGetCommunicator(x);
+#endif
+   }
 
    /// Returns the MPI global length for the internal N_Vector x.
    inline long GlobalSize() const { return N_VGetLength(x); }
@@ -396,24 +424,26 @@ protected:
    int root_components; /// Number of components in gout
 
    /// Wrapper to compute the ODE rhs function.
-   static int RHS(realtype t, const N_Vector y, N_Vector ydot, void *user_data);
+   static int RHS(sunrealtype t, const N_Vector y, N_Vector ydot,
+                  void *user_data);
 
    /// Setup the linear system $ A x = b $.
-   static int LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
-                          booleantype jok, booleantype *jcur,
-                          realtype gamma, void *user_data, N_Vector tmp1,
+   static int LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy, SUNMatrix A,
+                          sunbooleantype jok, sunbooleantype *jcur,
+                          sunrealtype gamma, void *user_data, N_Vector tmp1,
                           N_Vector tmp2, N_Vector tmp3);
 
    /// Solve the linear system $ A x = b $.
    static int LinSysSolve(SUNLinearSolver LS, SUNMatrix A, N_Vector x,
-                          N_Vector b, realtype tol);
+                          N_Vector b, sunrealtype tol);
 
    /// Prototype to define root finding for CVODE
-   static int root(realtype t, N_Vector y, realtype *gout, void *user_data);
+   static int root(sunrealtype t, N_Vector y, sunrealtype *gout,
+                   void *user_data);
 
    /// Typedef for root finding functions
-   typedef std::function<int(realtype t, Vector y, Vector gout, CVODESolver *)>
-   RootFunction;
+   typedef std::function<int(sunrealtype t, Vector y, Vector gout,
+                             CVODESolver *)> RootFunction;
 
    /// A class member to facilitate pointing to a user-specified root function
    RootFunction root_func;
@@ -421,7 +451,8 @@ protected:
    /// Typedef declaration for error weight functions
    typedef std::function<int(Vector y, Vector w, CVODESolver*)> EWTFunction;
 
-   /// A class member to facilitate pointing to a user-specified error weight function
+   /** @brief A class member to facilitate pointing to a user-specified error
+       weight function */
    EWTFunction ewt_func;
 
 public:
@@ -455,7 +486,7 @@ public:
        @note If this method is called a second time with a different problem
        size, then any non-default user-set options will be lost and will need
        to be set again. */
-   void Init(TimeDependentOperator &f_);
+   void Init(TimeDependentOperator &f_) override;
 
    /// Integrate the ODE with CVODE using the specified step mode.
    /** @param[in,out] x  On output, the solution vector at the requested output
@@ -531,14 +562,15 @@ protected:
    int indexB; ///< backward problem index
 
    /// Wrapper to compute the ODE RHS Quadrature function.
-   static int RHSQ(realtype t, const N_Vector y, N_Vector qdot, void *user_data);
+   static int RHSQ(sunrealtype t, const N_Vector y, N_Vector qdot,
+                   void *user_data);
 
    /// Wrapper to compute the ODE RHS backward function.
-   static int RHSB(realtype t, N_Vector y,
+   static int RHSB(sunrealtype t, N_Vector y,
                    N_Vector yB, N_Vector yBdot, void *user_dataB);
 
    /// Wrapper to compute the ODE RHS Backwards Quadrature function.
-   static int RHSQB(realtype t, N_Vector y, N_Vector yB,
+   static int RHSQB(sunrealtype t, N_Vector y, N_Vector yB,
                     N_Vector qBdot, void *user_dataB);
 
    /// Error control function
@@ -654,15 +686,15 @@ public:
    void SetSVtolerancesB(double reltol, Vector abstol);
 
    /// Setup the linear system A x = b
-   static int LinSysSetupB(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
+   static int LinSysSetupB(sunrealtype t, N_Vector y, N_Vector yB, N_Vector fyB,
                            SUNMatrix A,
-                           booleantype jok, booleantype *jcur,
-                           realtype gamma, void *user_data, N_Vector tmp1,
+                           sunbooleantype jok, sunbooleantype *jcur,
+                           sunrealtype gamma, void *user_data, N_Vector tmp1,
                            N_Vector tmp2, N_Vector tmp3);
 
    /// Solve the linear system A x = b
    static int LinSysSolveB(SUNLinearSolver LS, SUNMatrix A, N_Vector x,
-                           N_Vector b, realtype tol);
+                           N_Vector b, sunrealtype tol);
 
 
    /// Destroy the associated CVODES memory and SUNDIALS objects.
@@ -695,33 +727,35 @@ protected:
        RHS1 is explicit RHS and RHS2 the implicit RHS for IMEX integration. When
        purely implicit or explicit only RHS1 is used. */
    ///@{
-   static int RHS1(realtype t, const N_Vector y, N_Vector ydot, void *user_data);
-   static int RHS2(realtype t, const N_Vector y, N_Vector ydot, void *user_data);
+   static int RHS1(sunrealtype t, const N_Vector y, N_Vector ydot,
+                   void *user_data);
+   static int RHS2(sunrealtype t, const N_Vector y, N_Vector ydot,
+                   void *user_data);
    ///@}
 
    /// Setup the linear system $ A x = b $.
-   static int LinSysSetup(realtype t, N_Vector y, N_Vector fy, SUNMatrix A,
-                          SUNMatrix M, booleantype jok, booleantype *jcur,
-                          realtype gamma, void *user_data, N_Vector tmp1,
+   static int LinSysSetup(sunrealtype t, N_Vector y, N_Vector fy, SUNMatrix A,
+                          SUNMatrix M, sunbooleantype jok, sunbooleantype *jcur,
+                          sunrealtype gamma, void *user_data, N_Vector tmp1,
                           N_Vector tmp2, N_Vector tmp3);
 
    /// Solve the linear system $ A x = b $.
    static int LinSysSolve(SUNLinearSolver LS, SUNMatrix A, N_Vector x,
-                          N_Vector b, realtype tol);
+                          N_Vector b, sunrealtype tol);
 
    /// Setup the linear system $ M x = b $.
-   static int MassSysSetup(realtype t, SUNMatrix M, void *user_data,
+   static int MassSysSetup(sunrealtype t, SUNMatrix M, void *user_data,
                            N_Vector tmp1, N_Vector tmp2, N_Vector tmp3);
 
    /// Solve the linear system $ M x = b $.
    static int MassSysSolve(SUNLinearSolver LS, SUNMatrix M, N_Vector x,
-                           N_Vector b, realtype tol);
+                           N_Vector b, sunrealtype tol);
 
    /// Compute the matrix-vector product $ v = M x $.
    static int MassMult1(SUNMatrix M, N_Vector x, N_Vector v);
 
    /// Compute the matrix-vector product $v = M_t x $ at time t.
-   static int MassMult2(N_Vector x, N_Vector v, realtype t,
+   static int MassMult2(N_Vector x, N_Vector v, sunrealtype t,
                         void* mtimes_data);
 
 public:
@@ -757,7 +791,7 @@ public:
        @note If this method is called a second time with a different problem
        size, then any non-default user-set options will be lost and will need
        to be set again. */
-   void Init(TimeDependentOperator &f_);
+   void Init(TimeDependentOperator &f_) override;
 
    /// Integrate the ODE with ARKode using the specified step mode.
    /**
@@ -871,7 +905,7 @@ protected:
 
    /// Wrapper to compute the Jacobian-vector product $ J(u) v = Jv $.
    static int GradientMult(N_Vector v, N_Vector Jv, N_Vector u,
-                           booleantype *new_u, void *user_data);
+                           sunbooleantype *new_u, void *user_data);
 
    /// Setup the linear system $ J u = b $.
    static int LinSysSetup(N_Vector u, N_Vector fu, SUNMatrix J,
@@ -879,7 +913,7 @@ protected:
 
    /// Solve the linear system $ J u = b $.
    static int LinSysSolve(SUNLinearSolver LS, SUNMatrix J, N_Vector u,
-                          N_Vector b, realtype tol);
+                          N_Vector b, sunrealtype tol);
 
    /// Setup the preconditioner.
    static int PrecSetup(N_Vector uu,

linalg/vector.hpp

@@ -79,6 +79,7 @@ inline real_t rand_real()
 class Vector
 {
 protected:
+   template<typename mfem_type, int N, typename L> friend class MDSpan;
 
    Memory<real_t> data;
    int size;

mesh/CMakeLists.txt

@@ -32,6 +32,7 @@ set(SRCS
   vtk.cpp
   wedge.cpp
   submesh/submesh.cpp
+  submesh/ncsubmesh.cpp
   submesh/submesh_utils.cpp
   submesh/transfermap.cpp
   )
@@ -58,6 +59,7 @@ set(HDRS
   vertex.hpp
   vtk.hpp
   wedge.hpp
+  submesh/ncsubmesh.hpp
   submesh/submesh.hpp
   submesh/submesh_utils.hpp
   submesh/transfer_category.hpp
@@ -68,15 +70,17 @@ if (MFEM_USE_MPI)
   list(APPEND SRCS
     pmesh.cpp
     pncmesh.cpp
-    submesh/ptransfermap.cpp
-    submesh/psubmesh.cpp)
+    submesh/pncsubmesh.cpp
+    submesh/psubmesh.cpp
+    submesh/ptransfermap.cpp)
   # If this list (HDRS -> HEADERS) is used for install, we probably want the
   # headers added all the time.
   list(APPEND HDRS
     pmesh.hpp
     pncmesh.hpp
-    submesh/ptransfermap.hpp
-    submesh/psubmesh.hpp)
+    submesh/pncsubmesh.hpp
+    submesh/psubmesh.hpp
+    submesh/ptransfermap.hpp)
 endif()
 
 if (MFEM_USE_PUMI)

mesh/mesh.cpp

@@ -2033,6 +2033,18 @@ int Mesh::AddBdrElement(Element *elem)
    return NumOfBdrElements++;
 }
 
+void Mesh::AddBdrElements(Array<Element *> &bdr_elems,
+                          const Array<int> &new_be_to_face)
+{
+   boundary.Reserve(boundary.Size() + bdr_elems.Size());
+   MFEM_ASSERT(bdr_elems.Size() == new_be_to_face.Size(), "wrong size");
+   for (int i = 0; i < bdr_elems.Size(); i++)
+   {
+      AddBdrElement(bdr_elems[i]);
+   }
+   be_to_face.Append(new_be_to_face);
+}
+
 int Mesh::AddBdrSegment(int v1, int v2, int attr)
 {
    CheckEnlarge(boundary, NumOfBdrElements);
@@ -7346,6 +7358,12 @@ void Mesh::GetBdrElementAdjacentElement2(
    info = fi.Elem1Inf + ori;
 }
 
+void Mesh::SetAttribute(int i, int attr)
+{
+  elements[i]->SetAttribute(attr);
+  if (ncmesh) ncmesh->SetAttribute(i, attr);
+}
+
 Element::Type Mesh::GetElementType(int i) const
 {
    return elements[i]->GetType();
@@ -7672,7 +7690,6 @@ void Mesh::AddQuadFaceElement(int lf, int gf, int el,
 void Mesh::GenerateFaces()
 {
    int nfaces = GetNumFaces();
-
    for (auto &f : faces)
    {
       FreeElement(f);

mesh/mesh.hpp

@@ -993,6 +993,17 @@ public:
    /// @note Ownership of @a elem will pass to the Mesh object
    int AddBdrElement(Element *elem);
 
+   /**
+    * @brief Add an array of boundary elements to the mesh, along with map from
+    * the elements to their faces
+    * @param[in] bdr_elems The set of boundary element pointers, ownership of
+    * the pointers will be transferred to the Mesh object
+    * @param[in] be_to_face The map from the boundary element index to the face
+    * index
+    */
+   void AddBdrElements(Array<Element *> &bdr_elems,
+                       const Array<int> &be_to_face);
+
    int AddBdrSegment(int v1, int v2, int attr = 1);
    int AddBdrSegment(const int *vi, int attr = 1);
 
@@ -1102,6 +1113,15 @@ public:
        have two adjacent faces in 3D, or edges in 2D. */
    void RemoveInternalBoundaries();
 
+   /**
+    * @brief Clear the boundary element to edge map.
+    */
+   void DeleteBoundaryElementToEdge()
+   {
+      delete bel_to_edge;
+      bel_to_edge = nullptr;
+   }
+
    /// @}
 
    /// @name Element ordering methods
@@ -1366,7 +1386,7 @@ public:
    int GetAttribute(int i) const { return elements[i]->GetAttribute(); }
 
    /// Set the attribute of element i.
-   void SetAttribute(int i, int attr) { elements[i]->SetAttribute(attr); }
+   void SetAttribute(int i, int attr);
 
    /// Return the attribute of boundary element i.
    int GetBdrAttribute(int i) const { return boundary[i]->GetAttribute(); }

mesh/mesh_headers.hpp

@@ -25,6 +25,7 @@
 #include "ncmesh.hpp"
 #include "mesh.hpp"
 #include "mesh_operators.hpp"
+#include "submesh/ncsubmesh.hpp"
 #include "submesh/submesh.hpp"
 #include "submesh/submesh_utils.hpp"
 #include "submesh/transfermap.hpp"
@@ -36,6 +37,7 @@
 #ifdef MFEM_USE_MPI
 #include "pncmesh.hpp"
 #include "pmesh.hpp"
+#include "submesh/pncsubmesh.hpp"
 #include "submesh/psubmesh.hpp"
 #include "submesh/ptransfermap.hpp"
 #endif

mesh/ncmesh.cpp

@@ -58,23 +58,25 @@ void NCMesh::GeomInfo::InitGeom(Geometry::Type geom)
 {
    if (initialized) { return; }
 
-   mfem::Element *elem = NULL;
-   switch (geom)
+   auto elem = [&]()
    {
-      case Geometry::CUBE: elem = new Hexahedron; break;
-      case Geometry::PRISM: elem = new Wedge; break;
-      case Geometry::TETRAHEDRON: elem = new Tetrahedron; break;
-      case Geometry::PYRAMID: elem = new Pyramid; break;
-      case Geometry::SQUARE: elem = new Quadrilateral; break;
-      case Geometry::TRIANGLE: elem = new Triangle; break;
-      case Geometry::SEGMENT: elem = new Segment; break;
-      default: MFEM_ABORT("unsupported geometry " << geom);
-   }
+      switch (geom)
+      {
+         case Geometry::CUBE: return std::unique_ptr<mfem::Element>(new Hexahedron);
+         case Geometry::PRISM: return std::unique_ptr<mfem::Element>(new Wedge);
+         case Geometry::TETRAHEDRON: return std::unique_ptr<mfem::Element>
+                                               (new Tetrahedron);
+         case Geometry::PYRAMID: return std::unique_ptr<mfem::Element>(new Pyramid);
+         case Geometry::SQUARE: return std::unique_ptr<mfem::Element>(new Quadrilateral);
+         case Geometry::TRIANGLE: return std::unique_ptr<mfem::Element>(new Triangle);
+         case Geometry::SEGMENT: return std::unique_ptr<mfem::Element>(new Segment);
+         default: MFEM_ABORT("unsupported geometry " << geom);
+      }
+   }();
 
    nv = elem->GetNVertices();
    ne = elem->GetNEdges();
    nf = elem->GetNFaces();
-
    for (int i = 0; i < ne; i++)
    {
       for (int j = 0; j < 2; j++)
@@ -119,19 +121,9 @@ void NCMesh::GeomInfo::InitGeom(Geometry::Type geom)
       }
    }
 
-   delete elem;
    initialized = true;
 }
 
-static void CheckSupportedGeom(Geometry::Type geom)
-{
-   MFEM_VERIFY(geom == Geometry::SEGMENT ||
-               geom == Geometry::TRIANGLE || geom == Geometry::SQUARE ||
-               geom == Geometry::CUBE || geom == Geometry::PRISM ||
-               geom == Geometry::PYRAMID || geom == Geometry::TETRAHEDRON,
-               "Element type " << geom << " is not supported by NCMesh.");
-}
-
 NCMesh::NCMesh(const Mesh *mesh)
    : shadow(1024, 2048)
 {
@@ -157,7 +149,7 @@ NCMesh::NCMesh(const Mesh *mesh)
       }
 
       // create NCMesh::Element for this mfem::Element
-      int root_id = AddElement(Element(geom, elem->GetAttribute()));
+      int root_id = AddElement(geom, elem->GetAttribute());
       MFEM_ASSERT(root_id == i, "");
       Element &root_elem = elements[root_id];
 
@@ -248,11 +240,18 @@ NCMesh::NCMesh(const NCMesh &other)
    , nodes(other.nodes)
    , faces(other.faces)
    , elements(other.elements)
+   , free_element_ids(other.free_element_ids)
+   , root_state(other.root_state)
+   , coordinates(other.coordinates)
+   , NEdges(other.NEdges)
+   , NFaces(other.NFaces)
+   , NGhostEdges(other.NGhostEdges)
+   , NGhostFaces(other.NGhostFaces)
+   , boundary_faces(other.boundary_faces)
+   , face_geom(other.face_geom)
+   , element_vertex(other.element_vertex)
    , shadow(1024, 2048)
 {
-   other.free_element_ids.Copy(free_element_ids);
-   other.root_state.Copy(root_state);
-   other.coordinates.Copy(coordinates);
    Update();
 }
 
@@ -351,8 +350,8 @@ int NCMesh::GetMidFaceNode(int en1, int en2, int en3, int en4)
 
 void NCMesh::ReferenceElement(int elem)
 {
-   Element &el = elements[elem];
-   int* node = el.node;
+   const Element &el = elements[elem];
+   const int* node = el.node;
    GeomInfo& gi = GI[el.Geom()];
 
    // reference all vertices
@@ -507,7 +506,7 @@ int NCMesh::NewHexahedron(int n0, int n1, int n2, int n3,
                           int fattr3, int fattr4, int fattr5)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::CUBE, attr));
+   int new_id = AddElement(Geometry::CUBE, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2, el.node[3] = n3;
@@ -537,7 +536,7 @@ int NCMesh::NewWedge(int n0, int n1, int n2,
                      int fattr2, int fattr3, int fattr4)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::PRISM, attr));
+   int new_id = AddElement(Geometry::PRISM, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2;
@@ -566,7 +565,7 @@ int NCMesh::NewTetrahedron(int n0, int n1, int n2, int n3, int attr,
                            int fattr0, int fattr1, int fattr2, int fattr3)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::TETRAHEDRON, attr));
+   int new_id = AddElement(Geometry::TETRAHEDRON, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2, el.node[3] = n3;
@@ -592,7 +591,7 @@ int NCMesh::NewPyramid(int n0, int n1, int n2, int n3, int n4, int attr,
                        int fattr4)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::PYRAMID, attr));
+   int new_id = AddElement(Geometry::PYRAMID, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2, el.node[3] = n3;
@@ -622,7 +621,7 @@ int NCMesh::NewQuadrilateral(int n0, int n1, int n2, int n3,
                              int eattr0, int eattr1, int eattr2, int eattr3)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::SQUARE, attr));
+   int new_id = AddElement(Geometry::SQUARE, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2, el.node[3] = n3;
@@ -647,7 +646,7 @@ int NCMesh::NewTriangle(int n0, int n1, int n2,
                         int attr, int eattr0, int eattr1, int eattr2)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::TRIANGLE, attr));
+   int new_id = AddElement(Geometry::TRIANGLE, attr);
    Element &el = elements[new_id];
 
    el.node[0] = n0, el.node[1] = n1, el.node[2] = n2;
@@ -672,7 +671,7 @@ int NCMesh::NewTriangle(int n0, int n1, int n2,
 int NCMesh::NewSegment(int n0, int n1, int attr, int vattr1, int vattr2)
 {
    // create new element, initialize nodes
-   int new_id = AddElement(Element(Geometry::SEGMENT, attr));
+   int new_id = AddElement(Geometry::SEGMENT, attr);
    Element &el = elements[new_id];
    el.node[0] = n0, el.node[1] = n1;
 
@@ -2167,7 +2166,6 @@ void NCMesh::UpdateLeafElements()
    // final (Mesh) indices of leaves
    leaf_elements.Append(ghosts);
    leaf_sfc_index.SetSize(leaf_elements.Size());
-
    for (int i = 0; i < leaf_elements.Size(); i++)
    {
       Element &el = elements[leaf_elements[i]];
@@ -2234,7 +2232,6 @@ void NCMesh::UpdateVertices()
    }
 
    // STEP 2: assign indices of top-level local vertices, in original order
-
    NVertices = 0;
    for (auto &node : nodes)
    {
@@ -2246,7 +2243,6 @@ void NCMesh::UpdateVertices()
 
    // STEP 3: go over all elements (local and ghost) in SFC order and assign
    // remaining local vertices in that order.
-
    Array<int> sfc_order(leaf_elements.Size());
    for (int i = 0; i < sfc_order.Size(); i++)
    {
@@ -2264,7 +2260,6 @@ void NCMesh::UpdateVertices()
    }
 
    // STEP 4: create the mapping from Mesh vertex index to NCMesh node index
-
    vertex_nodeId.SetSize(NVertices);
    for (auto node = nodes.begin(); node != nodes.end(); ++node)
    {
@@ -2277,7 +2272,6 @@ void NCMesh::UpdateVertices()
 
    // STEP 5: assign remaining ghost vertices, ignore vertices beyond the ghost
    // layer
-
    NGhostVertices = 0;
    for (int i = 0; i < sfc_order.Size(); i++)
    {
@@ -2361,6 +2355,8 @@ void NCMesh::InitRootState(int root_count)
    root_state.SetSize(root_count);
    root_state = 0;
 
+   if (elements.Size() == 0) { return; }
+
    char* node_order;
    int nch;
 
@@ -2610,11 +2606,10 @@ void NCMesh::OnMeshUpdated(Mesh *mesh)
    {
       const int *ev = edge_vertex->GetRow(i);
       Node* node = nodes.Find(vertex_nodeId[ev[0]], vertex_nodeId[ev[1]]);
-
       MFEM_ASSERT(node && node->HasEdge(),
                   "edge (" << ev[0] << "," << ev[1] << ") not found, "
-                  "node = " << node);
-
+                  "node = " << node << " node->HasEdge() "
+                  << (node != nullptr ? node->HasEdge() : false));
       node->edge_index = i;
    }
 
@@ -2709,7 +2704,6 @@ void NCMesh::OnMeshUpdated(Mesh *mesh)
          if (face.index < 0)
          {
             face.index = NFaces + (nghosts++);
-
             // store the face geometry
             static const Geometry::Type types[5] =
             {
@@ -2793,10 +2787,186 @@ bool NCMesh::TriFaceSplit(int v1, int v2, int v3, int mid[3]) const
    if (mid) { mid[0] = e1, mid[1] = e2, mid[2] = e3; }
 
    // This is necessary but not sufficient to determine if a face has been
-   // split.
+   // split. All edges might have been split due to edge attached faces being
+   // refined. Need to check for existence of face made up of midpoints.
    return true;
 }
 
+bool contains_node(const std::array<int, 4> &nodes, int n)
+{
+   return std::find(nodes.begin(), nodes.end(), n) != nodes.end();
+};
+
+int NCMesh::ParentFaceNodes(std::array<int, 4> &face_nodes) const
+{
+   const bool is_tri = face_nodes[3] == -1;
+   const bool is_segment = (face_nodes[0] == face_nodes[1] &&
+                            face_nodes[2] == face_nodes[3]);
+   const bool is_quad = *std::min_element(face_nodes.begin(),
+                                          face_nodes.end()) >= 0;
+
+   MFEM_ASSERT((is_tri && !is_segment && !is_quad)
+               || (!is_tri && is_segment && !is_quad) || (!is_tri && !is_segment &&
+                                                          is_quad), "Inconsistent node geometry");
+
+   bool all_nodes_root = true;
+   for (auto x : face_nodes)
+   {
+      all_nodes_root = all_nodes_root && (x < 0 || (nodes[x].p1 == nodes[x].p2));
+   }
+   // This face is a root face -> nothing to do.
+   if (all_nodes_root) { return -1; }
+
+   int child = -1; // The index into parent.child that this face corresponds to.
+   auto parent_nodes = face_nodes;
+   if (is_quad)
+   {
+      // Logic for coarsening anisotropic faces is more complex, needs
+      // identification and handling of multiple "crux" points. Will require
+      // inspection of edge nodes.
+      MFEM_VERIFY(Iso,
+                  "ParentFaceNodes does not support anisotropic refinement yet!");
+
+      // Finds the first node whose parents aren't in the face_nodes. This is
+      // also the index of the child location in the parent face. Treated
+      // separately as ultimately multiple crux will need to be handled for
+      // anisotropic faces.
+      const auto crux = [&]()
+      {
+         for (int i = 0; i < static_cast<int>(face_nodes.size()); i++)
+         {
+            if ((!contains_node(face_nodes, nodes[face_nodes[i]].p1)
+                 && !contains_node(face_nodes, nodes[face_nodes[i]].p2))
+                || (nodes[face_nodes[i]].p1 == nodes[face_nodes[i]].p2) /* top level node */)
+            {
+               return i;
+            }
+         }
+         return -1;
+      }();
+      MFEM_ASSERT(crux != -1, "A root face should have been returned early");
+
+      // Loop over nodes, starting from diagonal to child, wrapping and skipping
+      // child. This will visit the node opposite child twice, thereby
+      // coarsening to the diagonally opposite. NOTE: This assumes that the
+      // nodes for a square are numbered (0 -> 1 -> 2 -> 3 -> 0).
+      for (int i = 0; i < static_cast<int>(face_nodes.size()) + 1; i++)
+      {
+         int ind = (crux + i + 2) %
+                   4; // Start and end with coarsening of the diagonally opposite
+         if (ind == crux) { continue; }
+         auto &x = parent_nodes[ind];
+
+         // Check against parent_nodes rather than face_nodes so on second lap
+         // the node opposite crux will coarsen again to the diagonally across
+         // in the parent face. A top level node has p1 == p2, thus these
+         // modifications do nothing.
+         if (contains_node(parent_nodes, nodes[x].p1))
+         {
+            MFEM_ASSERT(nodes[x].p2 == nodes[x].p1 ||
+                        !contains_node(parent_nodes, nodes[x].p2), "!");
+            x = nodes[x].p2;
+         }
+         else if (contains_node(parent_nodes, nodes[x].p2))
+         {
+            MFEM_ASSERT(nodes[x].p2 == nodes[x].p1 ||
+                        !contains_node(parent_nodes, nodes[x].p1), "!");
+            x = nodes[x].p1;
+         }
+         else { /* do nothing */ }
+      }
+   }
+   else if (is_tri)
+   {
+      for (int i = 0; i < 3; i++)
+      {
+         auto x = face_nodes[i];
+         if (x == -1) { continue; }
+         if (contains_node(face_nodes, nodes[x].p1))
+         {
+            MFEM_ASSERT(nodes[x].p2 == nodes[x].p1 ||
+                        !contains_node(face_nodes, nodes[x].p2), "!");
+            parent_nodes[i] = nodes[x].p2;
+         }
+         else if (contains_node(face_nodes, nodes[x].p2))
+         {
+            MFEM_ASSERT(nodes[x].p2 == nodes[x].p1 ||
+                        !contains_node(face_nodes, nodes[x].p1), "!");
+            parent_nodes[i] = nodes[x].p1;
+         }
+         else { /* do nothing */ }
+      }
+
+      if (std::equal(face_nodes.begin(), face_nodes.end(), parent_nodes.begin()))
+      {
+         // Having excluded root faces, this must be an interior face. We need
+         // to handle the special case of the interior face of the parent face.
+         std::array<std::array<int, 2>, 6> parent_pairs;
+         for (std::size_t i = 0; i < face_nodes.size() - 1; i++)
+         {
+            parent_pairs[i][0] = nodes[face_nodes[i]].p1;
+            parent_pairs[i][1] = nodes[face_nodes[i]].p2;
+         }
+         // Each node gets mapped to the common node from its parents and the
+         // predecessor node's parents.
+         for (int i = 0; i < 3; i++)
+         {
+            // Parenting convention here assumes parent face has the SAME
+            // orientation as the original. This is true on exterior boundaries,
+            // but for an interior boundary the master face will have an
+            // opposing orientation. TODO: Possibly fix for interior boundaries.
+            const auto &prev = parent_pairs[(i - 1 + 3) % 3]; // (0 -> 2, 1 -> 0, 2 -> 1)
+            const auto &next = parent_pairs[(i + 1 + 3) % 3]; // (0 -> 1, 1 -> 2, 2 -> 0)
+            for (auto x : next)
+            {
+               if (std::find(prev.begin(), prev.end(), x) != prev.end()) { parent_nodes[i] = x; }
+            }
+         }
+         child = 3; // The interior face is the final child.
+      }
+   }
+   else if (is_segment)
+   {
+      // Given this isn't a root face, one node must be the parent of the other.
+      if (face_nodes[0] == nodes[face_nodes[1]].p1)
+      {
+         face_nodes[1] = nodes[face_nodes[1]].p2;
+      }
+      else if (face_nodes[0] == nodes[face_nodes[1]].p2)
+      {
+         face_nodes[1] = nodes[face_nodes[1]].p1;
+      }
+      else if (face_nodes[1] == nodes[face_nodes[0]].p1)
+      {
+         face_nodes[0] = nodes[face_nodes[0]].p2;
+      }
+      else if (face_nodes[1] == nodes[face_nodes[0]].p2)
+      {
+         face_nodes[0] = nodes[face_nodes[0]].p1;
+      }
+      else
+      {
+         MFEM_ABORT("Internal logic error!");
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Unrecognized face geometry!");
+   }
+   for (int i = 0; i < 4 && face_nodes[i] >= 0; i++)
+   {
+      if (face_nodes[i] == parent_nodes[i])
+      {
+         MFEM_ASSERT(child == -1,
+                     "This face cannot be more than one child of the parent face!");
+         child = i;
+      }
+   }
+   MFEM_ASSERT(child != -1, "Root elements must have exited early!");
+   std::swap(face_nodes, parent_nodes);
+   return child;
+}
+
 int NCMesh::find_node(const Element &el, int node)
 {
    for (int i = 0; i < MaxElemNodes; i++)
@@ -3556,7 +3726,8 @@ NCMesh::NCList::BuildIndex() const
       int max_master_index = max_master != nullptr ? max_master->index : -1;
       int max_slave_index = max_slave != nullptr ? max_slave->index : -1;
 
-      inv_index.reserve(std::max({max_conforming_index, max_master_index, max_slave_index}));
+      inv_index.reserve(max(max_conforming_index, max_master_index, max_slave_index,
+                            0));
       for (int i = 0; i < conforming.Size(); i++)
       {
          inv_index.emplace(conforming[i].index, std::make_pair(MeshIdType::CONFORMING,
@@ -3571,8 +3742,6 @@ NCMesh::NCList::BuildIndex() const
          inv_index.emplace(slaves[i].index, std::make_pair(MeshIdType::SLAVE, i));
       }
    }
-   MFEM_ASSERT(inv_index.size() > 0,
-               "Empty inverse index, member lists must be populated before BuildIndex is called!");
 }
 
 //// Neighbors /////////////////////////////////////////////////////////////////
@@ -5260,12 +5429,21 @@ void NCMesh::GetElementFacesAttributes(int leaf_elem,
       face_attribs[i] = face->attribute;
    }
 }
-
 void NCMesh::FindFaceNodes(int face, int node[4]) const
+{
+   auto tmp = FindFaceNodes(face);
+   std::copy(tmp.begin(), tmp.end(), node);
+}
+
+std::array<int, 4> NCMesh::FindFaceNodes(int face) const
+{
+   return FindFaceNodes(faces[face]);
+}
+
+std::array<int, 4> NCMesh::FindFaceNodes(const Face &fa) const
 {
    // Obtain face nodes from one of its elements (note that face->p1, p2, p3
    // cannot be used directly since they are not in order and p4 is missing).
-   const Face &fa = faces[face];
    int elem = fa.elem[0];
    if (elem < 0) { elem = fa.elem[1]; }
    MFEM_ASSERT(elem >= 0, "Face has no elements?");
@@ -5277,10 +5455,12 @@ void NCMesh::FindFaceNodes(int face, int node[4]) const
                            find_node(el, fa.p3));
 
    const int* fv = GI[el.Geom()].faces[f];
+   std::array<int, 4> node;
    for (int i = 0; i < 4; i++)
    {
       node[i] = el.node[fv[i]];
    }
+   return node;
 }
 
 void NCMesh::GetBoundaryClosure(const Array<int> &bdr_attr_is_ess,
@@ -5294,13 +5474,11 @@ void NCMesh::GetBoundaryClosure(const Array<int> &bdr_attr_is_ess,
    if (Dim == 3)
    {
       GetFaceList(); // make sure 'boundary_faces' is up to date
-
       for (int f : boundary_faces)
       {
          if (bdr_attr_is_ess[faces[f].attribute - 1])
          {
-            int node[4];
-            FindFaceNodes(f, node);
+            auto node = FindFaceNodes(f);
             int nfv = (node[3] < 0) ? 3 : 4;
 
             for (int j = 0; j < nfv; j++)
@@ -5334,6 +5512,7 @@ void NCMesh::GetBoundaryClosure(const Array<int> &bdr_attr_is_ess,
    }
    else if (Dim == 2)
    {
+      GetFaceList();
       GetEdgeList(); // make sure 'boundary_faces' is up to date
 
       for (int f : boundary_faces)
@@ -5554,9 +5733,7 @@ void NCMesh::LimitNCLevel(int max_nc_level)
    {
       Array<Refinement> refinements;
       GetLimitRefinements(refinements, max_nc_level);
-
       if (!refinements.Size()) { break; }
-
       Refine(refinements);
    }
 }
@@ -5847,12 +6024,15 @@ void NCMesh::InitRootElements()
 
    // count the root elements
    int nroots = 0;
-   while (nroots < elements.Size() &&
-          elements[nroots].parent == -1)
-   {
-      nroots++;
-   }
-   MFEM_VERIFY(nroots, "invalid mesh file: no root elements found.");
+   for (const auto &e : elements)
+      if (e.parent == -1)
+      {
+         ++nroots;
+      }
+   MFEM_VERIFY(nroots > 0 ||
+               elements.Size() == 0,
+               "invalid mesh file: no root elements in non-empty mesh found.");
+
 
    // check that only the first 'nroot' elements are roots (have no parent)
    for (int i = nroots; i < elements.Size(); i++)
@@ -5892,6 +6072,9 @@ NCMesh::NCMesh(std::istream &input, int version, int &curved, int &is_nc)
    std::string ident;
    int count;
 
+   // Skip the version string
+   skip_comment_lines(input, 'M');
+
    // load dimension
    skip_comment_lines(input, '#');
    input >> ident;
@@ -6018,9 +6201,10 @@ NCMesh::NCMesh(std::istream &input, int version, int &curved, int &is_nc)
    {
       LoadCoordinates(input);
 
-      MFEM_VERIFY(coordinates.Size()/3 >= CountTopLevelNodes(),
+      MFEM_VERIFY(coordinates.Size() >= 3*CountTopLevelNodes(),
                   "Invalid mesh file: not all top-level nodes are covered by "
-                  "the 'coordinates' section of the mesh file.");
+                  "the 'coordinates' section of the mesh file: " << coordinates.Size() << ' ' <<
+                  3*CountTopLevelNodes());
       curved = 0;
    }
    else if (ident == "nodes")
@@ -6082,7 +6266,7 @@ void NCMesh::LoadCoarseElements(std::istream &input)
       int ref_type;
       input >> ref_type;
 
-      int elem = AddElement(Element(Geometry::INVALID, 0));
+      int elem = AddElement(Geometry::INVALID, 0);
       Element &el = elements[elem];
       el.ref_type = ref_type;
 
@@ -6169,7 +6353,7 @@ void NCMesh::LoadLegacyFormat(std::istream &input, int &curved, int &is_nc)
       CheckSupportedGeom(type);
       GI[geom].InitGeom(type);
 
-      int eid = AddElement(Element(type, attr));
+      int eid = AddElement(type, attr);
       MFEM_ASSERT(eid == i, "");
 
       Element &el = elements[eid];

mesh/ncmesh.hpp

@@ -29,10 +29,10 @@
 namespace mfem
 {
 
-/** Represents the index of an element to refine, plus a refinement type.
-    The refinement type is needed for anisotropic refinement of quads and hexes.
-    Bits 0,1 and 2 of 'ref_type' specify whether the element should be split
-    in the X, Y and Z directions, respectively (Z is ignored for quads). */
+/** Represents the index of an element to refine, plus a refinement type. The
+    refinement type is needed for anisotropic refinement of quads and hexes.
+    Bits 0,1 and 2 of 'ref_type' specify whether the element should be split in
+    the X, Y and Z directions, respectively (Z is ignored for quads). */
 struct Refinement
 {
    enum : char { X = 1, Y = 2, Z = 4, XY = 3, XZ = 5, YZ = 6, XYZ = 7 };
@@ -45,7 +45,6 @@ struct Refinement
       : index(index), ref_type(type) {}
 };
 
-
 /// Defines the position of a fine element within a coarse element.
 struct Embedding
 {
@@ -54,7 +53,8 @@ struct Embedding
 
    /** The (geom, matrix) pair determines the sub-element transformation for the
        fine element: CoarseFineTransformations::point_matrices[geom](matrix) is
-       the point matrix of the region within the coarse element reference domain.*/
+       the point matrix of the region within the coarse element reference
+       domain.*/
    unsigned geom : 4;
    unsigned matrix : 27;
 
@@ -66,7 +66,6 @@ struct Embedding
       : parent(elem), geom(geom), matrix(matrix), ghost(ghost) {}
 };
 
-
 /// Defines the coarse-fine transformations of all fine elements.
 struct CoarseFineTransformations
 {
@@ -96,24 +95,23 @@ void Swap(CoarseFineTransformations &a, CoarseFineTransformations &b);
 
 struct MatrixMap; // for internal use
 
-
-/** \brief A class for non-conforming AMR. The class is not used directly
- *  by the user, rather it is an extension of the Mesh class.
+/** \brief A class for non-conforming AMR. The class is not used directly by the
+ *  user, rather it is an extension of the Mesh class.
  *
  *  In general, the class is used by MFEM as follows:
  *
- *  1. NCMesh is constructed from elements of an existing Mesh. The elements
- *     are copied and become roots of the refinement hierarchy.
+ *  1. NCMesh is constructed from elements of an existing Mesh. The elements are
+ *     copied and become roots of the refinement hierarchy.
  *
  *  2. Some elements are refined with the Refine() method. Both isotropic and
  *     anisotropic refinements of quads/hexes are supported.
  *
- *  3. A new Mesh is created from NCMesh containing the leaf elements.
- *     This new Mesh may have non-conforming (hanging) edges and faces and
- *     is the one seen by the user.
+ *  3. A new Mesh is created from NCMesh containing the leaf elements. This new
+ *     Mesh may have non-conforming (hanging) edges and faces and is the one
+ *     seen by the user.
  *
- *  4. FiniteElementSpace asks NCMesh for a list of conforming, master and
- *     slave edges/faces and creates the conforming interpolation matrix P.
+ *  4. FiniteElementSpace asks NCMesh for a list of conforming, master and slave
+ *     edges/faces and creates the conforming interpolation matrix P.
  *
  *  5. A continuous/conforming solution is obtained by solving P'*A*P x = P'*b.
  *
@@ -121,8 +119,10 @@ struct MatrixMap; // for internal use
  */
 class NCMesh
 {
+protected:
+   NCMesh() = default;
 public:
-   //// Initialize with elements from an existing 'mesh'.
+   //// Initialize with elements from an existing Mesh.
    explicit NCMesh(const Mesh *mesh);
 
    /** Load from a stream. The id header is assumed to have been read already
@@ -155,8 +155,8 @@ public:
    virtual int GetNGhostElements() const { return 0; }
 
    /** Perform the given batch of refinements. Please note that in the presence
-       of anisotropic splits additional refinements may be necessary to keep
-       the mesh consistent. However, the function always performs at least the
+       of anisotropic splits additional refinements may be necessary to keep the
+       mesh consistent. However, the function always performs at least the
        requested refinements. */
    virtual void Refine(const Array<Refinement> &refinements);
 
@@ -172,14 +172,16 @@ public:
    const Table &GetDerefinementTable();
 
    /** Check derefinements returned by GetDerefinementTable and mark those that
-       can be done safely so that the maximum NC level condition is not violated.
-       On return, level_ok.Size() == deref_table.Size() and contains 0/1s. */
+       can be done safely so that the maximum NC level condition is not
+       violated. On return, level_ok.Size() == deref_table.Size() and contains
+       0/1s. */
    virtual void CheckDerefinementNCLevel(const Table &deref_table,
                                          Array<int> &level_ok, int max_nc_level);
 
-   /** Perform a subset of the possible derefinements (see GetDerefinementTable).
-       Note that if anisotropic refinements are present in the mesh, some of the
-       derefinements may have to be skipped to preserve mesh consistency. */
+   /** Perform a subset of the possible derefinements (see
+       GetDerefinementTable). Note that if anisotropic refinements are present
+       in the mesh, some of the derefinements may have to be skipped to preserve
+       mesh consistency. */
    virtual void Derefine(const Array<int> &derefs);
 
    // master/slave lists
@@ -340,9 +342,9 @@ public:
    const CoarseFineTransformations& GetRefinementTransforms() const;
 
    /** After derefinement, calculate the relations of previous fine elements
-       (some of which may no longer exist) to the current leaf elements.
-       Unlike for refinement, Derefine() may only be called once before this
-       function so there is no MarkFineLevel(). */
+       (some of which may no longer exist) to the current leaf elements. Unlike
+       for refinement, Derefine() may only be called once before this function
+       so there is no MarkFineLevel(). */
    const CoarseFineTransformations& GetDerefinementTransforms() const;
 
    /// Free all internal data created by the above three functions.
@@ -359,8 +361,8 @@ public:
    static void GridSfcOrdering2D(int width, int height,
                                  Array<int> &coords);
 
-   /** Return a space filling curve for a 3D rectangular grid of elements.
-       The Hilbert-curve-like algorithm works well for even dimensions. For odd
+   /** Return a space filling curve for a 3D rectangular grid of elements. The
+       Hilbert-curve-like algorithm works well for even dimensions. For odd
        width/height/depth it tends to produce some diagonal (edge-neighbor)
        steps. Even dimensions are recommended. */
    static void GridSfcOrdering3D(int width, int height, int depth,
@@ -428,17 +430,20 @@ public:
    /// Return the number of root elements.
    int GetNumRootElements() { return root_state.Size(); }
 
-   /// Return the distance of leaf 'i' from the root.
+   /// Return the distance of leaf @a i from the root.
    int GetElementDepth(int i) const;
 
    /** Return the size reduction compared to the root element (ignoring local
        stretching and curvature). */
    int GetElementSizeReduction(int i) const;
 
-   /// Return the faces and face attributes of leaf element 'i'.
+   /// Return the faces and face attributes of leaf element @a i.
    void GetElementFacesAttributes(int i, Array<int> &faces,
                                   Array<int> &fattr) const;
 
+   /// Set the attribute of leaf element @a i, which is a Mesh element index.
+   void SetAttribute(int i, int attr)
+   { elements[leaf_elements[i]].attribute = attr; }
 
    /** I/O: Print the mesh in "MFEM NC mesh v1.0" format. If @a comments is
        non-empty, it will be printed after the first line of the file, and each
@@ -459,8 +464,26 @@ public:
 
    int PrintMemoryDetail() const;
 
-   typedef std::int64_t RefCoord;
+   using RefCoord = std::int64_t;
 
+   static constexpr int MaxElemNodes =
+      8;       ///< Number of nodes an element can have
+   static constexpr int MaxElemEdges =
+      12;      ///< Number of edges an element can have
+   static constexpr int MaxElemFaces =
+      6;       ///< Number of faces an element can have
+   static constexpr int MaxElemChildren =
+      10;      ///< Number of children an element can have
+   static constexpr int MaxFaceNodes =
+      4;      ///< Number of faces an element can have
+
+   /**
+    * @brief Given a node index, return the vertex index associated
+    *
+    * @param node
+    * @return int
+    */
+   int GetNodeVertex(int node) { return nodes[node].vert_index; }
 
 protected: // non-public interface for the Mesh class
 
@@ -473,8 +496,8 @@ protected: // non-public interface for the Mesh class
        Face::index) after a new mesh was created from us. */
    void OnMeshUpdated(Mesh *mesh);
 
-   /** Delete top-level vertex coordinates if the Mesh became curved, e.g.,
-       by calling Mesh::SetCurvature or otherwise setting the Nodes. */
+   /** Delete top-level vertex coordinates if the Mesh became curved, e.g., by
+       calling Mesh::SetCurvature or otherwise setting the Nodes. */
    void MakeTopologyOnly() { coordinates.DeleteAll(); }
 
 protected: // implementation
@@ -485,23 +508,15 @@ protected: // implementation
    int Geoms; ///< bit mask of element geometries present, see InitGeomFlags()
    bool Legacy; ///< true if the mesh was loaded from the legacy v1.1 format
 
-   static const int MaxElemNodes =
-      8;       ///< Number of nodes of an element can have
-   static const int MaxElemEdges =
-      12;      ///< Number of edges of an element can have
-   static const int MaxElemFaces =
-      6;       ///< Number of faces of an element can have
-   static const int MaxElemChildren =
-      10;      ///< Number of children of an element can have
 
    /** A Node can hold a vertex, an edge, or both. Elements directly point to
-       their corner nodes, but edge nodes also exist and can be accessed using
-       a hash-table given their two end-point node IDs. All nodes can be
-       accessed in this way, with the exception of top-level vertex nodes.
-       When an element is being refined, the mid-edge nodes are readily
-       available with this mechanism. The new elements "sign in" to the nodes
-       by increasing the reference counts of their vertices and edges. The
-       parent element "signs off" its nodes by decrementing the ref counts. */
+       their corner nodes, but edge nodes also exist and can be accessed using a
+       hash-table given their two end-point node IDs. All nodes can be accessed
+       in this way, with the exception of top-level vertex nodes. When an
+       element is being refined, the mid-edge nodes are readily available with
+       this mechanism. The new elements "sign in" to the nodes by increasing the
+       reference counts of their vertices and edges. The parent element "signs
+       off" its nodes by decrementing the ref counts. */
    struct Node : public Hashed2
    {
       char vert_refc, edge_refc;
@@ -519,9 +534,9 @@ protected: // implementation
    };
 
    /** Similarly to nodes, faces can be accessed by hashing their four vertex
-       node IDs. A face knows about the one or two elements that are using it.
-       A face that is not on the boundary and only has one element referencing
-       it is either a master or a slave face. */
+       node IDs. A face knows about the one or two elements that are using it. A
+       face that is not on the boundary and only has one element referencing it
+       is either a master or a slave face. */
    struct Face : public Hashed4
    {
       int attribute; ///< boundary element attribute, -1 if internal face
@@ -539,11 +554,12 @@ protected: // implementation
 
       /// Return one of elem[0] or elem[1] and make sure the other is -1.
       int GetSingleElement() const;
+      int GetAttribute() const { return attribute; }
    };
 
-   /** This is an element in the refinement hierarchy. Each element has
-       either been refined and points to its children, or is a leaf and points
-       to its vertex nodes. */
+   /** This is an element in the refinement hierarchy. Each element has either
+       been refined and points to its children, or is a leaf and points to its
+       vertex nodes. */
    struct Element
    {
       char geom;     ///< Geometry::Type of the element (char for storage only)
@@ -559,46 +575,114 @@ protected: // implementation
          int child[MaxElemChildren]; ///< 2-10 children (if ref_type != 0)
       };
       int parent; ///< parent element, -1 if this is a root element, -2 if free'd
-
       Element(Geometry::Type geom, int attr);
 
       Geometry::Type Geom() const { return Geometry::Type(geom); }
       bool IsLeaf() const { return !ref_type && (parent != -2); }
+      int GetAttribute() const { return attribute; }
    };
 
 
    // primary data
-
    HashTable<Node> nodes; // associative container holding all Nodes
    HashTable<Face> faces; // associative container holding all Faces
-
    BlockArray<Element> elements; // storage for all Elements
    Array<int> free_element_ids;  // unused element ids - indices into 'elements'
+public:
+   /**
+    * @brief The number of Nodes.
+    *
+    * @return int
+    */
+   int GetNumNodes() const { return nodes.Size(); }
+   /**
+    * @brief Access a Node
+    *
+    * @param i Index of the node
+    * @return const Node&
+    */
+   const Node& GetNode(int i) const {return nodes[i]; }
+   /**
+    * @brief The number of faces
+    *
+    * @return int
+    */
+   int GetNumFaces() const { return faces.Size(); }
+   /**
+    * @brief Access a Face
+    *
+    * @param i Index of the face
+    * @return const Face&
+    */
+   const Face& GetFace(int i) const {return faces[i]; }
+   /**
+    * @brief The number of elements
+    *
+    * @return int
+    */
+   int GetNumElements() const { return elements.Size(); }
+   /**
+    * @brief Access an Element
+    *
+    * @param i Index of the element
+    * @return const Element&
+    */
+   const Element& GetElement(int i) const { return elements[i]; }
+
+   /**
+    * @brief Given a set of nodes defining a face, traverse the nodes structure
+    * to find the nodes that make up the parent face and replace the input nodes
+    * with the parent nodes. Additionally return the child index that the child
+    * face would be, relative to the discovered parent face.
+    * @details This method is concerned with the construction of an NCMesh
+    * structure for a d-1 manifold of an existing NCMesh. It forms a key element
+    * in a leaf -> root traversal of the parent ncmesh elements structure.
+    *
+    * @param[out] nodes The collection of nodes whose parent we are searching
+    * for
+    * @return int The child index corresponding to placing the face for the
+    * original nodes within the face defined by the returned parent nodes. If
+    * child index is -1, then the face is made up of root nodes, and nodes is
+    * unchanged.
+    */
+   int ParentFaceNodes(std::array<int, 4> &nodes) const;
+
+   /**
+    * @brief Method for finding the nodes associated to a @a face
+    * @return Nodes making up the face
+    */
+   std::array<int, 4> FindFaceNodes(int face) const;
+   std::array<int, 4> FindFaceNodes(const Face &fa) const;
+   /**
+    * @brief Backwards compatible method for finding the @a node associated to a
+    * @a face
+    */
+   MFEM_DEPRECATED void FindFaceNodes(int face, int node[4]) const;
+protected:
 
    /** Initial traversal state (~ element orientation) for each root element
-       NOTE: M = root_state.Size() is the number of root elements.
-       NOTE: the first M items of 'elements' is the coarse mesh. */
+       NOTE: M = root_state.Size() is the number of root elements. NOTE: the
+       first M items of 'elements' is the coarse mesh. */
    Array<int> root_state;
 
-   /** Coordinates of top-level vertices (organized as triples). If empty,
-       the Mesh is curved (Nodes != NULL) and NCMesh is topology-only. */
+   /** Coordinates of top-level vertices (organized as triples). If empty, the
+       Mesh is curved (Nodes != NULL) and NCMesh is topology-only. */
    Array<real_t> coordinates;
 
-
    // secondary data
-
    /** Apart from the primary data structure, which is the element/node/face
-       hierarchy, there is secondary data that is derived from the primary
-       data and needs to be updated when the primary data changes. Update()
-       takes care of that and needs to be called after each refinement and
+       hierarchy, there is secondary data that is derived from the primary data
+       and needs to be updated when the primary data changes. Update() takes
+       care of that and needs to be called after each refinement and
        derefinement. */
    virtual void Update();
 
    // set by UpdateLeafElements, UpdateVertices and OnMeshUpdated
    int NElements, NVertices, NEdges, NFaces;
 
-   // NOTE: the serial code understands the bare minimum about ghost elements and
-   // other ghost entities in order to be able to load parallel partial meshes
+   // NOTE: the serial code understands the bare minimum about ghost elements
+   // and other ghost entities in order to be able to load parallel partial
+   // meshes
    int NGhostElements, NGhostVertices, NGhostEdges, NGhostFaces;
 
    Array<int> leaf_elements; ///< finest elements, in Mesh ordering (+ ghosts)
@@ -623,19 +707,19 @@ protected: // implementation
        We must be careful to:
        1. Stay compatible with the conforming code, which expects top-level
           (original) vertices to be indexed first, otherwise GridFunctions
-          defined on a conforming mesh would no longer be valid when the
-          mesh is converted to an NC mesh.
+          defined on a conforming mesh would no longer be valid when the mesh is
+          converted to an NC mesh.
 
-       2. Make sure serial NCMesh is compatible with the parallel ParNCMesh,
-          so it is possible to read parallel partial solutions in serial code
+       2. Make sure serial NCMesh is compatible with the parallel ParNCMesh, so
+          it is possible to read parallel partial solutions in serial code
           (e.g., serial GLVis). This means handling ghost elements, if present.
 
-       3. Assign vertices in a globally consistent order for parallel meshes:
-          if two vertices i,j are shared by two ranks r1,r2, and i<j on r1,
-          then i<j on r2 as well. This is true for top-level vertices but also
-          for the remaining shared vertices thanks to the globally consistent
-          SFC ordering of the leaf elements. This property reduces communication
-          and simplifies ParNCMesh. */
+       3. Assign vertices in a globally consistent order for parallel meshes: if
+          two vertices i,j are shared by two ranks r1,r2, and i<j on r1, then
+          i<j on r2 as well. This is true for top-level vertices but also for
+          the remaining shared vertices thanks to the globally consistent SFC
+          ordering of the leaf elements. This property reduces communication and
+          simplifies ParNCMesh. */
    void UpdateVertices(); ///< update Vertex::index and vertex_nodeId
 
    /** Collect the leaf elements in leaf_elements, and the ghost elements in
@@ -646,8 +730,8 @@ protected: // implementation
                             int &counter);
 
    /** Try to find a space-filling curve friendly orientation of the root
-       elements: set 'root_state' based on the ordering of coarse elements.
-       Note that the coarse mesh itself must be ordered as an SFC by e.g.
+       elements: set 'root_state' based on the ordering of coarse elements. Note
+       that the coarse mesh itself must be ordered as an SFC by e.g.
        Mesh::GetGeckoElementOrdering. */
    void InitRootState(int root_count);
 
@@ -667,7 +751,6 @@ protected: // implementation
    /// Return true if the Element @a el is a ghost element.
    bool IsGhost(const Element &el) const { return el.rank != MyRank; }
 
-
    // refinement/derefinement
 
    Array<Refinement> ref_stack; ///< stack of scheduled refinements (temporary)
@@ -676,8 +759,8 @@ protected: // implementation
 
    Table derefinements; ///< possible derefinements, see GetDerefinementTable
 
-   /** Refine the element @a elem with the refinement @a ref_type
-       (c.f. Refinement::enum) */
+   /** Refine the element @a elem with the refinement @a ref_type (c.f.
+       Refinement::enum) */
    void RefineElement(int elem, char ref_type);
 
    /// Derefine the element @a elem, does nothing on leaf elements.
@@ -695,6 +778,7 @@ protected: // implementation
       }
       return elements.Append(el);
    }
+   int AddElement(Geometry::Type geom, int attr) { return AddElement(Element(geom,attr)); }
 
    // Free the element with index @a id.
    void FreeElement(int id)
@@ -826,6 +910,11 @@ protected: // implementation
 
    int GetMidFaceNode(int en1, int en2, int en3, int en4);
 
+   /**
+    * @brief Add references to all nodes, edges and faces of the element
+    *
+    * @param elem index into elements
+    */
    void ReferenceElement(int elem);
    void UnreferenceElement(int elem, Array<int> &elemFaces);
 
@@ -882,28 +971,28 @@ protected: // implementation
 
    // neighbors / element_vertex table
 
-   /** Return all vertex-, edge- and face-neighbors of a set of elements.
-       The neighbors are returned as a list (neighbors != NULL), as a set
+   /** Return all vertex-, edge- and face-neighbors of a set of elements. The
+       neighbors are returned as a list (neighbors != NULL), as a set
        (neighbor_set != NULL), or both. The sizes of the set arrays must match
-       that of leaf_elements. The function is intended to be used for large
-       sets of elements and its complexity is linear in the number of leaf
-       elements in the mesh. */
+       that of leaf_elements. The function is intended to be used for large sets
+       of elements and its complexity is linear in the number of leaf elements
+       in the mesh. */
    void FindSetNeighbors(const Array<char> &elem_set,
                          Array<int> *neighbors, /* append */
                          Array<char> *neighbor_set = NULL);
 
-   /** Return all vertex-, edge- and face-neighbors of a single element.
-       You can limit the number of elements being checked using 'search_set'.
-       The complexity of the function is linear in the size of the search set.*/
+   /** Return all vertex-, edge- and face-neighbors of a single element. You can
+       limit the number of elements being checked using 'search_set'. The
+       complexity of the function is linear in the size of the search set.*/
    void FindNeighbors(int elem,
                       Array<int> &neighbors, /* append */
                       const Array<int> *search_set = NULL);
 
-   /** Expand a set of elements by all vertex-, edge- and face-neighbors.
-       The output array 'expanded' will contain all items from 'elems'
-       (provided they are in 'search_set') plus their neighbors. The neighbor
-       search can be limited to the optional search set. The complexity is
-       linear in the sum of the sizes of 'elems' and 'search_set'. */
+   /** Expand a set of elements by all vertex-, edge- and face-neighbors. The
+       output array 'expanded' will contain all items from 'elems' (provided
+       they are in 'search_set') plus their neighbors. The neighbor search can
+       be limited to the optional search set. The complexity is linear in the
+       sum of the sizes of 'elems' and 'search_set'. */
    void NeighborExpand(const Array<int> &elems,
                        Array<int> &expanded,
                        const Array<int> *search_set = NULL);
@@ -981,18 +1070,17 @@ protected: // implementation
    /** @brief The PointMatrix stores the coordinates of the slave face using the
        master face coordinate as reference.
 
-       In 2D, the point matrix has the orientation of the parent
-       edge, so its columns need to be flipped when applying it, see
+       In 2D, the point matrix has the orientation of the parent edge, so its
+       columns need to be flipped when applying it, see
        ApplyLocalSlaveTransformation.
 
-       In 3D, the orientation part of Elem2Inf is encoded in the point
-       matrix.
+       In 3D, the orientation part of Elem2Inf is encoded in the point matrix.
 
-       The following transformation gives the relation between the
-       reference quad face coordinates (xi, eta) in [0,1]^2, and the fine quad
-       face coordinates (x, y):
-       x = a0*(1-xi)*(1-eta) + a1*xi*(1-eta) + a2*xi*eta + a3*(1-xi)*eta
-       y = b0*(1-xi)*(1-eta) + b1*xi*(1-eta) + b2*xi*eta + b3*(1-xi)*eta
+       The following transformation gives the relation between the reference
+       quad face coordinates (xi, eta) in [0,1]^2, and the fine quad face
+       coordinates (x, y):
+         x = a0*(1-xi)*(1-eta) + a1*xi*(1-eta) + a2*xi*eta + a3*(1-xi)*eta
+         y = b0*(1-xi)*(1-eta) + b1*xi*(1-eta) + b2*xi*eta + b3*(1-xi)*eta
    */
    struct PointMatrix
    {
@@ -1054,7 +1142,7 @@ protected: // implementation
    void GetPointMatrix(Geometry::Type geom, const char* ref_path,
                        DenseMatrix& matrix) const;
 
-   typedef std::map<std::string, int> RefPathMap;
+   using RefPathMap = std::map<std::string, int>;
 
    void TraverseRefinements(int elem, int coarse_index,
                             std::string &ref_path, RefPathMap &map) const;
@@ -1085,15 +1173,15 @@ protected: // implementation
 
    int GetEdgeMaster(int node) const;
 
-   void FindFaceNodes(int face, int node[4]) const;
-
    /**
     * @brief Return the number of splits of this edge that have occurred in the
-    * NCMesh. If zero, this means the segment is not the master of any other segments.
+    * NCMesh. If zero, this means the segment is not the master of any other
+    * segments.
     *
     * @param vn1 The first vertex making up the segment
     * @param vn2 The second vertex making up the segment
-    * @return int The depth of splits of this segment that are present in the mesh.
+    * @return int The depth of splits of this segment that are present in the
+    * mesh.
     */
    int EdgeSplitLevel(int vn1, int vn2) const;
    /**
@@ -1104,13 +1192,14 @@ protected: // implementation
     * @param vn1 The first vertex making up the triangle
     * @param vn2 The second vertex making up the triangle
     * @param vn3 The third vertex making up the triangle
-    * @return int The depth of splits of this triangle that are present in the mesh.
+    * @return int The depth of splits of this triangle that are present in the
+    * mesh.
     */
    int TriFaceSplitLevel(int vn1, int vn2, int vn3) const;
    /**
     * @brief Computes the number of horizontal and vertical splits of this quad
-    * that have occurred in the NCMesh. If zero, this means the quad is not
-    * the master of any other quad.
+    * that have occurred in the NCMesh. If zero, this means the quad is not the
+    * master of any other quad.
     *
     * @param vn1 The first vertex making up the quad
     * @param vn2 The second vertex making up the quad
@@ -1123,8 +1212,8 @@ protected: // implementation
                            int& h_level, int& v_level) const;
    /**
     * @brief Returns the total number of splits of this quad that have occurred
-    * in the NCMesh. If zero, this means the quad is not
-    * the master of any other quad.
+    * in the NCMesh. If zero, this means the quad is not the master of any other
+    * quad.
     * @details This is a convenience wrapper that sums the horizontal and
     * vertical levels from the full method.
     *
@@ -1141,6 +1230,17 @@ protected: // implementation
    void CountSplits(int elem, int splits[3]) const;
    void GetLimitRefinements(Array<Refinement> &refinements, int max_level);
 
+   // Checker helpers
+
+   static void CheckSupportedGeom(Geometry::Type geom)
+   {
+      MFEM_VERIFY(geom == Geometry::SEGMENT ||
+                  geom == Geometry::TRIANGLE || geom == Geometry::SQUARE ||
+                  geom == Geometry::CUBE || geom == Geometry::PRISM ||
+                  geom == Geometry::PYRAMID || geom == Geometry::TETRAHEDRON,
+                  "Element type " << geom << " is not supported by NCMesh.");
+   }
+
 
    // I/O
 
@@ -1149,8 +1249,8 @@ protected: // implementation
    /// Load the vertex parent hierarchy from a mesh file.
    void LoadVertexParents(std::istream &input);
 
-   /** Print the "boundary" section of the mesh file.
-       If out == NULL, only return the number of boundary elements. */
+   /** Print the "boundary" section of the mesh file. If out == NULL, only
+       return the number of boundary elements. */
    int PrintBoundary(std::ostream *out) const;
    /// Load the "boundary" section of the mesh file.
    void LoadBoundary(std::istream &input);
@@ -1185,6 +1285,7 @@ protected: // implementation
 
       bool initialized;
       GeomInfo() : initialized(false) {}
+      GeomInfo(Geometry::Type geom) : GeomInfo() { InitGeom(geom); }
       void InitGeom(Geometry::Type geom);
    };
 
@@ -1199,6 +1300,8 @@ public:
    friend class ParNCMesh; // for ParNCMesh::ElementSet
    friend struct MatrixMap;
    friend struct PointMatrixHash;
+   friend class NCSubMesh; // for faces, nodes
+   friend class ParNCSubMesh; // for faces, nodes
 };
 
 }

mesh/ncmesh_tables.hpp

@@ -9,14 +9,13 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#ifndef MFEM_NCMESH_TABLES
+#define MFEM_NCMESH_TABLES
+
 namespace mfem
 {
 
-namespace // make everything static
-{
-
-const int ref_type_num_children[8] = { 0, 2, 2, 4, 2, 4, 4, 8 };
-
+static constexpr int ref_type_num_children[8] = { 0, 2, 2, 4, 2, 4, 4, 8 };
 
 // derefinement tables
 // The first n numbers in each line are the refined elements that contain
@@ -24,14 +23,14 @@ const int ref_type_num_children[8] = { 0, 2, 2, 4, 2, 4, 4, 8 };
 // are the refined elements that contain the faces attributes of the parent
 // element.
 
-const int quad_deref_table[3][4 + 4] =
+static constexpr int quad_deref_table[3][4 + 4] =
 {
    { 0, 1, 1, 0, /**/ 1, 1, 0, 0 }, // 1 - X
    { 0, 0, 1, 1, /**/ 0, 0, 1, 1 }, // 2 - Y
    { 0, 1, 2, 3, /**/ 1, 1, 3, 3 }  // 3 - iso
 };
 
-const int hex_deref_table[7][8 + 6] =
+static constexpr int hex_deref_table[7][8 + 6] =
 {
    { 0, 1, 1, 0, 0, 1, 1, 0, /**/ 1, 1, 1, 0, 0, 0 }, // 1 - X
    { 0, 0, 1, 1, 0, 0, 1, 1, /**/ 0, 0, 0, 1, 1, 1 }, // 2 - Y
@@ -42,7 +41,7 @@ const int hex_deref_table[7][8 + 6] =
    { 0, 1, 2, 3, 4, 5, 6, 7, /**/ 1, 1, 1, 7, 7, 7 }  // 7 - iso
 };
 
-const int prism_deref_table[7][6 + 5] =
+static constexpr int prism_deref_table[7][6 + 5] =
 {
    {-1,-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 1
    {-1,-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 2
@@ -53,7 +52,7 @@ const int prism_deref_table[7][6 + 5] =
    { 0, 1, 2, 4, 5, 6, /**/  0, 5, 0, 5, 0 }  // 7 - iso
 };
 
-const int pyramid_deref_table[7][5 + 5] =
+static constexpr int pyramid_deref_table[7][5 + 5] =
 {
    {-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 1
    {-1,-1,-1,-1,-1, /**/ -1,-1,-1,-1,-1 }, // 2
@@ -66,19 +65,19 @@ const int pyramid_deref_table[7][5 + 5] =
 
 // child ordering tables
 
-const char quad_hilbert_child_order[8][4] =
+static constexpr char quad_hilbert_child_order[8][4] =
 {
    {0,1,2,3}, {0,3,2,1}, {1,2,3,0}, {1,0,3,2},
    {2,3,0,1}, {2,1,0,3}, {3,0,1,2}, {3,2,1,0}
 };
 
-const char quad_hilbert_child_state[8][4] =
+static constexpr char quad_hilbert_child_state[8][4] =
 {
    {1,0,0,5}, {0,1,1,4}, {3,2,2,7}, {2,3,3,6},
    {5,4,4,1}, {4,5,5,0}, {7,6,6,3}, {6,7,7,2}
 };
 
-const char hex_hilbert_child_order[24][8] =
+static constexpr char hex_hilbert_child_order[24][8] =
 {
    {0,1,2,3,7,6,5,4}, {0,3,7,4,5,6,2,1}, {0,4,5,1,2,6,7,3},
    {1,0,3,2,6,7,4,5}, {1,2,6,5,4,7,3,0}, {1,5,4,0,3,7,6,2},
@@ -90,7 +89,7 @@ const char hex_hilbert_child_order[24][8] =
    {7,3,2,6,5,1,0,4}, {7,4,0,3,2,1,5,6}, {7,6,5,4,0,1,2,3}
 };
 
-const char hex_hilbert_child_state[24][8] =
+static constexpr char hex_hilbert_child_state[24][8] =
 {
    {1,2,2,7,7,21,21,17},     {2,0,0,22,22,16,16,8},    {0,1,1,15,15,6,6,23},
    {4,5,5,10,10,18,18,14},   {5,3,3,19,19,13,13,11},   {3,4,4,12,12,9,9,20},
@@ -104,27 +103,26 @@ const char hex_hilbert_child_state[24][8] =
 
 
 // child/parent reference domain transforms
-
-typedef NCMesh::RefCoord RefCoord;
+using RefCoord = NCMesh::RefCoord;
 
 // reference domain coordinates as fixed point numbers
-const RefCoord T_HALF = (1ll << 59);
-const RefCoord T_ONE = (1ll << 60);
-const RefCoord T_TWO = (1ll << 61);
+static constexpr RefCoord T_HALF = (1ll << 59);
+static constexpr RefCoord T_ONE = (1ll << 60);
+static constexpr RefCoord T_TWO = (1ll << 61);
 
 // (scaling factors have a different fixed point multiplier)
-const RefCoord S_HALF = 1;
-const RefCoord S_ONE = 2;
-const RefCoord S_TWO = 4;
+static constexpr RefCoord S_HALF = 1;
+static constexpr RefCoord S_ONE = 2;
+static constexpr RefCoord S_TWO = 4;
 
-const RefCoord tri_corners[3][3] =
+static constexpr RefCoord tri_corners[3][3] =
 {
    {    0,     0, 0},
    {T_ONE,     0, 0},
    {    0, T_ONE, 0}
 };
 
-const RefCoord quad_corners[4][3] =
+static constexpr RefCoord quad_corners[4][3] =
 {
    {    0,     0, 0},
    {T_ONE,     0, 0},
@@ -132,7 +130,7 @@ const RefCoord quad_corners[4][3] =
    {    0, T_ONE, 0}
 };
 
-const RefCoord hex_corners[8][3] =
+static constexpr RefCoord hex_corners[8][3] =
 {
    {    0,     0,     0},
    {T_ONE,     0,     0},
@@ -144,7 +142,7 @@ const RefCoord hex_corners[8][3] =
    {    0, T_ONE, T_ONE}
 };
 
-const RefCoord prism_corners[6][3] =
+static constexpr RefCoord prism_corners[6][3] =
 {
    {    0,     0,     0},
    {T_ONE,     0,     0},
@@ -154,7 +152,7 @@ const RefCoord prism_corners[6][3] =
    {    0, T_ONE, T_ONE}
 };
 
-const RefCoord pyramid_corners[5][3] =
+static constexpr RefCoord pyramid_corners[5][3] =
 {
    {    0,     0,     0},
    {T_ONE,     0,     0},
@@ -164,7 +162,7 @@ const RefCoord pyramid_corners[5][3] =
 };
 
 typedef RefCoord RefPoint[3];
-const RefPoint* geom_corners[8] =
+static const RefPoint* geom_corners[8] =
 {
    NULL, // point
    NULL, // segment
@@ -190,31 +188,31 @@ struct RefTrf
    }
 };
 
-const RefTrf quad_parent_rt1[2] =
+static constexpr RefTrf quad_parent_rt1[2] =
 {
    { {S_HALF, S_ONE, 0}, {     0, 0, 0} },
    { {S_HALF, S_ONE, 0}, {T_HALF, 0, 0} }
 };
 
-const RefTrf quad_child_rt1[2] =
+static constexpr RefTrf quad_child_rt1[2] =
 {
    { {S_TWO, S_ONE, 0}, {     0, 0, 0} },
    { {S_TWO, S_ONE, 0}, {-T_ONE, 0, 0} }
 };
 
-const RefTrf quad_parent_rt2[2] =
+static constexpr RefTrf quad_parent_rt2[2] =
 {
    { {S_ONE, S_HALF, 0}, {0,      0, 0} },
    { {S_ONE, S_HALF, 0}, {0, T_HALF, 0} }
 };
 
-const RefTrf quad_child_rt2[2] =
+static constexpr RefTrf quad_child_rt2[2] =
 {
    { {S_ONE, S_TWO, 0}, {0,      0, 0} },
    { {S_ONE, S_TWO, 0}, {0, -T_ONE, 0} }
 };
 
-const RefTrf quad_parent_rt3[4] =
+static constexpr RefTrf quad_parent_rt3[4] =
 {
    { {S_HALF, S_HALF, 0}, {     0,      0, 0} },
    { {S_HALF, S_HALF, 0}, {T_HALF,      0, 0} },
@@ -222,7 +220,7 @@ const RefTrf quad_parent_rt3[4] =
    { {S_HALF, S_HALF, 0}, {     0, T_HALF, 0} }
 };
 
-const RefTrf quad_child_rt3[4] =
+static constexpr RefTrf quad_child_rt3[4] =
 {
    { {S_TWO, S_TWO, 0}, {     0,      0, 0} },
    { {S_TWO, S_TWO, 0}, {-T_ONE,      0, 0} },
@@ -230,7 +228,7 @@ const RefTrf quad_child_rt3[4] =
    { {S_TWO, S_TWO, 0}, {     0, -T_ONE, 0} }
 };
 
-const RefTrf* quad_parent[4] =
+static const RefTrf* quad_parent[4] =
 {
    NULL,
    quad_parent_rt1,
@@ -238,7 +236,7 @@ const RefTrf* quad_parent[4] =
    quad_parent_rt3
 };
 
-const RefTrf* quad_child[4] =
+static const RefTrf* quad_child[4] =
 {
    NULL,
    quad_child_rt1,
@@ -246,31 +244,31 @@ const RefTrf* quad_child[4] =
    quad_child_rt3
 };
 
-const RefTrf hex_parent_rt1[2] =
+static constexpr RefTrf hex_parent_rt1[2] =
 {
    { {S_HALF, S_ONE, S_ONE}, {     0, 0, 0} },
    { {S_HALF, S_ONE, S_ONE}, {T_HALF, 0, 0} }
 };
 
-const RefTrf hex_child_rt1[2] =
+static constexpr RefTrf hex_child_rt1[2] =
 {
    { {S_TWO, S_ONE, S_ONE}, {     0, 0, 0} },
    { {S_TWO, S_ONE, S_ONE}, {-T_ONE, 0, 0} }
 };
 
-const RefTrf hex_parent_rt2[2] =
+static constexpr RefTrf hex_parent_rt2[2] =
 {
    { {S_ONE, S_HALF, S_ONE}, {0,      0, 0} },
    { {S_ONE, S_HALF, S_ONE}, {0, T_HALF, 0} }
 };
 
-const RefTrf hex_child_rt2[2] =
+static constexpr RefTrf hex_child_rt2[2] =
 {
    { {S_ONE, S_TWO, S_ONE}, {0,      0, 0} },
    { {S_ONE, S_TWO, S_ONE}, {0, -T_ONE, 0} }
 };
 
-const RefTrf hex_parent_rt3[4] =
+static constexpr RefTrf hex_parent_rt3[4] =
 {
    { {S_HALF, S_HALF, S_ONE}, {     0,      0, 0} },
    { {S_HALF, S_HALF, S_ONE}, {T_HALF,      0, 0} },
@@ -278,7 +276,7 @@ const RefTrf hex_parent_rt3[4] =
    { {S_HALF, S_HALF, S_ONE}, {     0, T_HALF, 0} }
 };
 
-const RefTrf hex_child_rt3[4] =
+static constexpr RefTrf hex_child_rt3[4] =
 {
    { {S_TWO, S_TWO, S_ONE}, {     0,      0, 0} },
    { {S_TWO, S_TWO, S_ONE}, {-T_ONE,      0, 0} },
@@ -286,19 +284,19 @@ const RefTrf hex_child_rt3[4] =
    { {S_TWO, S_TWO, S_ONE}, {     0, -T_ONE, 0} }
 };
 
-const RefTrf hex_parent_rt4[2] =
+static constexpr RefTrf hex_parent_rt4[2] =
 {
    { {S_ONE, S_ONE, S_HALF}, {0, 0,      0} },
    { {S_ONE, S_ONE, S_HALF}, {0, 0, T_HALF} }
 };
 
-const RefTrf hex_child_rt4[2] =
+static constexpr RefTrf hex_child_rt4[2] =
 {
    { {S_ONE, S_ONE, S_TWO}, {0, 0,      0} },
    { {S_ONE, S_ONE, S_TWO}, {0, 0, -T_ONE} }
 };
 
-const RefTrf hex_parent_rt5[4] =
+static constexpr RefTrf hex_parent_rt5[4] =
 {
    { {S_HALF, S_ONE, S_HALF}, {     0, 0,      0} },
    { {S_HALF, S_ONE, S_HALF}, {T_HALF, 0,      0} },
@@ -306,7 +304,7 @@ const RefTrf hex_parent_rt5[4] =
    { {S_HALF, S_ONE, S_HALF}, {     0, 0, T_HALF} }
 };
 
-const RefTrf hex_child_rt5[4] =
+static constexpr RefTrf hex_child_rt5[4] =
 {
    { {S_TWO, S_ONE, S_TWO}, {     0, 0,      0} },
    { {S_TWO, S_ONE, S_TWO}, {-T_ONE, 0,      0} },
@@ -314,7 +312,7 @@ const RefTrf hex_child_rt5[4] =
    { {S_TWO, S_ONE, S_TWO}, {     0, 0, -T_ONE} }
 };
 
-const RefTrf hex_parent_rt6[4] =
+static constexpr RefTrf hex_parent_rt6[4] =
 {
    { {S_ONE, S_HALF, S_HALF}, {0,      0,      0} },
    { {S_ONE, S_HALF, S_HALF}, {0, T_HALF,      0} },
@@ -322,7 +320,7 @@ const RefTrf hex_parent_rt6[4] =
    { {S_ONE, S_HALF, S_HALF}, {0, T_HALF, T_HALF} }
 };
 
-const RefTrf hex_child_rt6[4] =
+static constexpr RefTrf hex_child_rt6[4] =
 {
    { {S_ONE, S_TWO, S_TWO}, {0,      0,      0} },
    { {S_ONE, S_TWO, S_TWO}, {0, -T_ONE,      0} },
@@ -330,7 +328,7 @@ const RefTrf hex_child_rt6[4] =
    { {S_ONE, S_TWO, S_TWO}, {0, -T_ONE, -T_ONE} }
 };
 
-const RefTrf hex_parent_rt7[8] =
+static constexpr RefTrf hex_parent_rt7[8] =
 {
    { {S_HALF, S_HALF, S_HALF}, {     0,      0,      0} },
    { {S_HALF, S_HALF, S_HALF}, {T_HALF,      0,      0} },
@@ -342,7 +340,7 @@ const RefTrf hex_parent_rt7[8] =
    { {S_HALF, S_HALF, S_HALF}, {     0, T_HALF, T_HALF} }
 };
 
-const RefTrf hex_child_rt7[8] =
+static constexpr RefTrf hex_child_rt7[8] =
 {
    { {S_TWO, S_TWO, S_TWO}, {     0,      0,      0} },
    { {S_TWO, S_TWO, S_TWO}, {-T_ONE,      0,      0} },
@@ -354,7 +352,7 @@ const RefTrf hex_child_rt7[8] =
    { {S_TWO, S_TWO, S_TWO}, {     0, -T_ONE, -T_ONE} }
 };
 
-const RefTrf* hex_parent[8] =
+static const RefTrf* hex_parent[8] =
 {
    NULL,
    hex_parent_rt1,
@@ -366,7 +364,7 @@ const RefTrf* hex_parent[8] =
    hex_parent_rt7
 };
 
-const RefTrf* hex_child[8] =
+static const RefTrf* hex_child[8] =
 {
    NULL,
    hex_child_rt1,
@@ -378,7 +376,7 @@ const RefTrf* hex_child[8] =
    hex_child_rt7
 };
 
-const RefTrf tri_parent_rt3[4] =
+static constexpr RefTrf tri_parent_rt3[4] =
 {
    { { S_HALF,  S_HALF, 0}, {     0,      0, 0} },
    { { S_HALF,  S_HALF, 0}, {T_HALF,      0, 0} },
@@ -386,7 +384,7 @@ const RefTrf tri_parent_rt3[4] =
    { {-S_HALF, -S_HALF, 0}, {T_HALF, T_HALF, 0} }
 };
 
-const RefTrf tri_child_rt3[4] =
+static constexpr RefTrf tri_child_rt3[4] =
 {
    { { S_TWO,  S_TWO, 0}, {     0,      0, 0} },
    { { S_TWO,  S_TWO, 0}, {-T_ONE,      0, 0} },
@@ -394,19 +392,19 @@ const RefTrf tri_child_rt3[4] =
    { {-S_TWO, -S_TWO, 0}, { T_ONE,  T_ONE, 0} }
 };
 
-const RefTrf* tri_parent[4] =
+static const RefTrf* tri_parent[4] =
 {
    NULL, NULL, NULL,
    tri_parent_rt3
 };
 
-const RefTrf* tri_child[4] =
+static const RefTrf* tri_child[4] =
 {
    NULL, NULL, NULL,
    tri_child_rt3
 };
 
-const RefTrf prism_parent_rt3[4] =
+static constexpr RefTrf prism_parent_rt3[4] =
 {
    { { S_HALF,  S_HALF, S_ONE}, {     0,      0, 0} },
    { { S_HALF,  S_HALF, S_ONE}, {T_HALF,      0, 0} },
@@ -414,7 +412,7 @@ const RefTrf prism_parent_rt3[4] =
    { {-S_HALF, -S_HALF, S_ONE}, {T_HALF, T_HALF, 0} }
 };
 
-const RefTrf prism_child_rt3[4] =
+static constexpr RefTrf prism_child_rt3[4] =
 {
    { { S_TWO,  S_TWO, S_ONE}, {     0,      0, 0} },
    { { S_TWO,  S_TWO, S_ONE}, {-T_ONE,      0, 0} },
@@ -422,19 +420,19 @@ const RefTrf prism_child_rt3[4] =
    { {-S_TWO, -S_TWO, S_ONE}, { T_ONE,  T_ONE, 0} }
 };
 
-const RefTrf prism_parent_rt4[2] =
+static constexpr RefTrf prism_parent_rt4[2] =
 {
    { {S_ONE, S_ONE, S_HALF}, {0, 0,      0} },
    { {S_ONE, S_ONE, S_HALF}, {0, 0, T_HALF} }
 };
 
-const RefTrf prism_child_rt4[2] =
+static constexpr RefTrf prism_child_rt4[2] =
 {
    { {S_ONE, S_ONE, S_TWO}, {0, 0,      0} },
    { {S_ONE, S_ONE, S_TWO}, {0, 0, -T_ONE} }
 };
 
-const RefTrf prism_parent_rt7[8] =
+static constexpr RefTrf prism_parent_rt7[8] =
 {
    { { S_HALF,  S_HALF, S_HALF}, {     0,      0,      0} },
    { { S_HALF,  S_HALF, S_HALF}, {T_HALF,      0,      0} },
@@ -446,7 +444,7 @@ const RefTrf prism_parent_rt7[8] =
    { {-S_HALF, -S_HALF, S_HALF}, {T_HALF, T_HALF, T_HALF} }
 };
 
-const RefTrf prism_child_rt7[8] =
+static constexpr RefTrf prism_child_rt7[8] =
 {
    { { S_TWO,  S_TWO, S_TWO}, {     0,      0,      0} },
    { { S_TWO,  S_TWO, S_TWO}, {-T_ONE,      0,      0} },
@@ -458,7 +456,7 @@ const RefTrf prism_child_rt7[8] =
    { {-S_TWO, -S_TWO, S_TWO}, { T_ONE,  T_ONE, -T_ONE} }
 };
 
-const RefTrf* prism_parent[8] =
+static const RefTrf* prism_parent[8] =
 {
    NULL, NULL, NULL,
    prism_parent_rt3,
@@ -467,7 +465,7 @@ const RefTrf* prism_parent[8] =
    prism_parent_rt7
 };
 
-const RefTrf* prism_child[8] =
+static const RefTrf* prism_child[8] =
 {
    NULL, NULL, NULL,
    prism_child_rt3,
@@ -476,7 +474,7 @@ const RefTrf* prism_child[8] =
    prism_child_rt7
 };
 
-const RefTrf** geom_parent[7] =
+static const RefTrf** geom_parent[7] =
 {
    NULL,
    NULL,
@@ -487,7 +485,7 @@ const RefTrf** geom_parent[7] =
    prism_parent
 };
 
-const RefTrf** geom_child[7] =
+static const RefTrf** geom_child[7] =
 {
    NULL,
    NULL,
@@ -498,7 +496,6 @@ const RefTrf** geom_child[7] =
    prism_child
 };
 
-
-} // namespace
-
 } // namespace mfem
+
+#endif // MFEM_NCMESH_TABLES

mesh/pmesh.hpp

@@ -109,8 +109,8 @@ protected:
    // Determine sedge_ledge and sface_lface.
    void FinalizeParTopo();
 
-   // Mark all tets to ensure consistency across MPI tasks; also mark the
-   // shared and boundary triangle faces using the consistently marked tets.
+   // Mark all tets to ensure consistency across MPI tasks; also mark the shared
+   // and boundary triangle faces using the consistently marked tets.
    void MarkTetMeshForRefinement(const DSTable &v_to_v) override;
 
    /// Return a number(0-1) identifying how the given edge has been split
@@ -337,12 +337,12 @@ public:
       have_face_nbr_data(false), pncmesh(NULL) { }
 
    /// Create a parallel mesh by partitioning a serial Mesh.
-   /** The mesh is partitioned automatically or using external partitioning
-       data (the optional parameter 'partitioning_[i]' contains the desired MPI
-       rank for element 'i'). Automatic partitioning uses METIS for conforming
-       meshes and quick space-filling curve equipartitioning for nonconforming
-       meshes (elements of nonconforming meshes should ideally be ordered as a
-       sequence of face-neighbors). */
+   /** The mesh is partitioned automatically or using external partitioning data
+       (the optional parameter 'partitioning_[i]' contains the desired MPI rank
+       for element 'i'). Automatic partitioning uses METIS for conforming meshes
+       and quick space-filling curve equipartitioning for nonconforming meshes
+       (elements of nonconforming meshes should ideally be ordered as a sequence
+       of face-neighbors). */
    ParMesh(MPI_Comm comm, Mesh &mesh, const int *partitioning_ = nullptr,
            int part_method = 1);
 
@@ -446,11 +446,42 @@ public:
    int GroupNTriangles(int group) const { return group_stria.RowSize(group-1); }
    int GroupNQuadrilaterals(int group) const { return group_squad.RowSize(group-1); }
 
+   /**
+    * @brief Accessors for entities within a shared group structure.
+    * @details For all vertex/edge/face the two argument version returns the
+    * local index, for those entities with an orientation. The two out parameter
+    * version additionally returns an orientation to use in manipulating the
+    * entity.
+    *
+    * @param group The communicator group's indices
+    * @param i the index within the group
+    * @return int The local index of the entity
+    */
    int GroupVertex(int group, int i) const
    { return svert_lvert[group_svert.GetRow(group-1)[i]]; }
    void GroupEdge(int group, int i, int &edge, int &o) const;
    void GroupTriangle(int group, int i, int &face, int &o) const;
    void GroupQuadrilateral(int group, int i, int &face, int &o) const;
+   int GroupEdge(int group, int i) const
+   {
+      int e, o;
+      GroupEdge(group, i, e, o);
+      return e;
+   }
+   int GroupTriangle(int group, int i) const
+   {
+      int f, o;
+      GroupTriangle(group, i, f, o);
+      return f;
+   }
+   int GroupQuadrilateral(int group, int i) const
+   {
+      int f, o;
+      GroupQuadrilateral(group, i, f, o);
+      return f;
+   }
+
+
    ///@}
 
    /**
@@ -496,18 +527,15 @@ public:
    void GenerateOffsets(int N, HYPRE_BigInt loc_sizes[],
                         Array<HYPRE_BigInt> *offsets[]) const;
 
-   /** Return true if the face is interior or shared. In parallel, this
-       method only works if the face neighbor data is exchanged. */
-   inline bool FaceIsTrueInterior(int FaceNo) const { return Mesh::FaceIsTrueInterior(FaceNo); }
-
+   using Mesh::FaceIsTrueInterior;
    void ExchangeFaceNbrData();
    void ExchangeFaceNbrNodes();
 
    void SetCurvature(int order, bool discont = false, int space_dim = -1,
                      int ordering = 1) override;
 
-   /** Replace the internal node GridFunction with a new GridFunction defined
-       on the given FiniteElementSpace. The new node coordinates are projected
+   /** Replace the internal node GridFunction with a new GridFunction defined on
+       the given FiniteElementSpace. The new node coordinates are projected
        (derived) from the current nodes/vertices. */
    void SetNodalFESpace(FiniteElementSpace *nfes) override;
    void SetNodalFESpace(ParFiniteElementSpace *npfes);
@@ -571,15 +599,15 @@ public:
                                       IsoparametricTransformation &ElTr2,
                                       int mask = 31) const override;
 
-   /// @brief Get the FaceElementTransformations for the given shared face
-   /// (edge 2D) using the shared face index @a sf. @a fill2 specify if the
-   /// information for elem2 of the face should be computed or not.
-   /// In the returned object, 1 and 2 refer to the local and the neighbor
-   /// elements, respectively.
+   /// @brief Get the FaceElementTransformations for the given shared face (edge
+   /// 2D) using the shared face index @a sf. @a fill2 specify if the
+   /// information for elem2 of the face should be computed or not. In the
+   /// returned object, 1 and 2 refer to the local and the neighbor elements,
+   /// respectively.
    ///
    /// @note The returned object is owned by the class and is shared, i.e.,
-   /// calling this function resets pointers obtained from previous calls.
-   /// Also, the returned object should NOT be deleted by the caller.
+   /// calling this function resets pointers obtained from previous calls. Also,
+   /// the returned object should NOT be deleted by the caller.
    FaceElementTransformations *
    GetSharedFaceTransformations(int sf, bool fill2 = true);
 
@@ -591,15 +619,14 @@ public:
                                      IsoparametricTransformation &ElTr2,
                                      bool fill2 = true) const;
 
-   /// @brief Get the FaceElementTransformations for the given shared face
-   /// (edge 2D) using the face index @a FaceNo. @a fill2 specify if the
-   /// information for elem2 of the face should be computed or not.
-   /// In the returned object, 1 and 2 refer to the local and the neighbor
-   /// elements, respectively.
+   /// @brief Get the FaceElementTransformations for the given shared face (edge
+   /// 2D) using the face index @a FaceNo. @a fill2 specify if the information
+   /// for elem2 of the face should be computed or not. In the returned object,
+   /// 1 and 2 refer to the local and the neighbor elements, respectively.
    ///
    /// @note The returned object is owned by the class and is shared, i.e.,
-   /// calling this function resets pointers obtained from previous calls.
-   /// Also, the returned object should NOT be deleted by the caller.
+   /// calling this function resets pointers obtained from previous calls. Also,
+   /// the returned object should NOT be deleted by the caller.
    FaceElementTransformations *
    GetSharedFaceTransformationsByLocalIndex(int FaceNo, bool fill2 = true);
 
@@ -615,8 +642,8 @@ public:
    /// neighbor.
    ///
    /// @note The returned object is owned by the class and is shared, i.e.,
-   /// calling this function resets pointers obtained from previous calls.
-   /// Also, the returned object should NOT be deleted by the caller.
+   /// calling this function resets pointers obtained from previous calls. Also,
+   /// the returned object should NOT be deleted by the caller.
    ElementTransformation *GetFaceNbrElementTransformation(int FaceNo);
 
    /// @brief Variant of GetFaceNbrElementTransformation using a user allocated
@@ -637,11 +664,11 @@ public:
    /** @brief Returns the number of local faces according to the requested type,
        does not count master non-conforming faces.
 
-       If type==Boundary returns only the number of true boundary faces
-       contrary to GetNBE() that returns all "boundary" elements which may
-       include actual interior faces.
-       Similarly, if type==Interior, only the true interior faces (including
-       shared faces) are counted excluding all master non-conforming faces. */
+       If type==Boundary returns only the number of true boundary faces contrary
+       to GetNBE() that returns all "boundary" elements which may include actual
+       interior faces. Similarly, if type==Interior, only the true interior
+       faces (including shared faces) are counted excluding all master
+       non-conforming faces. */
    int GetNFbyType(FaceType type) const override;
 
    void GenerateBoundaryElements() override
@@ -657,9 +684,9 @@ public:
        sequence of elements. Works for nonconforming meshes only. */
    void Rebalance();
 
-   /** Load balance a nonconforming mesh using a user-defined partition.
-       Each local element 'i' is migrated to processor rank 'partition[i]',
-       for 0 <= i < GetNE(). */
+   /** Load balance a nonconforming mesh using a user-defined partition. Each
+       local element 'i' is migrated to processor rank 'partition[i]', for 0 <=
+       i < GetNE(). */
    void Rebalance(const Array<int> &partition);
 
    /** Save the mesh in a parallel mesh format. If @a comments is non-empty, it

mesh/pncmesh.hpp

@@ -63,6 +63,8 @@ class FiniteElementSpace;
  */
 class ParNCMesh : public NCMesh
 {
+protected:
+   ParNCMesh() = default;
 public:
    /// Construct by partitioning a serial NCMesh.
    /** SFC partitioning is used by default. A user-specified partition can be
@@ -252,6 +254,7 @@ public:
 protected: // interface for ParMesh
 
    friend class ParMesh;
+   friend class ParSubMesh;
 
    /** For compatibility with conforming code in ParMesh and ParFESpace.
        Initializes shared structures in ParMesh: gtopo, shared_*, group_s*,

mesh/submesh/ncsubmesh.cpp

@@ -0,0 +1,133 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "ncsubmesh.hpp"
+
+#include <unordered_map>
+#include "submesh_utils.hpp"
+#include "submesh.hpp"
+
+namespace mfem
+{
+
+using namespace SubMeshUtils;
+
+NCSubMesh::NCSubMesh(SubMesh& submesh, const NCMesh &parent, From from,
+                     const Array<int> &attributes)
+   : NCMesh(), parent_(&parent)
+{
+   Dim = submesh.Dimension();
+   spaceDim = submesh.SpaceDimension();
+   MyRank = 0;
+   Iso = true;
+   Legacy = false;
+
+   if (from == From::Domain)
+   {
+      SubMeshUtils::ConstructVolumeTree(*this, attributes);
+   }
+   else if (from == From::Boundary)
+   {
+      SubMeshUtils::ConstructFaceTree(*this, attributes);
+   }
+
+   // Loop over all nodes, and reparent based on the node relations of the
+   // parent
+   for (int i = 0; i < parent_node_ids_.Size(); i++)
+   {
+      const auto &parent_node = parent.nodes[parent_node_ids_[i]];
+      const int submesh_p1 = parent_to_submesh_node_ids_[parent_node.p1];
+      const int submesh_p2 = parent_to_submesh_node_ids_[parent_node.p2];
+      nodes.Reparent(i, submesh_p1, submesh_p2);
+   }
+
+   nodes.UpdateUnused();
+   for (int i = 0; i < elements.Size(); i++)
+   {
+      if (elements[i].IsLeaf())
+      {
+         // Register all faces
+         RegisterFaces(i);
+      }
+   }
+
+   InitRootElements();
+   InitRootState(root_state.Size());
+   InitGeomFlags();
+   Update(); // Fills in secondary information based off of elements, nodes and faces.
+
+   // If parent has coordinates defined, copy the relevant portion
+   if (parent.coordinates.Size() > 0)
+   {
+      coordinates.SetSize(3*parent_node_ids_.Size());
+      parent.tmp_vertex = new TmpVertex[parent.nodes.NumIds()];
+      for (int n = 0; n < parent_node_ids_.Size(); n++)
+      {
+         std::memcpy(&coordinates[3*n], parent.CalcVertexPos(parent_node_ids_[n]),
+                     3*sizeof(real_t));
+      }
+      delete [] parent.tmp_vertex;
+   }
+
+   // The element indexing was changed as part of generation of leaf elements.
+   // We need to update the map.
+   if (from == From::Domain)
+   {
+      // The element indexing was changed as part of generation of leaf
+      // elements. We need to update the map.
+      submesh.parent_to_submesh_element_ids_ = -1;
+      for (int i = 0; i < submesh.parent_element_ids_.Size(); i++)
+      {
+         submesh.parent_element_ids_[i] =
+            parent.elements[parent_element_ids_[leaf_elements[i]]].index;
+         submesh.parent_to_submesh_element_ids_[submesh.parent_element_ids_[i]] = i;
+      }
+   }
+   else
+   {
+      submesh.parent_to_submesh_element_ids_ = -1;
+      // parent elements are BOUNDARY elements, need to map face index to be.
+      const auto &parent_face_to_be = submesh.GetParent()->GetFaceToBdrElMap();
+      MFEM_ASSERT(NElements == submesh.GetNE(), "!");
+      auto new_parent_to_submesh_element_ids = submesh.parent_to_submesh_element_ids_;
+      Array<int> new_parent_element_ids;
+      new_parent_element_ids.Reserve(submesh.parent_element_ids_.Size());
+      for (int i = 0; i < submesh.parent_element_ids_.Size(); i++)
+      {
+         new_parent_element_ids.Append(
+            parent_face_to_be[parent.faces[parent_element_ids_[leaf_elements[i]]].index]);
+         new_parent_to_submesh_element_ids[new_parent_element_ids[i]] = i;
+      }
+
+      MFEM_ASSERT(new_parent_element_ids.Size() == submesh.parent_element_ids_.Size(),
+                  "!");
+#ifdef MFEM_DEBUG
+      for (auto x : new_parent_element_ids)
+      {
+         MFEM_ASSERT(std::find(submesh.parent_element_ids_.begin(),
+                               submesh.parent_element_ids_.end(), x)
+                     != submesh.parent_element_ids_.end(),
+                     x << " not found in submesh.parent_element_ids_");
+      }
+      for (auto x : submesh.parent_element_ids_)
+      {
+         MFEM_ASSERT(std::find(new_parent_element_ids.begin(),
+                               new_parent_element_ids.end(), x)
+                     != new_parent_element_ids.end(), x << " not found in new_parent_element_ids_");
+      }
+#endif
+      submesh.parent_element_ids_ = std::move(new_parent_element_ids);
+      submesh.parent_to_submesh_element_ids_ =
+         std::move(new_parent_to_submesh_element_ids);
+   }
+}
+
+} // namespace mfem

mesh/submesh/ncsubmesh.hpp

@@ -0,0 +1,97 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_NCSUBMESH
+#define MFEM_NCSUBMESH
+
+#include "../ncmesh.hpp"
+#include "submesh.hpp"
+#include "submesh_utils.hpp"
+#include <unordered_map>
+
+namespace mfem
+{
+
+/**
+ * @brief Class representing a Nonconformal SubMesh. This is only used by
+ * SubMesh.
+ */
+class NCSubMesh : public NCMesh
+{
+   friend class SubMesh; ///< Only SubMesh can use methods in this class
+public:
+   using From = SubMesh::From; ///< Convenience type alias
+   /// Get the parent NCMesh object
+   const NCMesh* GetParent() const
+   {
+      return parent_;
+   }
+
+   /**
+   * @brief Check if NCMesh @a m is a NCSubMesh.
+   *
+   * @param m The input NCMesh
+   */
+   static bool IsNCSubMesh(const NCMesh *m)
+   {
+      return dynamic_cast<const NCSubMesh *>(m) != nullptr;
+   }
+private:
+
+   /// Private constructor
+   NCSubMesh(SubMesh& submesh, const NCMesh &parent, From from,
+             const Array<int> &attributes);
+
+   /// The parent NCMesh. Not owned.
+   const NCMesh *parent_;
+
+   /// Mapping from submesh element nc ids (index of the array), to the parent
+   /// element ids. If from a boundary, these map to faces in the parent.
+   Array<int> parent_element_ids_;
+
+   /// Mapping from NCSubMesh node ids (index of the array), to the parent
+   /// NCMesh node ids.
+   Array<int> parent_node_ids_;
+
+   /// Mapping from parent NCMesh node ids to submesh NCMesh node ids.
+   // Inverse map of parent_node_ids_.
+   std::unordered_map<int, int> parent_to_submesh_node_ids_;
+
+   /// Mapping from parent NCMesh element ids to submesh NCMesh element ids.
+   // Inverse map of parent_element_ids_.
+   std::unordered_map<int, int> parent_to_submesh_element_ids_;
+
+   // Helper friend methods for construction.
+   friend void SubMeshUtils::ConstructFaceTree<NCSubMesh>(NCSubMesh &submesh,
+                                                          const Array<int> &attributes);
+   friend void SubMeshUtils::ConstructVolumeTree<NCSubMesh>(NCSubMesh &submesh,
+                                                            const Array<int> &attributes);
+
+   /**
+    * @brief Accessor for parent nodes
+    * @details Required to bypass access protection in parent class.
+    *
+    * @return const HashTable<Node>&
+    */
+   const HashTable<Node> &ParentNodes() const { return parent_->nodes; }
+
+   /**
+    * @brief Accessor for parent faces
+    * @details Required to bypass access protection in parent class.
+    *
+    * @return const HashTable<Face>&
+    */
+   const HashTable<Face> &ParentFaces() const { return parent_->faces; }
+};
+
+} // namespace mfem
+
+#endif // MFEM_NCSUBMESH

mesh/submesh/pncsubmesh.cpp

@@ -0,0 +1,157 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "pncsubmesh.hpp"
+
+#include <numeric>
+#include <unordered_map>
+#include "submesh_utils.hpp"
+#include "psubmesh.hpp"
+namespace mfem
+{
+
+using namespace SubMeshUtils;
+
+
+ParNCSubMesh::ParNCSubMesh(ParSubMesh& submesh, const ParNCMesh &parent,
+                           From from, const Array<int> &attributes)
+   : ParNCMesh(), parent_(&parent)
+{
+   MyComm = submesh.GetComm();
+   NRanks = submesh.GetNRanks();
+   MyRank = submesh.GetMyRank();
+
+   Dim = submesh.Dimension();
+   spaceDim = submesh.SpaceDimension();
+   Iso = true;
+   Legacy = false;
+
+   // Loop over parent leaf elements and add nodes for all vertices. Register as
+   // top level nodes, will reparent when looping over edges. Cannot add edge
+   // nodes at same time because top level vertex nodes must be contiguous and
+   // first in node list (see coordinates).
+   if (from == From::Domain)
+   {
+      SubMeshUtils::ConstructVolumeTree(*this, attributes);
+   }
+   else if (from == From::Boundary)
+   {
+      SubMeshUtils::ConstructFaceTree(*this, attributes);
+   }
+
+   // Loop over all nodes, and reparent based on the node relations of the
+   // parent
+   for (int i = 0; i < parent_node_ids_.Size(); i++)
+   {
+      const auto &parent_node = parent.nodes[parent_node_ids_[i]];
+      const int submesh_p1 = parent_to_submesh_node_ids_[parent_node.p1];
+      const int submesh_p2 = parent_to_submesh_node_ids_[parent_node.p2];
+      nodes.Reparent(i, submesh_p1, submesh_p2);
+   }
+
+   nodes.UpdateUnused();
+   for (int i = 0; i < elements.Size(); i++)
+   {
+      if (elements[i].IsLeaf())
+      {
+         // Register all faces
+         RegisterFaces(i);
+      }
+   }
+
+   InitRootElements();
+   InitRootState(root_state.Size());
+   InitGeomFlags();
+   Update(); // Fills in secondary information based off of elements, nodes and faces.
+#ifdef MFEM_DEBUG
+   // Check all processors have the same number of roots
+   {
+      int p[2] = {root_state.Size(), -root_state.Size()};
+      MPI_Allreduce(MPI_IN_PLACE, p, 2, MPI_INT, MPI_MIN, submesh.GetComm());
+      MFEM_ASSERT(p[0] == -p[1], "Ranks must agree on number of root elements: min "
+                  << p[0] << " max " << -p[1] << " local " << root_state.Size() << " MyRank " <<
+                  submesh.GetMyRank());
+   }
+#endif
+
+   // If parent has coordinates defined, copy the relevant portion
+   if (parent.coordinates.Size() > 0)
+   {
+      // Loop over new_nodes -> coordinates is indexed by node.
+      coordinates.SetSize(3*parent_node_ids_.Size());
+      parent.tmp_vertex = new TmpVertex[parent.nodes.NumIds()];
+      for (int n = 0; n < parent_node_ids_.Size(); n++)
+      {
+         std::memcpy(&coordinates[3*n], parent.CalcVertexPos(parent_node_ids_[n]),
+                     3*sizeof(real_t));
+      }
+      delete [] parent.tmp_vertex;
+   }
+
+   // The element indexing was changed as part of generation of leaf elements.
+   // We need to update the map.
+   if (from == From::Domain)
+   {
+      // The element indexing was changed as part of generation of leaf
+      // elements. We need to update the map.
+      submesh.parent_to_submesh_element_ids_ = -1;
+      for (int i = 0; i < submesh.parent_element_ids_.Size(); i++)
+      {
+         submesh.parent_element_ids_[i] =
+            parent.elements[parent_element_ids_[leaf_elements[i]]].index;
+         submesh.parent_to_submesh_element_ids_[submesh.parent_element_ids_[i]] = i;
+      }
+   }
+   else
+   {
+      submesh.parent_to_submesh_element_ids_ = -1;
+      // parent elements are BOUNDARY elements, need to map face index to be.
+      const auto &parent_face_to_be = submesh.GetParent()->GetFaceToBdrElMap();
+      MFEM_ASSERT(NElements == submesh.GetNE(), NElements << ' ' << submesh.GetNE());
+      auto new_parent_to_submesh_element_ids = submesh.parent_to_submesh_element_ids_;
+      Array<int> new_parent_element_ids;
+      new_parent_element_ids.Reserve(submesh.parent_element_ids_.Size());
+      for (int i = 0; i < submesh.parent_element_ids_.Size(); i++)
+      {
+         new_parent_element_ids.Append(
+            parent_face_to_be[parent.faces[parent_element_ids_[leaf_elements[i]]].index]);
+         new_parent_to_submesh_element_ids[new_parent_element_ids[i]] = i;
+      }
+
+      MFEM_ASSERT(new_parent_element_ids.Size() == submesh.parent_element_ids_.Size(),
+                  new_parent_element_ids.Size() << ' ' << submesh.parent_element_ids_.Size());
+#ifdef MFEM_DEBUG
+      for (auto x : new_parent_element_ids)
+      {
+         MFEM_ASSERT(std::find(submesh.parent_element_ids_.begin(),
+                               submesh.parent_element_ids_.end(), x)
+                     != submesh.parent_element_ids_.end(),
+                     x << " not found in submesh.parent_element_ids_");
+      }
+      for (auto x : submesh.parent_element_ids_)
+      {
+         MFEM_ASSERT(std::find(new_parent_element_ids.begin(),
+                               new_parent_element_ids.end(), x)
+                     != new_parent_element_ids.end(), x << " not found in new_parent_element_ids_");
+      }
+#endif
+      submesh.parent_element_ids_ = new_parent_element_ids;
+      submesh.parent_to_submesh_element_ids_ = new_parent_to_submesh_element_ids;
+   }
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_MPI

mesh/submesh/pncsubmesh.hpp

@@ -0,0 +1,102 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PNCSUBMESH
+#define MFEM_PNCSUBMESH
+
+#include "../../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "../pncmesh.hpp"
+#include "psubmesh.hpp"
+#include "submesh_utils.hpp"
+#include <unordered_map>
+
+namespace mfem
+{
+
+/**
+ * @brief Class representing a Parallel Nonconformal SubMesh. This is only used
+ * by ParSubMesh.
+ */
+class ParNCSubMesh : public ParNCMesh
+{
+   friend class ParSubMesh; ///< Only ParSubMesh can use methods in this class
+public:
+   using From = SubMesh::From; ///< Convenience type alias
+   /**
+   * @brief Check if NCMesh @a m is a ParNCSubMesh.
+   *
+   * @param m The input Mesh
+   */
+   static bool IsParNCSubMesh(const NCMesh *m)
+   {
+      return dynamic_cast<const ParNCSubMesh *>(m) != nullptr;
+   }
+   /// Get the parent ParNCMesh object
+   const ParNCMesh* GetParent() const
+   {
+      return parent_;
+   }
+
+protected:
+   /// protected constructor
+   ParNCSubMesh(ParSubMesh& submesh, const ParNCMesh &parent, From from,
+                const Array<int> &attributes);
+
+   /// The parent ParNCMesh. Not owned.
+   const ParNCMesh *parent_;
+
+   /// Mapping from submesh element nc ids (index of the array), to the parent
+   /// element ids. If from a boundary, these map to faces in the parent.
+   Array<int> parent_element_ids_;
+
+   /// Mapping from ParNCSubMesh node ids (index of the array), to the parent
+   /// NCMesh node ids.
+   Array<int> parent_node_ids_;
+
+   /// Mapping from parent NCMesh node ids to submesh NCMesh node ids.
+   // Inverse map of parent_node_ids_.
+   std::unordered_map<int, int> parent_to_submesh_node_ids_;
+
+   /// Mapping from parent NCMesh element ids to submesh NCMesh element ids.
+   // Inverse map of parent_element_ids_.
+   std::unordered_map<int, int> parent_to_submesh_element_ids_;
+
+   // Helper friend methods for construction.
+   friend void SubMeshUtils::ConstructFaceTree<ParNCSubMesh>
+   (ParNCSubMesh &submesh, const Array<int> &attributes);
+   friend void SubMeshUtils::ConstructVolumeTree<ParNCSubMesh>
+   (ParNCSubMesh &submesh, const Array<int> &attributes);
+
+   /**
+    * @brief Accessor for parent nodes
+    * @details Required to bypass access protection in parent class.
+    *
+    * @return const HashTable<Node>&
+    */
+   const HashTable<Node> &ParentNodes() const { return parent_->nodes; }
+
+   /**
+    * @brief Accessor for parent faces
+    * @details Required to bypass access protection in parent class.
+    *
+    * @return const HashTable<Face>&
+    */
+   const HashTable<Face> &ParentFaces() const { return parent_->faces; }
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_MPI
+
+#endif // MFEM_PNCSUBMESH

mesh/submesh/psubmesh.cpp

@@ -17,6 +17,7 @@
 #include <unordered_set>
 #include <algorithm>
 #include "psubmesh.hpp"
+#include "pncsubmesh.hpp"
 #include "submesh_utils.hpp"
 #include "../segment.hpp"
 
@@ -24,33 +25,29 @@ namespace mfem
 {
 
 ParSubMesh ParSubMesh::CreateFromDomain(const ParMesh &parent,
-                                        Array<int> &domain_attributes)
+                                        const Array<int> &domain_attributes)
 {
    return ParSubMesh(parent, SubMesh::From::Domain, domain_attributes);
 }
 
 ParSubMesh ParSubMesh::CreateFromBoundary(const ParMesh &parent,
-                                          Array<int> &boundary_attributes)
+                                          const Array<int> &boundary_attributes)
 {
    return ParSubMesh(parent, SubMesh::From::Boundary, boundary_attributes);
 }
 
 ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
-                       Array<int> &attributes) : parent_(parent), from_(from), attributes_(attributes)
+                       const Array<int> &attributes) : parent_(parent), from_(from),
+   attributes_(attributes)
 {
-   if (Nonconforming())
-   {
-      MFEM_ABORT("SubMesh does not support non-conforming meshes");
-   }
-
    MyComm = parent.GetComm();
    NRanks = parent.GetNRanks();
    MyRank = parent.GetMyRank();
 
-   // This violation of const-ness may be justified in this instance because
-   // the exchange of face neighbor information only establishes or updates
-   // derived information without altering the primary mesh information,
-   // i.e., the topology, geometry, or region attributes.
+   // This violation of const-ness may be justified in this instance because the
+   // exchange of face neighbor information only establishes or updates derived
+   // information without altering the primary mesh information, i.e., the
+   // topology, geometry, or region attributes.
    const_cast<ParMesh&>(parent).ExchangeFaceNbrData();
 
    if (from == SubMesh::From::Domain)
@@ -70,11 +67,6 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
                                                                       attributes_, true);
    }
 
-   // Don't let boundary elements get generated automatically. This would
-   // generate boundary elements on each rank locally, which is topologically
-   // wrong for the distributed SubMesh.
-   FinalizeTopology(false);
-
    parent_to_submesh_vertex_ids_.SetSize(parent_.GetNV());
    parent_to_submesh_vertex_ids_ = -1;
    for (int i = 0; i < parent_vertex_ids_.Size(); i++)
@@ -82,6 +74,43 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       parent_to_submesh_vertex_ids_[parent_vertex_ids_[i]] = i;
    }
 
+   parent_to_submesh_element_ids_.SetSize(from == From::Boundary ? parent.GetNBE()
+                                          : parent.GetNE());
+   parent_to_submesh_element_ids_ = -1;
+   for (int i = 0; i < parent_element_ids_.Size(); i++)
+   {
+      parent_to_submesh_element_ids_[parent_element_ids_[i]] = i;
+   }
+
+   // Don't let boundary elements get generated automatically. This would
+   // generate boundary elements on each rank locally, which is topologically
+   // wrong for the distributed SubMesh.
+   FinalizeTopology(false);
+
+   if (parent.Nonconforming())
+   {
+      pncmesh = new ParNCSubMesh(*this, *parent.pncmesh, from, attributes);
+      pncsubmesh_ = dynamic_cast<ParNCSubMesh*>(pncmesh);
+      ncmesh = pncmesh;
+      InitFromNCMesh(*pncmesh);
+      pncmesh->OnMeshUpdated(this);
+
+      // Update the submesh to parent vertex mapping, NCSubMesh reordered the
+      // vertices so the map to parent is no longer valid.
+      parent_to_submesh_vertex_ids_ = -1;
+      for (int i = 0; i < parent_vertex_ids_.Size(); i++)
+      {
+         // vertex -> node -> parent node -> parent vertex
+         auto node = pncsubmesh_->vertex_nodeId[i];
+         auto parent_node = pncsubmesh_->parent_node_ids_[node];
+         auto parent_vertex = parent.pncmesh->GetNodeVertex(parent_node);
+         parent_vertex_ids_[i] = parent_vertex;
+         parent_to_submesh_vertex_ids_[parent_vertex] = i;
+      }
+      GenerateNCFaceInfo();
+      SetAttributes();
+   }
+
    DSTable v2v(parent_.GetNV());
    parent_.GetVertexToVertexTable(v2v);
    for (int i = 0; i < NumOfEdges; i++)
@@ -115,7 +144,6 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       }
 
       parent_face_ori_.SetSize(NumOfFaces);
-
       for (int i = 0; i < NumOfFaces; i++)
       {
          Array<int> sub_vert;
@@ -191,7 +219,6 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
    // Every rank containing elements of the ParSubMesh attributes now has a
    // local ParSubMesh. We have to connect the local meshes and assign global
    // boundaries correctly.
-
    Array<int> rhvtx;
    FindSharedVerticesRanks(rhvtx);
    AppendSharedVerticesGroups(groups, rhvtx);
@@ -207,6 +234,7 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
       AppendSharedFacesGroups(groups, rht, rhq);
    }
 
+
    // Build the group communication topology
    gtopo.SetComm(MyComm);
    gtopo.Create(groups, 822);
@@ -239,113 +267,17 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
 
    ExchangeFaceNbrData();
 
-   // Add boundaries
+   SubMeshUtils::AddBoundaryElements(*this,
+                                     (from == SubMesh::From::Domain)
+                                     ? FindGhostBoundaryElementAttributes()
+                                     : std::unordered_map<int,int> {});
+
+   if (Dim > 1)
    {
-      const int num_codim_1 = [this]()
-      {
-         if (Dim == 1) { return NumOfVertices; }
-         else if (Dim == 2) { return NumOfEdges; }
-         else if (Dim == 3) { return NumOfFaces; }
-         else { MFEM_ABORT("Invalid dimension."); return -1; }
-      }();
-
-      if (Dim == 3)
-      {
-         // In 3D we check for `bel_to_edge`. It shouldn't have been set
-         // previously.
-         delete bel_to_edge;
-         bel_to_edge = nullptr;
-      }
-
-      NumOfBdrElements = 0;
-      for (int i = 0; i < num_codim_1; i++)
-      {
-         if (GetFaceInformation(i).IsBoundary())
-         {
-            NumOfBdrElements++;
-         }
-      }
-
-      boundary.SetSize(NumOfBdrElements);
-      be_to_face.SetSize(NumOfBdrElements);
-      Array<int> parent_face_to_be = parent.GetFaceToBdrElMap();
-      int max_bdr_attr = parent.bdr_attributes.Max();
-
-      for (int i = 0, j = 0; i < num_codim_1; i++)
-      {
-         if (GetFaceInformation(i).IsBoundary())
-         {
-            boundary[j] = faces[i]->Duplicate(this);
-            be_to_face[j] = i;
-
-            if (from == SubMesh::From::Domain && Dim >= 2)
-            {
-               int pbeid = Dim == 3 ? parent_face_to_be[parent_face_ids_[i]] :
-                           parent_face_to_be[parent_edge_ids_[i]];
-               if (pbeid != -1)
-               {
-                  boundary[j]->SetAttribute(parent.GetBdrAttribute(pbeid));
-               }
-               else
-               {
-                  boundary[j]->SetAttribute(max_bdr_attr + 1);
-               }
-            }
-            else
-            {
-               boundary[j]->SetAttribute(SubMesh::GENERATED_ATTRIBUTE);
-            }
-            ++j;
-         }
-      }
-
-      if (from == SubMesh::From::Domain && Dim >= 2)
-      {
-         // Search for and count interior boundary elements
-         int InteriorBdrElems = 0;
-         for (int i=0; i<parent.GetNBE(); i++)
-         {
-            const int parentFaceIdx = parent.GetBdrElementFaceIndex(i);
-            const int submeshFaceIdx =
-               Dim == 3 ?
-               parent_to_submesh_face_ids_[parentFaceIdx] :
-               parent_to_submesh_edge_ids_[parentFaceIdx];
-
-            if (submeshFaceIdx == -1) { continue; }
-            if (GetFaceInformation(submeshFaceIdx).IsBoundary()) { continue; }
-
-            InteriorBdrElems++;
-         }
-
-         if (InteriorBdrElems > 0)
-         {
-            const int OldNumOfBdrElements = NumOfBdrElements;
-            NumOfBdrElements += InteriorBdrElems;
-            boundary.SetSize(NumOfBdrElements);
-            be_to_face.SetSize(NumOfBdrElements);
-
-            // Search for and transfer interior boundary elements
-            for (int i=0, j = OldNumOfBdrElements; i<parent.GetNBE(); i++)
-            {
-               const int parentFaceIdx = parent.GetBdrElementFaceIndex(i);
-               const int submeshFaceIdx =
-                  parent_to_submesh_face_ids_[parentFaceIdx];
-
-               if (submeshFaceIdx == -1) { continue; }
-               if (GetFaceInformation(submeshFaceIdx).IsBoundary())
-               { continue; }
-
-               boundary[j] = faces[submeshFaceIdx]->Duplicate(this);
-               be_to_face[j] = submeshFaceIdx;
-               boundary[j]->SetAttribute(parent.GetBdrAttribute(i));
-
-               ++j;
-            }
-         }
-      }
+      if (!el_to_edge) { el_to_edge = new Table; }
+      NumOfEdges = GetElementToEdgeTable(*el_to_edge);
    }
-
-   if (Dim == 3)
+   if (Dim > 2)
    {
       GetElementToFaceTable();
    }
@@ -376,84 +308,6 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,
 
       Transfer(*pn, *n);
    }
-
-   if (Dim > 1)
-   {
-      if (!el_to_edge) { el_to_edge = new Table; }
-      NumOfEdges = GetElementToEdgeTable(*el_to_edge);
-   }
-
-   if (Dim > 1 && from == SubMesh::From::Domain)
-   {
-      // Order 0 Raviart-Thomas space will have precisely 1 DoF per face.
-      // We can use this DoF to communicate boundary attribute numbers.
-      RT_FECollection fec_rt(0, Dim);
-      ParFiniteElementSpace parent_fes_rt(const_cast<ParMesh*>(&parent),
-                                          &fec_rt);
-
-      ParGridFunction parent_bdr_attr_gf(&parent_fes_rt);
-      parent_bdr_attr_gf = 0.0;
-
-      Array<int> vdofs;
-      DofTransformation doftrans;
-      int dof, faceIdx;
-      real_t sign, w;
-
-      // Copy boundary attribute numbers into local portion of a parallel
-      // grid function
-      parent_bdr_attr_gf.HostReadWrite(); // not modifying all entries
-      for (int i=0; i<parent.GetNBE(); i++)
-      {
-         faceIdx = parent.GetBdrElementFaceIndex(i);
-         const FaceInformation &faceInfo = parent.GetFaceInformation(faceIdx);
-         parent_fes_rt.GetBdrElementDofs(i, vdofs, doftrans);
-         dof = ParFiniteElementSpace::DecodeDof(vdofs[0], sign);
-
-         // Shared interior boundary elements are not duplicated across
-         // processor boundaries but ParGridFunction::ParallelAverage will
-         // assume both processors contribute to the averaged DoF value. So,
-         // we multiply shared boundary values by 2 so that the average
-         // produces the desired value.
-         w = faceInfo.IsShared() ? 2.0 : 1.0;
-
-         // The DoF sign is needed to ensure that non-shared interior
-         // boundary values sum properly rather than canceling.
-         parent_bdr_attr_gf[dof] = sign * w * parent.GetBdrAttribute(i);
-      }
-
-      Vector parent_bdr_attr(parent_fes_rt.GetTrueVSize());
-
-      // Compute the average of the attribute numbers
-      parent_bdr_attr_gf.ParallelAverage(parent_bdr_attr);
-      // Distribute boundary attributes to neighboring processors
-      parent_bdr_attr_gf.Distribute(parent_bdr_attr);
-
-      ParFiniteElementSpace submesh_fes_rt(this,
-                                           &fec_rt);
-
-      ParGridFunction submesh_bdr_attr_gf(&submesh_fes_rt);
-
-      // Transfer the averaged boundary attribute values to the submesh
-      auto transfer_map = ParSubMesh::CreateTransferMap(parent_bdr_attr_gf,
-                                                        submesh_bdr_attr_gf);
-      transfer_map.Transfer(parent_bdr_attr_gf, submesh_bdr_attr_gf);
-
-      // Extract the boundary attribute numbers from the local portion
-      // of the ParGridFunction and set the corresponding boundary element
-      // attributes.
-      int attr;
-      for (int i=0; i<NumOfBdrElements; i++)
-      {
-         submesh_fes_rt.GetBdrElementDofs(i, vdofs, doftrans);
-         dof = ParFiniteElementSpace::DecodeDof(vdofs[0], sign);
-         attr = (int)std::round(std::abs(submesh_bdr_attr_gf[dof]));
-         if (attr != 0)
-         {
-            SetBdrAttribute(i, attr);
-         }
-      }
-   }
-
    SetAttributes();
    Finalize();
 }
@@ -494,6 +348,7 @@ void ParSubMesh::FindSharedVerticesRanks(Array<int> &rhvtx)
       }
    }
 
+
    // Compute the sum on the root rank and broadcast the result to all ranks.
    svert_comm.Reduce(rhvtx, GroupCommunicator::Sum);
    svert_comm.Bcast<int>(rhvtx, 0);
@@ -511,8 +366,8 @@ void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
    rhe.SetSize(nsedges);
    rhe = 0;
 
-   // On each rank of the group, locally determine if the shared edge is in
-   // the SubMesh.
+   // On each rank of the group, locally determine if the shared edge is in the
+   // SubMesh.
    for (int g = 1, se = 0; g < parent_.GetNGroups(); g++)
    {
       const int group_sz = parent_.gtopo.GetGroupSize(g);
@@ -528,8 +383,7 @@ void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
 
       for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
       {
-         int ple, o;
-         parent_.GroupEdge(g, ge, ple, o);
+         int ple = parent_.GroupEdge(g, ge);
          int submesh_edge_id = parent_to_submesh_edge_ids_[ple];
          if (submesh_edge_id != -1)
          {
@@ -538,6 +392,7 @@ void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
       }
    }
 
+
    // Compute the sum on the root rank and broadcast the result to all ranks.
    sedge_comm.Reduce(rhe, GroupCommunicator::Sum);
    sedge_comm.Bcast<int>(rhe, 0);
@@ -545,50 +400,21 @@ void ParSubMesh::FindSharedEdgesRanks(Array<int> &rhe)
 
 void ParSubMesh::FindSharedFacesRanks(Array<int>& rht, Array<int> &rhq)
 {
-   GroupCommunicator squad_comm(parent_.gtopo);
-   parent_.GetSharedQuadCommunicator(squad_comm);
-
-   int nsquad = squad_comm.GroupLDofTable().Size_of_connections();
-
-   rhq.SetSize(nsquad);
-   rhq = 0;
-
-   for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
-   {
-      for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
-      {
-         // Group size of a shared face is always 2
-
-         int plq, o;
-         parent_.GroupQuadrilateral(g, gq, plq, o);
-         int submesh_face_id = parent_to_submesh_face_ids_[plq];
-         if (submesh_face_id != -1)
-         {
-            rhq[sq] = 1;
-         }
-      }
-   }
-
-   // Compute the sum on the root rank and broadcast the result to all ranks.
-   squad_comm.Reduce(rhq, GroupCommunicator::Sum);
-   squad_comm.Bcast<int>(rhq, 0);
-
    GroupCommunicator stria_comm(parent_.gtopo);
    parent_.GetSharedTriCommunicator(stria_comm);
-
    int nstria = stria_comm.GroupLDofTable().Size_of_connections();
-
    rht.SetSize(nstria);
    rht = 0;
 
    for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
    {
+      MFEM_ASSERT(parent_.gtopo.GetGroupSize(g) == 2
+                  || parent_.GroupNTriangles(g) == 0,
+                  parent_.gtopo.GetGroupSize(g) << ' ' << parent_.GroupNTriangles(g));
       for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
       {
          // Group size of a shared face is always 2
-
-         int plt, o;
-         parent_.GroupTriangle(g, gt, plt, o);
+         int plt = parent_.GroupTriangle(g, gt);
          int submesh_face_id = parent_to_submesh_face_ids_[plt];
          if (submesh_face_id != -1)
          {
@@ -600,6 +426,33 @@ void ParSubMesh::FindSharedFacesRanks(Array<int>& rht, Array<int> &rhq)
    // Compute the sum on the root rank and broadcast the result to all ranks.
    stria_comm.Reduce(rht, GroupCommunicator::Sum);
    stria_comm.Bcast<int>(rht, 0);
+
+   GroupCommunicator squad_comm(parent_.gtopo);
+   parent_.GetSharedQuadCommunicator(squad_comm);
+   int nsquad = squad_comm.GroupLDofTable().Size_of_connections();
+   rhq.SetSize(nsquad);
+   rhq = 0;
+
+   for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
+   {
+      MFEM_ASSERT(parent_.gtopo.GetGroupSize(g) == 2
+                  || parent_.GroupNQuadrilaterals(g) == 0,
+                  parent_.gtopo.GetGroupSize(g) << ' ' << parent_.GroupNQuadrilaterals(g));
+      for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
+      {
+         // Group size of a shared face is always 2
+         int plq = parent_.GroupQuadrilateral(g, gq);
+         int submesh_face_id = parent_to_submesh_face_ids_[plq];
+         if (submesh_face_id != -1)
+         {
+            rhq[sq] = 1;
+         }
+      }
+   }
+
+   // Compute the sum on the root rank and broadcast the result to all ranks.
+   squad_comm.Reduce(rhq, GroupCommunicator::Sum);
+   squad_comm.Bcast<int>(rhq, 0);
 }
 
 
@@ -608,6 +461,7 @@ void ParSubMesh::AppendSharedVerticesGroups(ListOfIntegerSets &groups,
 {
    IntegerSet group;
 
+   // g = 0 corresponds to the singleton group of each rank alone.
    for (int g = 1, sv = 0; g < parent_.GetNGroups(); g++)
    {
       const int group_sz = parent_.gtopo.GetGroupSize(g);
@@ -679,8 +533,7 @@ void ParSubMesh::AppendSharedEdgesGroups(ListOfIntegerSets &groups,
 
       for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
       {
-         int ple, o;
-         parent_.GroupEdge(g, ge, ple, o);
+         int ple = parent_.GroupEdge(g, ge);
          int submesh_edge = parent_to_submesh_edge_ids_[ple];
 
          // Reusing the `rhe` array as shared edge to group array.
@@ -729,8 +582,7 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
          const int group_sz = parent_.gtopo.GetGroupSize(g);
          MFEM_ASSERT(group_sz == 2, "internal error");
 
-         int plq, o;
-         parent_.GroupQuadrilateral(g, gq, plq, o);
+         int plq = parent_.GroupQuadrilateral(g, gq);
          int submesh_face_id = parent_to_submesh_face_ids_[plq];
 
          // Reusing the `rhq` array as shared face to group array.
@@ -743,8 +595,8 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
          {
             // shared face is present on this rank and others
 
-            // There can only be two ranks in this group sharing faces. Add
-            // all ranks to a new communication group.
+            // There can only be two ranks in this group sharing faces. Add all
+            // ranks to a new communication group.
             Array<int> &ranks = quad_group;
             ranks.SetSize(0);
             ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[0]));
@@ -770,8 +622,7 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
          const int group_sz = parent_.gtopo.GetGroupSize(g);
          MFEM_ASSERT(group_sz == 2, "internal error");
 
-         int plt, o;
-         parent_.GroupTriangle(g, gt, plt, o);
+         int plt = parent_.GroupTriangle(g, gt);
          int submesh_face_id = parent_to_submesh_face_ids_[plt];
 
          // Reusing the `rht` array as shared face to group array.
@@ -784,8 +635,8 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
          {
             // shared face is present on this rank and others
 
-            // There can only be two ranks in this group sharing faces. Add
-            // all ranks to a new communication group.
+            // There can only be two ranks in this group sharing faces. Add all
+            // ranks to a new communication group.
             Array<int> &ranks = tria_group;
             ranks.SetSize(0);
             ranks.Append(parent_.gtopo.GetNeighborRank(group_lproc[0]));
@@ -802,96 +653,46 @@ void ParSubMesh::AppendSharedFacesGroups(ListOfIntegerSets &groups,
    }
 }
 
-void ParSubMesh::BuildVertexGroup(int ngroups, const Array<int>& rhvtx,
-                                  int& nsverts)
+void BuildGroup(Table &group, int ngroups, const Array<int>& rh, int &ns)
 {
-   group_svert.MakeI(ngroups);
-   for (int i = 0; i < rhvtx.Size(); i++)
+   group.MakeI(ngroups);
+   for (int i = 0; i < rh.Size(); i++)
    {
-      if (rhvtx[i] >= 0)
+      if (rh[i] >= 0)
       {
-         group_svert.AddAColumnInRow(rhvtx[i]);
+         group.AddAColumnInRow(rh[i]);
       }
    }
 
-   group_svert.MakeJ();
-   nsverts = 0;
-   for (int i = 0; i < rhvtx.Size(); i++)
+   group.MakeJ();
+   ns = 0;
+   for (int i = 0; i < rh.Size(); i++)
    {
-      if (rhvtx[i] >= 0)
+      if (rh[i] >= 0)
       {
-         group_svert.AddConnection(rhvtx[i], nsverts++);
+         group.AddConnection(rh[i], ns++);
       }
    }
-   group_svert.ShiftUpI();
+   group.ShiftUpI();
+}
+
+void ParSubMesh::BuildVertexGroup(int ngroups, const Array<int>& rhvtx,
+                                  int& nsverts)
+{
+   BuildGroup(group_svert, ngroups, rhvtx, nsverts);
 }
 
 void ParSubMesh::BuildEdgeGroup(int ngroups, const Array<int>& rhe,
                                 int& nsedges)
 {
-   group_sedge.MakeI(ngroups);
-   for (int i = 0; i < rhe.Size(); i++)
-   {
-      if (rhe[i] >= 0)
-      {
-         group_sedge.AddAColumnInRow(rhe[i]);
-      }
-   }
-
-   group_sedge.MakeJ();
-   nsedges = 0;
-   for (int i = 0; i < rhe.Size(); i++)
-   {
-      if (rhe[i] >= 0)
-      {
-         group_sedge.AddConnection(rhe[i], nsedges++);
-      }
-   }
-   group_sedge.ShiftUpI();
+   BuildGroup(group_sedge, ngroups, rhe, nsedges);
 }
 
 void ParSubMesh::BuildFaceGroup(int ngroups, const Array<int>& rht,
                                 int& nstrias, const Array<int>& rhq, int& nsquads)
 {
-   group_squad.MakeI(ngroups);
-   for (int i = 0; i < rhq.Size(); i++)
-   {
-      if (rhq[i] >= 0)
-      {
-         group_squad.AddAColumnInRow(rhq[i]);
-      }
-   }
-
-   group_squad.MakeJ();
-   nsquads = 0;
-   for (int i = 0; i < rhq.Size(); i++)
-   {
-      if (rhq[i] >= 0)
-      {
-         group_squad.AddConnection(rhq[i], nsquads++);
-      }
-   }
-   group_squad.ShiftUpI();
-
-   group_stria.MakeI(ngroups);
-   for (int i = 0; i < rht.Size(); i++)
-   {
-      if (rht[i] >= 0)
-      {
-         group_stria.AddAColumnInRow(rht[i]);
-      }
-   }
-
-   group_stria.MakeJ();
-   nstrias = 0;
-   for (int i = 0; i < rht.Size(); i++)
-   {
-      if (rht[i] >= 0)
-      {
-         group_stria.AddConnection(rht[i], nstrias++);
-      }
-   }
-   group_stria.ShiftUpI();
+   BuildGroup(group_squad, ngroups, rhq, nsquads);
+   BuildGroup(group_stria, ngroups, rht, nstrias);
 }
 
 void ParSubMesh::BuildSharedVerticesMapping(const int nsverts,
@@ -943,8 +744,8 @@ void ParSubMesh::BuildSharedEdgesMapping(const int sedges_ct,
             int v0 = parent_to_submesh_vertex_ids_[vert[(1-o)/2]];
             int v1 = parent_to_submesh_vertex_ids_[vert[(1+o)/2]];
 
-            // The orienation of the shared edge relative to the local edge
-            // will be determined by whether v0 < v1 or v1 < v0
+            // The orienation of the shared edge relative to the local edge will
+            // be determined by whether v0 < v1 or v1 < v0
             shared_edges.Append(new Segment(v0, v1, 1));
             sedge_ledge.Append(submesh_edge_id);
          }
@@ -960,9 +761,8 @@ void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
    shared_quads.Reserve(nsquads);
    sface_lface.Reserve(nstrias + nsquads);
 
-   // sface_lface should list the triangular shared faces first
-   // followed by the quadrilateral shared faces.
-
+   // sface_lface should list the triangular shared faces first followed by the
+   // quadrilateral shared faces.
    for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
    {
       for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
@@ -1028,7 +828,7 @@ void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
             int v2 = vert[2];
             int v3 = vert[3];
 
-            // See Mesh::GetQuadOrientation for info on interpretting "o"
+            // See Mesh::GetQuadOrientation for info on interpreting "o"
             switch (o)
             {
                case 1:
@@ -1057,10 +857,254 @@ void ParSubMesh::BuildSharedFacesMapping(const int nstrias,
    }
 }
 
+std::unordered_map<int, int>
+ParSubMesh::FindGhostBoundaryElementAttributes() const
+{
+   // Loop over shared faces in the parent mesh, find their attributes if they
+   // exist, and map to local faces in the submesh.
+   std::unordered_map<int,int> lface_boundary_attribute;
+   const auto &face_to_be = parent_.GetFaceToBdrElMap();
+   if (Dim == 3)
+   {
+      GroupCommunicator squad_comm(parent_.gtopo);
+      parent_.GetSharedQuadCommunicator(squad_comm);
+      int nsquad = squad_comm.GroupLDofTable().Size_of_connections();
+
+      GroupCommunicator stria_comm(parent_.gtopo);
+      parent_.GetSharedTriCommunicator(stria_comm);
+      int nstria = stria_comm.GroupLDofTable().Size_of_connections();
+
+      Array<int> stba(nstria), sqba(nsquad);
+      Array<int> parent_ltface(nstria), parent_lqface(nsquad);
+      stba = 0; sqba = 0;
+      parent_ltface = -1; parent_lqface = -1;
+      for (int g = 1, st = 0; g < parent_.GetNGroups(); g++)
+      {
+         for (int gt = 0; gt < parent_.GroupNTriangles(g); gt++, st++)
+         {
+            // Group size of a shared face is always 2
+            int plt = parent_.GroupTriangle(g, gt);
+            auto pbe = face_to_be[plt];
+            if (pbe >= 0)
+            {
+               stba[st] = parent_.GetBdrAttribute(pbe);
+            }
+            parent_ltface[st] = plt;
+         }
+      }
+      for (int g = 1, sq = 0; g < parent_.GetNGroups(); g++)
+      {
+         for (int gq = 0; gq < parent_.GroupNQuadrilaterals(g); gq++, sq++)
+         {
+            // Group size of a shared face is always 2
+            int plq = parent_.GroupQuadrilateral(g, gq);
+            auto pbe = face_to_be[plq];
+            if (pbe >= 0)
+            {
+               sqba[sq] = parent_.GetBdrAttribute(pbe);
+            }
+            parent_lqface[sq] = plq;
+         }
+      }
+#ifdef MFEM_DEBUG
+      auto pre_stba = stba;
+      auto pre_sqba = sqba;
+#endif
+      stria_comm.Reduce(stba, GroupCommunicator::Sum);
+      stria_comm.Bcast<int>(stba, 0);
+      squad_comm.Reduce(sqba, GroupCommunicator::Sum);
+      squad_comm.Bcast<int>(sqba, 0);
+#ifdef MFEM_DEBUG
+      {
+         Array<int> fail_indices;
+         fail_indices.Reserve(stba.Size());
+         for (int i = 0; i < stba.Size(); i++)
+            if (pre_stba[i] != 0 && pre_stba[i] != stba[i])
+            {
+               fail_indices.Append(i);
+            }
+         MFEM_ASSERT(fail_indices.Size() == 0, [&]()
+         {
+            std::stringstream msg;
+            msg << "More than one rank found attribute on shared tri face: ";
+            for (auto x : fail_indices)
+            {
+               msg << x << ' ';
+            }
+            return msg.str();
+         }());
+      }
+
+      {
+         Array<int> fail_indices;
+         fail_indices.Reserve(sqba.Size());
+         for (int i = 0; i < sqba.Size(); i++)
+            if (pre_sqba[i] != 0 && pre_sqba[i] != sqba[i])
+            {
+               fail_indices.Append(i);
+            }
+         MFEM_ASSERT(fail_indices.Size() == 0, [&]()
+         {
+            std::stringstream msg;
+            msg << "More than one rank found attribute on shared quad face: ";
+            for (auto x : fail_indices)
+            {
+               msg << x << ' ';
+            }
+            return msg.str();
+         }());
+      }
+#endif
+      int nghost = 0;
+      for (auto x : stba)
+         if (x > 0) { ++nghost; }
+
+      for (auto x : sqba)
+         if (x > 0) { ++nghost; }
+
+      lface_boundary_attribute.reserve(nghost);
+      for (int i = 0; i < stba.Size(); i++)
+         if (stba[i] > 0)
+         {
+            MFEM_ASSERT(parent_ltface[i] > -1, i);
+            lface_boundary_attribute[parent_ltface[i]] = stba[i];
+         }
+      for (int i = 0; i < sqba.Size(); i++)
+         if (sqba[i] > 0)
+         {
+            MFEM_ASSERT(parent_lqface[i] > -1, i);
+            lface_boundary_attribute[parent_lqface[i]] = sqba[i];
+         }
+   }
+   else if (Dim == 2)
+   {
+      GroupCommunicator sedge_comm(parent_.gtopo);
+      parent_.GetSharedEdgeCommunicator(sedge_comm);
+      int nsedge = sedge_comm.GroupLDofTable().Size_of_connections();
+
+      Array<int> seba(nsedge), parent_ledge(nsedge);
+      seba = 0; parent_ledge = -1;
+      for (int g = 1, se = 0; g < parent_.GetNGroups(); g++)
+      {
+         for (int ge = 0; ge < parent_.GroupNEdges(g); ge++, se++)
+         {
+            // Group size of a shared edge is always 2
+            int ple = parent_.GroupEdge(g, ge);
+            auto pbe = face_to_be[ple];
+            if (pbe >= 0)
+            {
+               seba[se] = parent_.GetBdrAttribute(pbe);
+            }
+            parent_ledge[se] = ple;
+         }
+      }
+
+#ifdef MFEM_DEBUG
+      auto pre_seba = seba;
+#endif
+      sedge_comm.Reduce(seba, GroupCommunicator::Sum);
+      sedge_comm.Bcast<int>(seba, 0);
+#ifdef MFEM_DEBUG
+      {
+         Array<int> fail_indices;
+         fail_indices.Reserve(seba.Size());
+         for (int i = 0; i < seba.Size(); i++)
+            if (pre_seba[i] != 0 && pre_seba[i] != seba[i])
+            {
+               fail_indices.Append(i);
+            }
+         MFEM_ASSERT(fail_indices.Size() == 0, [&]()
+         {
+            std::stringstream msg;
+            msg << "More than one rank found attribute on shared edge: ";
+            for (auto x : fail_indices)
+            {
+               msg << x << ' ';
+            }
+            return msg.str();
+         }());
+      }
+#endif
+      int nghost = 0;
+      for (auto x : seba)
+         if (x > 0) { ++nghost; }
+
+      lface_boundary_attribute.reserve(nghost);
+      for (int i = 0; i < seba.Size(); i++)
+         if (seba[i] > 0)
+         {
+            MFEM_ASSERT(parent_ledge[i] > -1, i);
+            lface_boundary_attribute[parent_ledge[i]] = seba[i];
+         }
+   }
+   else if (Dim == 1)
+   {
+      GroupCommunicator svert_comm(parent_.gtopo);
+      parent_.GetSharedVertexCommunicator(svert_comm);
+      int nsvtx = svert_comm.GroupLDofTable().Size_of_connections();
+
+      Array<int> svba(nsvtx), parent_lvtx(nsvtx);
+      svba = 0; parent_lvtx = -1;
+      for (int g = 1, sv = 0; g < parent_.GetNGroups(); g++)
+      {
+         for (int gv = 0; gv < parent_.GroupNVertices(g); gv++, sv++)
+         {
+            // Group size of a shared vertex is always 2
+            int plv = parent_.GroupVertex(g, gv);
+            auto pbe = face_to_be[plv];
+            if (pbe >= 0)
+            {
+               svba[sv] = parent_.GetBdrAttribute(pbe);
+            }
+            parent_lvtx[sv] = plv;
+         }
+      }
+
+#ifdef MFEM_DEBUG
+      auto pre_svba = svba;
+#endif
+      svert_comm.Reduce(svba, GroupCommunicator::Sum);
+      svert_comm.Bcast<int>(svba, 0);
+#ifdef MFEM_DEBUG
+      {
+         Array<int> fail_indices;
+         fail_indices.Reserve(svba.Size());
+         for (int i = 0; i < svba.Size(); i++)
+            if (pre_svba[i] != 0 && pre_svba[i] != svba[i])
+            {
+               fail_indices.Append(i);
+            }
+         MFEM_ASSERT(fail_indices.Size() == 0, [&]()
+         {
+            std::stringstream msg;
+            msg << "More than one rank found attribute on shared vertex: ";
+            for (auto x : fail_indices)
+            {
+               msg << x << ' ';
+            }
+            return msg.str();
+         }());
+      }
+#endif
+      int nghost = 0;
+      for (auto x : svba)
+         if (x > 0) { ++nghost; }
+
+      lface_boundary_attribute.reserve(nghost);
+      for (int i = 0; i < svba.Size(); i++)
+         if (svba[i] > 0)
+         {
+            MFEM_ASSERT(parent_lvtx[i] > -1, i);
+            lface_boundary_attribute[parent_lvtx[i]] = svba[i];
+         }
+   }
+   return lface_boundary_attribute;
+}
+
+
 void ParSubMesh::Transfer(const ParGridFunction &src, ParGridFunction &dst)
 {
-   ParTransferMap map(src, dst);
-   map.Transfer(src, dst);
+   CreateTransferMap(src, dst).Transfer(src, dst);
 }
 
 ParTransferMap ParSubMesh::CreateTransferMap(const ParGridFunction &src,

mesh/submesh/psubmesh.hpp

@@ -24,6 +24,8 @@
 namespace mfem
 {
 
+class ParNCSubMesh;
+
 /**
  * @brief Subdomain representation of a topological parent in another ParMesh.
  *
@@ -50,11 +52,13 @@ namespace mfem
 
 class ParSubMesh : public ParMesh
 {
+   friend class ParNCSubMesh;
 public:
+   using From = SubMesh::From; ///< Convenience type-alias.
    ParSubMesh() = delete;
 
    /**
-    * @brief Create a domain ParSubMesh from it's parent.
+    * @brief Create a domain ParSubMesh from its parent.
     *
     * The ParSubMesh object expects the parent ParMesh object to be valid for
     * the entire object lifetime. The @a domain_attributes have to mark exactly
@@ -64,10 +68,10 @@ public:
     * @param[in] domain_attributes Domain attributes to extract
     */
    static ParSubMesh CreateFromDomain(const ParMesh &parent,
-                                      Array<int> &domain_attributes);
+                                      const Array<int> &domain_attributes);
 
    /**
-   * @brief Create a surface ParSubMesh from it's parent.
+   * @brief Create a surface ParSubMesh from its parent.
    *
    * The ParSubMesh object expects the parent ParMesh object to be valid for the
    * entire object lifetime. The @a boundary_attributes have to mark exactly one
@@ -77,7 +81,7 @@ public:
    * @param[in] boundary_attributes Boundary attributes to extract
    */
    static ParSubMesh CreateFromBoundary(const ParMesh &parent,
-                                        Array<int> &boundary_attributes);
+                                        const Array<int> &boundary_attributes);
 
    /**
     * @brief Get the parent ParMesh object
@@ -118,6 +122,16 @@ public:
       return parent_vertex_ids_;
    }
 
+   /**
+    * @brief Get the parent edge id map
+    *
+    * Submesh edge id (array index) to parent Mesh edge id.
+    */
+   const Array<int>& GetParentEdgeIDMap() const
+   {
+      return parent_edge_ids_;
+   }
+
    /**
     * @brief Get the parent face id map.
     *
@@ -139,13 +153,51 @@ public:
    }
 
    /**
-    * @brief Get the ParSubMesh face id map.
-    *
-    * ParMesh face id (array index) to ParSubMesh face id.
+    * @brief Get the submesh element corresponding to a parent element. -1 ==
+    * not present.
+    * @param pe The parent element id.
+    * @return int
     */
-   const Array<int>& GetParentToSubMeshFaceIDMap() const
+   int GetSubMeshElementFromParent(int pe) const
    {
-      return parent_to_submesh_face_ids_;
+      return (pe == -1 || pe >= parent_to_submesh_element_ids_.Size())
+             ? -1 : parent_to_submesh_element_ids_[pe];
+   }
+
+   /**
+    * @brief Get the submesh vertex corresponding to a parent element. -1 == not
+    * present.
+    * @param pv The parent vertex id.
+    * @return int
+    */
+   int GetSubMeshVertexFromParent(int pv) const
+   {
+      return (pv == -1 || pv >= parent_to_submesh_vertex_ids_.Size())
+             ? -1 : parent_to_submesh_vertex_ids_[pv];
+   }
+
+   /**
+    * @brief Get the submesh edge corresponding to a parent element. -1 == not
+    * present.
+    * @param pe The parent edge id.
+    * @return int
+    */
+   int GetSubMeshEdgeFromParent(int pe) const
+   {
+      return (pe == -1 || pe >= parent_to_submesh_edge_ids_.Size())
+             ? pe : parent_to_submesh_edge_ids_[pe];
+   }
+
+   /**
+    * @brief Get the submesh face corresponding to a parent element. -1 == not
+    * present.
+    * @param pf The parent face id.
+    * @return int
+    */
+   int GetSubMeshFaceFromParent(int pf) const
+   {
+      return (pf == -1 || pf >= parent_to_submesh_face_ids_.Size())
+             ? pf : parent_to_submesh_face_ids_[pf];
    }
 
    /**
@@ -183,7 +235,8 @@ public:
    }
 
 private:
-   ParSubMesh(const ParMesh &parent, SubMesh::From from, Array<int> &attributes);
+   ParSubMesh(const ParMesh &parent, SubMesh::From from,
+              const Array<int> &attributes);
 
    /**
     * @brief Find shared vertices on the ParSubMesh.
@@ -223,8 +276,8 @@ private:
    /**
     * @brief Find shared edges on the ParSubMesh.
     *
-    * Uses the parent GroupCommunicator to determine shared edges.
-    * Collective. Limited to 32 ranks.
+    * Uses the parent GroupCommunicator to determine shared edges. Collective.
+    * Limited to groups containing less than 32 ranks.
     *
     * See FindSharedVerticesRanks for the encoding for @a rhe.
     *
@@ -232,6 +285,7 @@ private:
     */
    void FindSharedEdgesRanks(Array<int> &rhe);
 
+
    /**
     * @brief Find shared faces on the ParSubMesh.
     *
@@ -275,10 +329,10 @@ private:
     * @param[in,out] groups
     * @param[in,out] rht Encoding of which rank contains which face triangle.
     * The output is reused s.t. the array index i (the face triangle id) is the
-    * associated group.
+    * associated group. "Rank Has Triangle"
     * @param[in,out] rhq Encoding of which rank contains which face
     * quadrilateral. The output is reused s.t. the array index i (the face
-    * quadrilateral id) is the associated group.
+    * quadrilateral id) is the associated group. "Rank Has Quad"
     */
    void AppendSharedFacesGroups(ListOfIntegerSets &groups, Array<int>& rht,
                                 Array<int> &rhq);
@@ -342,15 +396,22 @@ private:
    void BuildSharedFacesMapping(const int nstrias, const Array<int>& rht,
                                 const int nsquads, const Array<int>& rhq);
 
+
+   std::unordered_map<int, int>
+   FindGhostBoundaryElementAttributes() const;
+
    /// The parent Mesh
    const ParMesh &parent_;
 
-   /// Indicator from which part of the parent ParMesh the ParSubMesh is going to
-   /// be created.
+   /// Optional nonconformal submesh. Managed via pncmesh pointer in base class.
+   ParNCSubMesh *pncsubmesh_;
+
+   /// Indicator from which part of the parent ParMesh the ParSubMesh is going
+   /// to be created.
    SubMesh::From from_;
 
-   /// Attributes on the parent ParMesh on which the ParSubMesh is created. Could
-   /// either be domain or boundary attributes (determined by from_).
+   /// Attributes on the parent ParMesh on which the ParSubMesh is created.
+   /// Could either be domain or boundary attributes (determined by from_).
    Array<int> attributes_;
 
    /// Mapping from ParSubMesh element ids (index of the array), to the parent
@@ -369,10 +430,14 @@ private:
    /// ParMesh face ids.
    Array<int> parent_face_ids_;
 
-   /// Mapping from SubMesh face ids (index of the array), to the orientation
-   /// of the face relative to the parent face.
+   /// Mapping from SubMesh face ids (index of the array), to the orientation of
+   /// the face relative to the parent face.
    Array<int> parent_face_ori_;
 
+   /// Mapping from parent ParMesh element ids (index of the array), to the
+   /// ParSubMesh element ids. Inverse map of parent_element_ids_.
+   Array<int> parent_to_submesh_element_ids_;
+
    /// Mapping from parent ParMesh vertex ids (index of the array), to the
    /// ParSubMesh vertex ids. Inverse map of parent_vertex_ids_.
    Array<int> parent_to_submesh_vertex_ids_;

mesh/submesh/ptransfermap.hpp

@@ -104,10 +104,10 @@ private:
    std::unique_ptr<const ParFiniteElementSpace> root_fes_;
 
    /// Pointer to the supplemental FiniteElementCollection used with root_fes_.
-   /// This is only used if this TransferMap represents a SubMesh to
-   /// SubMesh transfer where the root requires a different type of collection
-   /// than the SubMesh objects. For example, when the subpaces are L2 on
-   /// boundaries of the parent mesh and the root space can be RT.
+   /// This is only used if this TransferMap represents a SubMesh to SubMesh
+   /// transfer where the root requires a different type of collection than the
+   /// SubMesh objects. For example, when the subpaces are L2 on boundaries of
+   /// the parent mesh and the root space can be RT.
    std::unique_ptr<const FiniteElementCollection> root_fec_;
 
    const GroupCommunicator *root_gc_ = nullptr;

mesh/submesh/submesh.cpp

@@ -12,36 +12,34 @@
 #include "submesh.hpp"
 #include "submesh_utils.hpp"
 #include "../../fem/gridfunc.hpp"
+#include "../ncmesh.hpp"
+#include "ncsubmesh.hpp"
 
 namespace mfem
 {
 
 SubMesh SubMesh::CreateFromDomain(const Mesh &parent,
-                                  Array<int> domain_attributes)
+                                  const Array<int> &domain_attributes)
 {
    return SubMesh(parent, From::Domain, domain_attributes);
 }
 
 SubMesh SubMesh::CreateFromBoundary(const Mesh &parent,
-                                    Array<int> boundary_attributes)
+                                    const Array<int> &boundary_attributes)
 {
    return SubMesh(parent, From::Boundary, boundary_attributes);
 }
 
 SubMesh::SubMesh(const Mesh &parent, From from,
-                 Array<int> attributes) : parent_(parent), from_(from), attributes_(attributes)
+                 const Array<int> &attributes) : parent_(&parent), from_(from),
+   attributes_(attributes)
 {
-   if (parent.Nonconforming())
-   {
-      MFEM_ABORT("SubMesh does not support non-conforming meshes");
-   }
-
    if (from == From::Domain)
    {
       InitMesh(parent.Dimension(), parent.SpaceDimension(), 0, 0, 0);
 
       std::tie(parent_vertex_ids_,
-               parent_element_ids_) = SubMeshUtils::AddElementsToMesh(parent_, *this,
+               parent_element_ids_) = SubMeshUtils::AddElementsToMesh(parent, *this,
                                                                       attributes_);
    }
    else if (from == From::Boundary)
@@ -49,39 +47,83 @@ SubMesh::SubMesh(const Mesh &parent, From from,
       InitMesh(parent.Dimension() - 1, parent.SpaceDimension(), 0, 0, 0);
 
       std::tie(parent_vertex_ids_,
-               parent_element_ids_) = SubMeshUtils::AddElementsToMesh(parent_, *this,
+               parent_element_ids_) = SubMeshUtils::AddElementsToMesh(parent, *this,
                                                                       attributes_, true);
    }
 
-   FinalizeTopology(true);
+   parent_to_submesh_vertex_ids_.SetSize(parent.GetNV());
+   parent_to_submesh_vertex_ids_ = -1;
+   for (int i = 0; i < parent_vertex_ids_.Size(); i++)
+   {
+      parent_to_submesh_vertex_ids_[parent_vertex_ids_[i]] = i;
+   }
+
+   parent_to_submesh_element_ids_.SetSize(from == From::Boundary ? parent.GetNBE()
+                                          : parent.GetNE());
+   parent_to_submesh_element_ids_ = -1;
+   for (int i = 0; i < parent_element_ids_.Size(); i++)
+   {
+      parent_to_submesh_element_ids_[parent_element_ids_[i]] = i;
+   }
+
+   FinalizeTopology(false);
+
+   if (parent.Nonconforming())
+   {
+      ncmesh = new NCSubMesh(*this, *parent.ncmesh, from, attributes);
+      ncsubmesh_ = dynamic_cast<NCSubMesh*>(ncmesh);
+      InitFromNCMesh(*ncsubmesh_);
+      ncsubmesh_->OnMeshUpdated(this);
+
+      // Update the submesh to parent vertex mapping, ncsubmesh_ reordered the
+      // vertices so the map to parent is no longer valid.
+      parent_to_submesh_vertex_ids_ = -1;
+      for (int i = 0; i < parent_vertex_ids_.Size(); i++)
+      {
+         // vertex -> node -> parent node -> parent vertex
+         auto node = ncsubmesh_->vertex_nodeId[i];
+         auto parent_node = ncsubmesh_->parent_node_ids_[node];
+         auto parent_vertex = parent.ncmesh->GetNodeVertex(parent_node);
+         parent_vertex_ids_[i] = parent_vertex;
+         parent_to_submesh_vertex_ids_[parent_vertex] = i;
+      }
+      GenerateNCFaceInfo();
+      SetAttributes();
+   }
+
+   DSTable v2v(parent_->GetNV());
+   parent_->GetVertexToVertexTable(v2v);
+   for (int i = 0; i < NumOfEdges; i++)
+   {
+      Array<int> lv;
+      GetEdgeVertices(i, lv);
+
+      // Find vertices/edge in parent mesh
+      int parent_edge_id = v2v(parent_vertex_ids_[lv[0]],
+                               parent_vertex_ids_[lv[1]]);
+      parent_edge_ids_.Append(parent_edge_id);
+   }
+
+   parent_to_submesh_edge_ids_.SetSize(parent.GetNEdges());
+   parent_to_submesh_edge_ids_ = -1;
+   for (int i = 0; i < parent_edge_ids_.Size(); i++)
+   {
+      parent_to_submesh_edge_ids_[parent_edge_ids_[i]] = i;
+   }
 
    if (Dim == 3)
    {
       parent_face_ids_ = SubMeshUtils::BuildFaceMap(parent, *this,
                                                     parent_element_ids_);
 
-      Array<int> parent_face_to_be = parent.GetFaceToBdrElMap();
-      int max_bdr_attr = parent.bdr_attributes.Max();
-
-      for (int i = 0; i < NumOfBdrElements; i++)
+      parent_to_submesh_face_ids_.SetSize(parent.GetNFaces());
+      parent_to_submesh_face_ids_ = -1;
+      for (int i = 0; i < parent_face_ids_.Size(); i++)
       {
-         int pbeid = parent_face_to_be[parent_face_ids_[GetBdrElementFaceIndex(i)]];
-         if (pbeid != -1)
-         {
-            int attr = parent.GetBdrElement(pbeid)->GetAttribute();
-            GetBdrElement(i)->SetAttribute(attr);
-         }
-         else
-         {
-            // This case happens when a domain is extracted, but the root parent
-            // mesh didn't have a boundary element on the surface that defined
-            // it's boundary. It still creates a valid mesh, so we allow it.
-            GetBdrElement(i)->SetAttribute(max_bdr_attr + 1);
-         }
+         parent_to_submesh_face_ids_[parent_face_ids_[i]] = i;
       }
 
       parent_face_ori_.SetSize(NumOfFaces);
-
       for (int i = 0; i < NumOfFaces; i++)
       {
          Array<int> sub_vert;
@@ -95,7 +137,6 @@ SubMesh::SubMesh(const Mesh &parent, From from,
 
          Array<int> par_vert;
          parent.GetFaceVertices(parent_face_ids_[i], par_vert);
-
          if (par_vert.Size() == 3)
          {
             parent_face_ori_[i] = GetTriOrientation(par_vert, sub_par_vert);
@@ -112,6 +153,14 @@ SubMesh::SubMesh(const Mesh &parent, From from,
       {
          parent_edge_ids_ = SubMeshUtils::BuildFaceMap(parent, *this,
                                                        parent_element_ids_);
+
+         parent_to_submesh_edge_ids_.SetSize(parent.GetNEdges());
+         parent_to_submesh_edge_ids_ = -1;
+         for (int i = 0; i < parent_edge_ids_.Size(); i++)
+         {
+            parent_to_submesh_edge_ids_[parent_edge_ids_[i]] = i;
+         }
+
          Array<int> parent_face_to_be = parent.GetFaceToBdrElMap();
          int max_bdr_attr = parent.bdr_attributes.Max();
 
@@ -125,9 +174,10 @@ SubMesh::SubMesh(const Mesh &parent, From from,
             }
             else
             {
-               // This case happens when a domain is extracted, but the root parent
-               // mesh didn't have a boundary element on the surface that defined
-               // it's boundary. It still creates a valid mesh, so we allow it.
+               // This case happens when a domain is extracted, but the root
+               // parent mesh didn't have a boundary element on the surface that
+               // defined it's boundary. It still creates a valid mesh, so we
+               // allow it.
                GetBdrElement(i)->SetAttribute(max_bdr_attr + 1);
             }
          }
@@ -172,6 +222,19 @@ SubMesh::SubMesh(const Mesh &parent, From from,
       }
    }
 
+   SubMeshUtils::AddBoundaryElements(*this);
+
+   if (Dim > 1)
+   {
+      delete el_to_edge;
+      el_to_edge = new Table;
+      NumOfEdges = GetElementToEdgeTable(*el_to_edge);
+   }
+   if (Dim > 2)
+   {
+      GetElementToFaceTable();
+   }
+
    // If the parent Mesh has nodes and therefore is defined on a higher order
    // geometry, we define this SubMesh as a curved Mesh and transfer the
    // GridFunction from the parent Mesh to the SubMesh.
@@ -195,8 +258,7 @@ SubMesh::SubMesh(const Mesh &parent, From from,
 
 void SubMesh::Transfer(const GridFunction &src, GridFunction &dst)
 {
-   TransferMap map(src, dst);
-   map.Transfer(src, dst);
+   CreateTransferMap(src, dst).Transfer(src, dst);
 }
 
 TransferMap SubMesh::CreateTransferMap(const GridFunction &src,

mesh/submesh/submesh.hpp

@@ -14,11 +14,12 @@
 
 #include "../mesh.hpp"
 #include "transfermap.hpp"
-#include <unordered_map>
 
 namespace mfem
 {
 
+class NCSubMesh;
+
 /**
  * @brief Subdomain representation of a topological parent in another Mesh.
  *
@@ -41,17 +42,18 @@ namespace mfem
  */
 class SubMesh : public Mesh
 {
+   friend class NCSubMesh;
 public:
    /// Indicator from which part of the parent Mesh the SubMesh is created.
-   enum From
+   enum class From
    {
       Domain,
       Boundary
    };
 
-   static const int GENERATED_ATTRIBUTE = 900;
-
    SubMesh() = delete;
+   SubMesh(SubMesh &&) = default;
+   SubMesh &operator=(SubMesh &&) = default;
 
    /**
     * @brief Create a domain SubMesh from its parent.
@@ -64,7 +66,7 @@ public:
     * @param[in] domain_attributes Domain attributes to extract
     */
    static SubMesh CreateFromDomain(const Mesh &parent,
-                                   Array<int> domain_attributes);
+                                   const Array<int> &domain_attributes);
 
    /**
    * @brief Create a surface SubMesh from its parent.
@@ -78,22 +80,18 @@ public:
 
    */
    static SubMesh CreateFromBoundary(const Mesh &parent,
-                                     Array<int> boundary_attributes);
+                                     const Array<int> &boundary_attributes);
 
-   /**
-    * @brief Get the parent Mesh object
-    *
-    */
+   ///Get the parent Mesh object
    const Mesh* GetParent() const
    {
-      return &parent_;
+      return parent_;
    }
 
    /**
     * @brief Get the From indicator.
     *
-    * Indicates whether the SubMesh has been created from a domain or
-    * surface.
+    * Indicates whether the SubMesh has been created from a domain or surface.
     */
    From GetFrom() const
    {
@@ -113,13 +111,23 @@ public:
    /**
     * @brief Get the face id map
     *
-    * SubMesh element id (array index) to parent Mesh face id.
+    * SubMesh face id (array index) to parent Mesh face id.
     */
    const Array<int>& GetParentFaceIDMap() const
    {
       return parent_face_ids_;
    }
 
+   /**
+    * @brief Get the edge id map
+    *
+    * Submesh edge id (array index) to parent Mesh edge id.
+    */
+   const Array<int>& GetParentEdgeIDMap() const
+   {
+      return parent_edge_ids_;
+   }
+
    /**
     * @brief Get the relative face orientations
     *
@@ -140,6 +148,47 @@ public:
       return parent_vertex_ids_;
    }
 
+   /**
+    * @brief Get the submesh element corresponding to a parent element. -1 ==
+    * not present.
+    * @param pe The parent element id.
+    * @return int
+    */
+   int GetSubMeshElementFromParent(int pe) const
+   {
+      return pe == -1 ? pe : parent_to_submesh_element_ids_[pe];
+   }
+   /**
+    * @brief Get the submesh vertex corresponding to a parent element. -1 == not
+    * present.
+    * @param pv The parent vertex id.
+    * @return int
+    */
+   int GetSubMeshVertexFromParent(int pv) const
+   {
+      return pv == -1 ? pv : parent_to_submesh_vertex_ids_[pv];
+   }
+   /**
+    * @brief Get the submesh edge corresponding to a parent element. -1 == not
+    * present.
+    * @param pe The parent edge id.
+    * @return int
+    */
+   int GetSubMeshEdgeFromParent(int pe) const
+   {
+      return pe == -1 ? pe : parent_to_submesh_edge_ids_[pe];
+   }
+   /**
+    * @brief Get the submesh face corresponding to a parent element. -1 == not
+    * present.
+    * @param pf The parent face id.
+    * @return int
+    */
+   int GetSubMeshFaceFromParent(int pf) const
+   {
+      return pf == -1 ? pf : parent_to_submesh_face_ids_[pf];
+   }
+
    /**
     * @brief Transfer the dofs of a GridFunction.
     *
@@ -156,8 +205,8 @@ public:
    /**
     * @brief Create a Transfer Map object.
     *
-    * The @a src GridFunction can either be defined on a Mesh or a
-    * SubMesh and is transferred appropriately.
+    * The @a src GridFunction can either be defined on a Mesh or a SubMesh and
+    * is transferred appropriately.
     *
     * @note Either @a src or @a dst has to be defined on a SubMesh.
     */
@@ -176,10 +225,13 @@ public:
 
 private:
    /// Private constructor
-   SubMesh(const Mesh &parent, From from, Array<int> attributes);
+   SubMesh(const Mesh &parent, From from, const Array<int> &attributes);
 
-   /// The parent Mesh
-   const Mesh &parent_;
+   /// The parent Mesh. Not owned.
+   const Mesh *parent_;
+
+   /// Optional nonconformal submesh. Managed via ncmesh pointer in base class.
+   NCSubMesh *ncsubmesh_;
 
    /// Indicator from which part of the parent ParMesh the ParSubMesh is going
    /// to be created.
@@ -205,11 +257,25 @@ private:
    /// face ids.
    Array<int> parent_face_ids_;
 
-   /// Mapping from SubMesh face ids (index of the array), to the orientation
-   /// of the face relative to the parent face.
+   /// Mapping from SubMesh face ids (index of the array), to the orientation of
+   /// the face relative to the parent face.
    Array<int> parent_face_ori_;
 
-   Array<int> face_to_be;
+   /// Mapping from parent Mesh vertex ids (index of the array), to the SubMesh
+   /// vertex ids. Inverse map of parent_element_ids_.
+   Array<int> parent_to_submesh_element_ids_;
+
+   /// Mapping from parent Mesh vertex ids (index of the array), to the SubMesh
+   /// vertex ids. Inverse map of parent_vertex_ids_.
+   Array<int> parent_to_submesh_vertex_ids_;
+
+   /// Mapping from parent Mesh edge ids (index of the array), to the SubMesh
+   /// edge ids. Inverse map of parent_edge_ids_.
+   Array<int> parent_to_submesh_edge_ids_;
+
+   /// Mapping from parent Mesh face ids (index of the array), to the SubMesh
+   /// face ids. Inverse map of parent_face_ids_.
+   Array<int> parent_to_submesh_face_ids_;
 };
 
 } // namespace mfem

mesh/submesh/submesh_utils.cpp

@@ -10,6 +10,12 @@
 // CONTRIBUTING.md for details.
 
 #include "submesh_utils.hpp"
+#include "ncsubmesh.hpp"
+#include "submesh.hpp"
+#include "pncsubmesh.hpp"
+#include "psubmesh.hpp"
+
+#include <numeric>
 
 namespace mfem
 {
@@ -31,7 +37,8 @@ int UniqueIndexGenerator::Get(int i, bool &new_index)
    }
 }
 
-bool ElementHasAttribute(const Element &el, const Array<int> &attributes)
+template <typename ElementT>
+bool ElementHasAttribute(const ElementT &el, const Array<int> &attributes)
 {
    for (int a = 0; a < attributes.Size(); a++)
    {
@@ -49,41 +56,38 @@ AddElementsToMesh(const Mesh& parent,
                   const Array<int> &attributes,
                   bool from_boundary)
 {
-   Array<int> parent_vertex_ids, parent_element_ids;
    UniqueIndexGenerator vertex_ids;
+   Array<int> parent_vertex_ids, parent_element_ids;
+   Array<int> vert, submesh_vert;
+
    const int ne = from_boundary ? parent.GetNBE() : parent.GetNE();
    for (int i = 0; i < ne; i++)
    {
       const Element *pel = from_boundary ?
                            parent.GetBdrElement(i) : parent.GetElement(i);
-      if (!ElementHasAttribute(*pel, attributes)) { continue; }
-
-      Array<int> v;
-      pel->GetVertices(v);
-      Array<int> submesh_v(v.Size());
-
-      for (int iv = 0; iv < v.Size(); iv++)
+      if (!HasAttribute(*pel, attributes)) { continue; }
+      pel->GetVertices(vert);
+      submesh_vert.SetSize(vert.Size());
+      for (int iv = 0; iv < vert.Size(); iv++)
       {
          bool new_vertex;
-         int mesh_vertex_id = v[iv];
+         int mesh_vertex_id = vert[iv];
          int submesh_vertex_id = vertex_ids.Get(mesh_vertex_id, new_vertex);
          if (new_vertex)
          {
             mesh.AddVertex(parent.GetVertex(mesh_vertex_id));
             parent_vertex_ids.Append(mesh_vertex_id);
          }
-         submesh_v[iv] = submesh_vertex_id;
+         submesh_vert[iv] = submesh_vertex_id;
       }
-
       Element *el = mesh.NewElement(from_boundary ?
                                     parent.GetBdrElementType(i) : parent.GetElementType(i));
-      el->SetVertices(submesh_v);
+      el->SetVertices(submesh_vert);
       el->SetAttribute(pel->GetAttribute());
       mesh.AddElement(el);
       parent_element_ids.Append(i);
    }
-   return std::tuple<Array<int>, Array<int>>(parent_vertex_ids,
-                                             parent_element_ids);
+   return {parent_vertex_ids, parent_element_ids};
 }
 
 void BuildVdofToVdofMap(const FiniteElementSpace& subfes,
@@ -94,7 +98,6 @@ void BuildVdofToVdofMap(const FiniteElementSpace& subfes,
 {
    auto *m = subfes.GetMesh();
    vdof_to_vdof_map.SetSize(subfes.GetVSize());
-
    const int vdim = parentfes.GetVDim();
 
    IntegrationPointTransformation Tr;
@@ -188,6 +191,29 @@ void BuildVdofToVdofMap(const FiniteElementSpace& subfes,
             (sub_sign * parent_sign > 0.0) ? parent_vdof : (-1-parent_vdof);
       }
    }
+
+#ifdef MFEM_DEBUG
+   auto tmp = vdof_to_vdof_map;
+   tmp.Sort();
+   tmp.Unique();
+
+   if (tmp.Size() != vdof_to_vdof_map.Size())
+   {
+      std::stringstream msg;
+      for (int i = 0; i < vdof_to_vdof_map.Size(); i++)
+         for (int j = i + 1; j < vdof_to_vdof_map.Size(); j++)
+         {
+            auto x = vdof_to_vdof_map[i];
+            auto y = vdof_to_vdof_map[j];
+            if (x == y)
+            {
+               msg << "i " << i << " (" << x << ") j " << j << " (" << y << ")\n";
+            }
+         }
+      MFEM_ABORT("vdof_to_vdof_map should be 1 to 1:\n" << msg.str());
+   }
+#endif
+
 }
 
 Array<int> BuildFaceMap(const Mesh& pm, const Mesh& sm,
@@ -227,5 +253,607 @@ Array<int> BuildFaceMap(const Mesh& pm, const Mesh& sm,
    return pfids;
 }
 
+template <typename SubMeshT>
+void AddBoundaryElements(SubMeshT &mesh,
+                         const std::unordered_map<int,int> &lface_to_boundary_attribute)
+{
+   mesh.Dimension();
+   const int num_codim_1 = [&mesh]()
+   {
+      auto Dim = mesh.Dimension();
+      if (Dim == 1) { return mesh.GetNV(); }
+      else if (Dim == 2) { return mesh.GetNEdges(); }
+      else if (Dim == 3) { return mesh.GetNFaces(); }
+      else { MFEM_ABORT("Invalid dimension."); return -1; }
+   }();
+
+   if (mesh.Dimension() == 3)
+   {
+      // In 3D we check for `bel_to_edge`. It shouldn't have been set
+      // previously.
+      mesh.DeleteBoundaryElementToEdge();
+   }
+   int NumOfBdrElements = 0;
+   for (int i = 0; i < num_codim_1; i++)
+   {
+      if (mesh.GetFaceInformation(i).IsBoundary())
+      {
+         NumOfBdrElements++;
+      }
+   }
+
+   Array<Element *> boundary;
+   Array<int> be_to_face;
+   boundary.Reserve(NumOfBdrElements);
+   be_to_face.Reserve(NumOfBdrElements);
+
+   const auto &parent = *mesh.GetParent();
+   const auto &parent_face_ids = mesh.GetParentFaceIDMap();
+   const auto &parent_edge_ids = mesh.GetParentEdgeIDMap();
+   const auto &parent_vertex_ids = mesh.GetParentVertexIDMap();
+   const auto &parent_face_to_be = parent.GetFaceToBdrElMap();
+   const auto &face_to_be = mesh.GetFaceToBdrElMap();
+   int max_bdr_attr = parent.bdr_attributes.Max();
+   for (int i = 0; i < num_codim_1; i++)
+   {
+      auto pfid = [&](int i)
+      {
+         switch (mesh.Dimension())
+         {
+            case 3: return parent_face_ids[i];
+            case 2: return parent_edge_ids[i];
+            case 1: return parent_vertex_ids[i];
+         }
+         MFEM_ABORT("!");
+         return -1;
+      };
+      if (mesh.GetFaceInformation(i).IsBoundary()
+          && (face_to_be.IsEmpty() || face_to_be[i] == -1))
+      {
+         auto * be = mesh.GetFace(i)->Duplicate(&mesh);
+
+         if (mesh.GetFrom() == SubMesh::From::Domain && mesh.Dimension() >= 2)
+         {
+            int pbeid = parent_face_to_be[pfid(i)];
+            if (pbeid != -1)
+            {
+               be->SetAttribute(parent.GetBdrAttribute(pbeid));
+            }
+            else
+            {
+               auto ghost_attr = lface_to_boundary_attribute.find(pfid(i));
+               int battr = ghost_attr != lface_to_boundary_attribute.end() ?
+                           ghost_attr->second : max_bdr_attr + 1;
+               be->SetAttribute(battr);
+            }
+         }
+         else
+         {
+            auto ghost_attr = lface_to_boundary_attribute.find(pfid(i));
+            int battr = ghost_attr != lface_to_boundary_attribute.end() ?
+                        ghost_attr->second : max_bdr_attr + 1;
+            be->SetAttribute(battr);
+         }
+         be_to_face.Append(i);
+         boundary.Append(be);
+      }
+   }
+
+   if (mesh.GetFrom() == SubMesh::From::Domain && mesh.Dimension() >= 2)
+   {
+      // Search for and count interior boundary elements
+      int InteriorBdrElems = 0;
+      for (int i=0; i<parent.GetNBE(); i++)
+      {
+         const int parentFaceIdx = parent.GetBdrElementFaceIndex(i);
+         const int submeshFaceIdx =
+            mesh.Dimension() == 3 ?
+            mesh.GetSubMeshFaceFromParent(parentFaceIdx) :
+            mesh.GetSubMeshEdgeFromParent(parentFaceIdx);
+
+         if (submeshFaceIdx == -1) { continue; }
+         if (mesh.GetFaceInformation(submeshFaceIdx).IsBoundary()) { continue; }
+         InteriorBdrElems++;
+      }
+
+      if (InteriorBdrElems > 0)
+      {
+         NumOfBdrElements += InteriorBdrElems;
+         boundary.Reserve(NumOfBdrElements);
+         be_to_face.Reserve(NumOfBdrElements);
+
+         // Search for and transfer interior boundary elements
+         for (int i = 0; i < parent.GetNBE(); i++)
+         {
+            const int parentFaceIdx = parent.GetBdrElementFaceIndex(i);
+            const int submeshFaceIdx =
+               mesh.GetSubMeshFaceFromParent(parentFaceIdx);
+
+            if (submeshFaceIdx == -1) { continue; }
+            if (mesh.GetFaceInformation(submeshFaceIdx).IsBoundary())
+            { continue; }
+
+            auto * be = mesh.GetFace(submeshFaceIdx)->Duplicate(&mesh);
+            be->SetAttribute(parent.GetBdrAttribute(i));
+            boundary.Append(be);
+            be_to_face.Append(submeshFaceIdx);
+         }
+      }
+   }
+   mesh.AddBdrElements(boundary, be_to_face);
+}
+
+// Explicit instantiations
+template void AddBoundaryElements(SubMesh &mesh,
+                                  const std::unordered_map<int,int> &);
+
+#ifdef MFEM_USE_MPI
+template void AddBoundaryElements(ParSubMesh &mesh,
+                                  const std::unordered_map<int,int> &);
+#endif
+
+namespace
+{
+/**
+ * @brief Helper class for storing and comparing arrays of face nodes.
+ * @details The comparison operator uses the sorted nodes and a lexicographic
+ * compare so that two different orientations of the same set of nodes will be
+ * identical. The actual nodes are stored unsorted as the ordering is important
+ * for constructing the leaf-root relations.
+ */
+struct FaceNodes
+{
+   std::array<int, NCMesh::MaxFaceNodes> nodes;
+   bool operator<(FaceNodes t2) const
+   {
+      std::array<int, NCMesh::MaxFaceNodes> t1 = nodes;
+      std::sort(t1.begin(), t1.end());
+      std::sort(t2.nodes.begin(), t2.nodes.end());
+      return std::lexicographical_compare(t1.begin(), t1.end(),
+                                          t2.nodes.begin(), t2.nodes.end());
+   };
+};
+
+/**
+ * @brief Establish the Geometry::Type from an array of nodes
+ *
+ * @param nodes
+ * @return Geometry::Type
+ */
+Geometry::Type FaceGeomFromNodes(const std::array<int, NCMesh::MaxFaceNodes>
+                                 &nodes)
+{
+   if (nodes[3] == -1) { return Geometry::Type::TRIANGLE; }
+   if (nodes[0] == nodes[1] && nodes[2] == nodes[3]) { return Geometry::Type::SEGMENT; }
+   return Geometry::Type::SQUARE;
+};
+
+} // namespace
+
+template<typename NCSubMeshT>
+void ConstructFaceTree(NCSubMeshT &submesh, const Array<int> &attributes)
+{
+   // Convenience references to avoid `submesh.` repeatedly.
+   auto &parent_node_ids = submesh.parent_node_ids_;
+   auto &parent_element_ids = submesh.parent_element_ids_;
+   auto &parent_to_submesh_node_ids = submesh.parent_to_submesh_node_ids_;
+   auto &parent_to_submesh_element_ids = submesh.parent_to_submesh_element_ids_;
+   const auto &parent = *submesh.GetParent();
+
+   // Collect parent vertex nodes to add in sequence. Map from parent nodes to
+   // the new element in the ncsubmesh.
+   UniqueIndexGenerator node_ids;
+   std::map<FaceNodes, int> pnodes_new_elem;
+   std::set<int> new_nodes;
+   parent_to_submesh_element_ids.reserve(parent.GetNumFaces());
+   parent_element_ids.Reserve(parent.GetNumFaces());
+   // Base class cast then const cast because GetFaceList uses just in time
+   // construction.
+   const auto &face_list = const_cast<NCMesh&>(static_cast<const NCMesh&>
+                                               (parent)).GetFaceList();
+   // Double indexing loop because begin() and end() do not align with index 0
+   // and size-1.
+   for (int i = 0, ipe = 0; ipe < parent.GetNumFaces(); i++)
+   {
+      const auto &face = parent.GetFace(i);
+      if (face.Unused()) { continue; }
+      ipe++; // actual possible parent element.
+      if (!HasAttribute(face, attributes)
+          || face_list.GetMeshIdType(face.index) == NCMesh::NCList::MeshIdType::MASTER
+         ) { continue; }
+
+      FaceNodes fn{submesh.parent_->FindFaceNodes(face)};
+      if (pnodes_new_elem.find(fn) != pnodes_new_elem.end()) { continue; }
+
+      // TODO: Internal nc submesh can be constructed and solved on, but the
+      // transfer to the parent mesh can be erroneous, this is likely due to not
+      // treating the changing orientation of internal faces for ncmesh within
+      // the ptransfermap.
+      MFEM_ASSERT(face.elem[0] < 0 || face.elem[1] < 0,
+                  "Internal nonconforming boundaries are not reliably supported yet.");
+      auto face_geom = FaceGeomFromNodes(fn.nodes);
+      int new_elem_id = submesh.AddElement(face_geom, face.attribute);
+
+      // Rank needs to be established by presence (or lack of) in the submesh.
+      submesh.elements[new_elem_id].rank = [&parent, &face]()
+      {
+         auto rank0 = face.elem[0] >= 0 ? parent.GetElement(face.elem[0]).rank : -1;
+         auto rank1 = face.elem[1] >= 0 ? parent.GetElement(face.elem[1]).rank : -1;
+         if (rank0 < 0) { return rank1; }
+         if (rank1 < 0) { return rank0; }
+         return rank0 < rank1 ? rank0 : rank1;
+      }();
+      pnodes_new_elem[fn] = new_elem_id;
+      parent_element_ids.Append(i);
+      parent_to_submesh_element_ids[i] = new_elem_id;
+
+      // Copy in the parent nodes. These will be relabeled once the tree is
+      // built.
+      std::copy(fn.nodes.begin(), fn.nodes.end(), submesh.elements[new_elem_id].node);
+      for (auto x : fn.nodes)
+         if (x != -1)
+         {
+            new_nodes.insert(x);
+         }
+      auto &gi = submesh.GI[face_geom];
+      gi.InitGeom(face_geom);
+      for (int e = 0; e < gi.ne; e++)
+      {
+         new_nodes.insert(submesh.ParentNodes().FindId(fn.nodes[gi.edges[e][0]],
+                                                       fn.nodes[gi.edges[e][1]]));
+      }
+
+      /*
+         - Check not top level face
+         - Check for parent of the newly entered element
+            - if not present, add in
+            - if present but different order and this path is non-ambiguous,
+               reorder so consistent with child elements.
+         - Set .parent in the newly entered element
+         Break if top level face or joined existing branch (without reordering).
+
+         child element indices will be set afterwards because the orientation can change
+         during traversal.
+      */
+      bool root_path_is_ambiguous=false;
+      bool fix_parent = false, tri_face = (face_geom == Geometry::TRIANGLE);
+      while (true)
+      {
+         int child = submesh.parent_->ParentFaceNodes(fn.nodes);
+         if (tri_face && child == 3)
+         {
+            // Traversing a central triangle face involves flipping the face orientation.
+            // Do not use this pathway for reordering any parent face's nodes.
+            root_path_is_ambiguous = true;
+         }
+
+         if (child == -1) // A root face
+         {
+            submesh.elements[new_elem_id].parent = -1;
+            break;
+         }
+         auto pelem = pnodes_new_elem.find(fn);
+         bool new_parent = pelem == pnodes_new_elem.end();
+         if (new_parent)
+         {
+            // Add in this parent
+            int pelem_id = submesh.AddElement(FaceGeomFromNodes(fn.nodes), face.attribute);
+            pelem = pnodes_new_elem.emplace(fn, pelem_id).first;
+            auto parent_face_id = submesh.ParentFaces().FindId(fn.nodes[0], fn.nodes[1],
+                                                               fn.nodes[2],
+                                                               fn.nodes[3]);
+            parent_element_ids.Append(parent_face_id);
+         }
+         else
+         {
+            // There are two scenarios where the parent nodes should be
+            // rearranged:
+            // 1. The found face is a slave, then the master might have been
+            //    added in reverse orientation
+            // 2. The parent face was added from the central face of a triangle,
+            //    the orientation of the parent face is only fixed relative to
+            //    the outer child faces not the interior. If either of these
+            //    scenarios, and there's a mismatch, then reorder the parent and
+            //    all ancestors if necessary.
+            if (!root_path_is_ambiguous &&
+                !std::equal(fn.nodes.begin(), fn.nodes.end(), pelem->first.nodes.begin()))
+            {
+               fix_parent = true;
+               auto pelem_id = pelem->second;
+               MFEM_ASSERT(!submesh.elements[pelem_id].IsLeaf(), pelem_id);
+
+               // Re-key the map, the existing entry is inconsistent with the tree.
+               pnodes_new_elem.erase(pelem->first);
+               pelem = pnodes_new_elem.emplace(fn, pelem_id).first;
+            }
+         }
+         // Ensure parent element is marked as non-leaf, and attach to the child.
+         submesh.elements[pelem->second].ref_type = submesh.Dim == 2 ? Refinement::XY :
+                                                    Refinement::X;
+         submesh.elements[new_elem_id].parent = pelem->second;
+
+         // If this was neither new nor a fixed parent, the higher levels of the
+         // tree have been built, otherwise we recurse up the tree to add more parents, or
+         // to potentially fix any ambiguously added FaceNodes.
+         if (!new_parent && !fix_parent) { break; }
+
+         new_elem_id = pelem->second;
+      }
+   }
+   parent_element_ids.ShrinkToFit();
+   MFEM_ASSERT(parent_element_ids.Size() == submesh.elements.Size(),
+               parent_element_ids.Size() << ' ' << submesh.elements.Size());
+
+   // All elements have been added, with their parents, and the nodal orientation of parents is
+   // consistent with children, but the children indices have not been marked. Traverse the
+   // tree from root to leaf to fill the child arrays.
+   for (const auto & fn_elem : pnodes_new_elem)
+   {
+      auto fn = fn_elem.first;
+      const auto &child_elem = submesh.elements[fn_elem.second];
+      if (child_elem.parent == -1) { continue; }
+      int child = submesh.parent_->ParentFaceNodes(fn.nodes);
+      MFEM_ASSERT(pnodes_new_elem[fn] == child_elem.parent,
+                  pnodes_new_elem[fn] << ' ' << child_elem.parent);
+      MFEM_ASSERT(submesh.elements[child_elem.parent].ref_type != char(0),
+                  int(submesh.elements[child_elem.parent].ref_type));
+      submesh.elements[child_elem.parent].child[child] = fn_elem.second;
+   }
+
+   /*
+      All elements have been added into the tree but a) The nodes are all from
+      the parent ncmesh b) The nodes do not know their parents c) The element
+      ordering is wrong, root elements are not first d) The parent and child
+      element numbers reflect the incorrect ordering
+
+      1. Add in nodes in the same order from the parent ncmesh
+      2. Compute reordering of elements with parent elements first, that is
+         stable across processors.
+   */
+   // Build an inverse (and consecutive) map.
+   Array<FaceNodes> new_elem_to_parent_face_nodes(pnodes_new_elem.size());
+   for (const auto &kv : pnodes_new_elem)
+   {
+      new_elem_to_parent_face_nodes[kv.second] = kv.first;
+   }
+   pnodes_new_elem.clear(); // no longer needed
+
+   // Add new nodes preserving parent mesh ordering
+   parent_node_ids.Reserve(static_cast<int>(new_nodes.size()));
+   parent_to_submesh_node_ids.reserve(new_nodes.size());
+   for (auto n : new_nodes)
+   {
+      bool new_node;
+      auto new_node_id = node_ids.Get(n, new_node);
+      MFEM_ASSERT(new_node, "!");
+      submesh.nodes.Alloc(new_node_id, new_node_id, new_node_id);
+      parent_node_ids.Append(n);
+      parent_to_submesh_node_ids[n] = new_node_id;
+   }
+   parent_node_ids.ShrinkToFit();
+   new_nodes.clear(); // not needed any more.
+
+   // Comparator for deciding order of elements. Building the ordering from the
+   // parent ncmesh ensures the root ordering is common across ranks.
+   auto comp_elements = [&](int l, int r)
+   {
+      const auto &elem_l = submesh.elements[l];
+      const auto &elem_r = submesh.elements[r];
+      if (elem_l.parent == elem_r.parent)
+      {
+         const auto &fnl = new_elem_to_parent_face_nodes[l].nodes;
+         const auto &fnr = new_elem_to_parent_face_nodes[r].nodes;
+         return std::lexicographical_compare(fnl.begin(), fnl.end(), fnr.begin(),
+                                             fnr.end());
+      }
+      else
+      {
+         return elem_l.parent < elem_r.parent;
+      }
+   };
+   Array<int> indices(submesh.elements.Size());
+   auto parental_sorted = [&]()
+   {
+      std::iota(indices.begin(), indices.end(), 0);
+      return std::is_sorted(indices.begin(), indices.end(), comp_elements);
+   };
+
+   Array<int> new_to_old(submesh.elements.Size()),
+         old_to_new(submesh.elements.Size());
+   while (!parental_sorted())
+   {
+      // Stably reorder elements in order of refinement, and by parental nodes
+      // within a nuclear family.
+      new_to_old.SetSize(submesh.elements.Size()),
+                         old_to_new.SetSize(submesh.elements.Size());
+      std::iota(new_to_old.begin(), new_to_old.end(), 0);
+      std::stable_sort(new_to_old.begin(), new_to_old.end(), comp_elements);
+      // Build the inverse relation for converting the old elements to new
+      for (int i = 0; i < submesh.elements.Size(); i++)
+      {
+         old_to_new[new_to_old[i]] = i;
+      }
+
+      // Permute whilst reordering new_to_old. Avoids unnecessary copies.
+      Permute(std::move(new_to_old), submesh.elements, parent_element_ids,
+              new_elem_to_parent_face_nodes);
+      parent_to_submesh_element_ids.clear();
+      for (int i = 0; i < parent_element_ids.Size(); i++)
+      {
+         if (parent_element_ids[i] == -1) {continue;}
+         parent_to_submesh_element_ids[parent_element_ids[i]] = i;
+      }
+
+      // Apply the new ordering to child and parent elements
+      for (auto &elem : submesh.elements)
+      {
+         if (!elem.IsLeaf())
+         {
+            // Parent rank is minimum of child ranks.
+            elem.rank = std::numeric_limits<int>::max();
+            for (int c = 0; c < NCMesh::MaxElemChildren && elem.child[c] >= 0; c++)
+            {
+               elem.child[c] = old_to_new[elem.child[c]];
+               elem.rank = std::min(elem.rank, submesh.elements[elem.child[c]].rank);
+            }
+         }
+         elem.parent = elem.parent == -1 ? -1 : old_to_new[elem.parent];
+      }
+   }
+
+   // Apply new node ordering to relations, and sign in on edges/vertices
+   for (auto &elem : submesh.elements)
+   {
+      if (elem.IsLeaf())
+      {
+         bool new_id;
+         auto &gi = submesh.GI[elem.Geom()];
+         gi.InitGeom(elem.Geom());
+         for (int e = 0; e < gi.ne; e++)
+         {
+            const int pid = submesh.ParentNodes().FindId(
+                               elem.node[gi.edges[e][0]], elem.node[gi.edges[e][1]]);
+            MFEM_ASSERT(pid >= 0,
+                        elem.node[gi.edges[e][0]] << ' ' << elem.node[gi.edges[e][1]]);
+            auto submesh_node_id = node_ids.Get(pid, new_id);
+            MFEM_ASSERT(!new_id, "!");
+            submesh.nodes[submesh_node_id].edge_refc++;
+         }
+         for (int n = 0; n < gi.nv; n++)
+         {
+            MFEM_ASSERT(parent_to_submesh_node_ids.find(elem.node[n]) !=
+                        parent_to_submesh_node_ids.end(), "!");
+            elem.node[n] = parent_to_submesh_node_ids[elem.node[n]];
+            submesh.nodes[elem.node[n]].vert_refc++;
+         }
+         // Register faces
+         for (int f = 0; f < gi.nf; f++)
+         {
+            auto *face = submesh.faces.Get(
+                            elem.node[gi.faces[f][0]],
+                            elem.node[gi.faces[f][1]],
+                            elem.node[gi.faces[f][2]],
+                            elem.node[gi.faces[f][3]]);
+            face->attribute = -1;
+            face->index = -1;
+         }
+      }
+   }
+}
+
+// Explicit instantiations
+template void ConstructFaceTree(NCSubMesh &submesh,
+                                const Array<int> &attributes);
+#ifdef MFEM_USE_MPI
+template void ConstructFaceTree(ParNCSubMesh &submesh,
+                                const Array<int> &attributes);
+#endif
+
+template <typename NCSubMeshT>
+void ConstructVolumeTree(NCSubMeshT &submesh, const Array<int> &attributes)
+{
+   // Convenience references to avoid `submesh.` repeatedly.
+   auto &parent_node_ids = submesh.parent_node_ids_;
+   auto &parent_element_ids = submesh.parent_element_ids_;
+   auto &parent_to_submesh_node_ids = submesh.parent_to_submesh_node_ids_;
+   auto &parent_to_submesh_element_ids = submesh.parent_to_submesh_element_ids_;
+   const auto &parent = *submesh.GetParent();
+
+   UniqueIndexGenerator node_ids;
+   parent_to_submesh_element_ids.reserve(parent.GetNumElements());
+   std::set<int> new_nodes;
+   for (int ipe = 0; ipe < parent.GetNumElements(); ipe++)
+   {
+      const auto& pe = parent.GetElement(ipe);
+      if (!HasAttribute(pe, attributes)) { continue; }
+      const int elem_id = submesh.AddElement(pe);
+      auto &el = submesh.elements[elem_id];
+      parent_element_ids.Append(ipe); // submesh -> parent
+      parent_to_submesh_element_ids[ipe] = elem_id; // parent -> submesh
+      if (!pe.IsLeaf()) { continue; }
+      const auto gi = submesh.GI[pe.Geom()];
+      for (int n = 0; n < gi.nv; n++)
+      {
+         new_nodes.insert(el.node[n]);
+      }
+      for (int e = 0; e < gi.ne; e++)
+      {
+         new_nodes.insert(submesh.ParentNodes().FindId(el.node[gi.edges[e][0]],
+                                                       el.node[gi.edges[e][1]]));
+      }
+   }
+
+   parent_node_ids.Reserve(static_cast<int>(new_nodes.size()));
+   parent_to_submesh_node_ids.reserve(new_nodes.size());
+   for (const auto &n : new_nodes)
+   {
+      bool new_node;
+      auto new_node_id = node_ids.Get(n, new_node);
+      MFEM_ASSERT(new_node, "!");
+      submesh.nodes.Alloc(new_node_id, new_node_id, new_node_id);
+      parent_node_ids.Append(n);
+      parent_to_submesh_node_ids[n] = new_node_id;
+   }
+
+   // Loop over elements and reference edges and faces (creating any nodes on
+   // first encounter).
+   for (auto &el : submesh.elements)
+   {
+      if (el.IsLeaf())
+      {
+         const auto gi = submesh.GI[el.Geom()];
+         bool new_id = false;
+
+         for (int n = 0; n < gi.nv; n++)
+         {
+            // Relabel nodes from parent to submesh.
+            el.node[n] = node_ids.Get(el.node[n], new_id);
+            MFEM_ASSERT(new_id == false, "Should not be new.");
+            submesh.nodes[el.node[n]].vert_refc++;
+         }
+         for (int e = 0; e < gi.ne; e++)
+         {
+            const int pid = submesh.ParentNodes().FindId(
+                               parent_node_ids[el.node[gi.edges[e][0]]],
+                               parent_node_ids[el.node[gi.edges[e][1]]]);
+            MFEM_ASSERT(pid >= 0, "Edge not found");
+            auto submesh_node_id = node_ids.Get(pid, new_id);
+            MFEM_ASSERT(new_id == false, "Should not be new.");
+            submesh.nodes[submesh_node_id].edge_refc++; // Register the edge
+         }
+         for (int f = 0; f < gi.nf; f++)
+         {
+            const int *fv = gi.faces[f];
+            const int pid = submesh.ParentFaces().FindId(
+                               parent_node_ids[el.node[fv[0]]],
+                               parent_node_ids[el.node[fv[1]]],
+                               parent_node_ids[el.node[fv[2]]],
+                               el.node[fv[3]] >= 0 ? parent_node_ids[el.node[fv[3]]]: - 1);
+            MFEM_ASSERT(pid >= 0, "Face not found");
+            const int id = submesh.faces.GetId(
+                              el.node[fv[0]], el.node[fv[1]], el.node[fv[2]], el.node[fv[3]]);
+            submesh.faces[id].attribute = submesh.ParentFaces()[pid].attribute;
+         }
+      }
+      else
+      {
+         // All elements have been collected, remap the child ids.
+         for (int i = 0; i < NCMesh::MaxElemChildren && el.child[i] >= 0; i++)
+         {
+            el.child[i] = parent_to_submesh_element_ids[el.child[i]];
+         }
+      }
+      el.parent = el.parent < 0 ? el.parent
+                  : parent_to_submesh_element_ids.at(el.parent);
+   }
+}
+
+// Explicit instantiations
+template void ConstructVolumeTree(NCSubMesh &submesh,
+                                  const Array<int> &attributes);
+#ifdef MFEM_USE_MPI
+template void ConstructVolumeTree(ParNCSubMesh &submesh,
+                                  const Array<int> &attributes);
+#endif
 } // namespace SubMeshUtils
 } // namespace mfem

mesh/submesh/submesh_utils.hpp

@@ -19,6 +19,9 @@
 
 namespace mfem
 {
+class NCSubMesh;
+class ParNCSubMesh;
+
 namespace SubMeshUtils
 {
 
@@ -40,15 +43,6 @@ struct UniqueIndexGenerator
    int Get(int i, bool &new_index);
 };
 
-/**
- * @brief Given an element @a el and a list of @a attributes, determine if that
- * element is in at least one attribute of @a attributes.
- *
- * @param el The element
- * @param attributes The attributes
- */
-bool ElementHasAttribute(const Element &el, const Array<int> &attributes);
-
 /**
  * @brief Given a Mesh @a parent and another Mesh @a mesh using the list of
  * attributes in @a attributes, this function adds matching elements with those
@@ -111,10 +105,10 @@ void BuildVdofToVdofMap(const FiniteElementSpace& subfes,
  * @tparam T The type of the input object which has to fulfill the
  * SubMesh::GetParent() interface.
  */
-template <class T, class RT = decltype(std::declval<T>().GetParent())>
-RT GetRootParent(const T &m)
+template <class T>
+auto GetRootParent(const T &m) -> decltype(std::declval<T>().GetParent())
 {
-   RT parent = m.GetParent();
+   auto parent = m.GetParent();
    while (true)
    {
       const T* next = dynamic_cast<const T*>(parent);
@@ -123,6 +117,154 @@ RT GetRootParent(const T &m)
    }
 }
 
+/**
+ * @brief Add boundary elements to the SubMesh.
+ * @details An attempt to call this function for anything other than SubMesh or
+ * ParSubMesh will result in a linker error as the template is only explicitly
+ * instantiated for those types.
+ * @param mesh The SubMesh to add boundary elements to.
+ * @param lface_to_boundary_attribute Map from local faces in the submesh to
+ * boundary attributes. Only necessary for interior boundary attributes of
+ * volume submeshes, where the face owning the attribute might be on a
+ * neighboring rank.
+ * @tparam SubMeshT The SubMesh type, options SubMesh and ParSubMesh.
+ */
+template <typename SubMeshT>
+void AddBoundaryElements(SubMeshT &mesh,
+                         const std::unordered_map<int,int> &lface_to_boundary_attribute = {});
+
+/**
+ * @brief Construct a nonconformal mesh (serial or parallel) for a surface
+ * submesh, from an existing nonconformal volume mesh (serial or parallel).
+ * @details This function is only instantiated for NCSubMesh and ParNCSubMesh
+ *    Attempting to use it with other classes will result in a linker error.
+ * @tparam NCSubMeshT The NCSubMesh type
+ * @param[out] submesh The surface submesh to be filled.
+ * @param attributes The set of attributes defining the submesh.
+ */
+template<typename NCSubMeshT>
+void ConstructFaceTree(NCSubMeshT &submesh, const Array<int> &attributes);
+
+/**
+ * @brief Construct a nonconformal mesh (serial or parallel) for a volume
+ * submesh, from an existing nonconformal volume mesh (serial or parallel).
+ * @details This function is only instantiated for NCSubMesh and ParNCSubMesh
+ *    Attempting to use it with other classes will result in a linker error.
+ * @tparam NCSubMeshT The NCSubMesh type
+ * @param[out] submesh The volume submesh to be filled from parent.
+ * @param attributes The set of attributes defining the submesh.
+ */
+template <typename NCSubMeshT>
+void ConstructVolumeTree(NCSubMeshT &submesh, const Array<int> &attributes);
+
+/**
+ * @brief Helper for checking if an object's attributes match a list
+ *
+ * @tparam T Object Type
+ * @param el Instance of T, requires method `GetAttribute()`
+ * @param attributes Set of attributes to match against
+ * @return true The attribute of el is contained within attributes
+ * @return false
+ */
+template <typename T>
+bool HasAttribute(const T &el, const Array<int> &attributes)
+{
+   for (int a = 0; a < attributes.Size(); a++)
+   {
+      if (el.GetAttribute() == attributes[a])
+      {
+         return true;
+      }
+   }
+   return false;
+}
+
+/**
+ * @brief Forwarding dispatch to HasAttribute for backwards compatability
+ *
+ * @param el Instance of T, requires method `GetAttribute()`
+ * @param attributes Set of attributes to match against
+ * @return true The attribute of el is contained within attributes
+ * @return false
+ */
+MFEM_DEPRECATED inline bool ElementHasAttribute(const Element &el,
+                                                const Array<int> &attributes)
+{
+   return HasAttribute(el,attributes);
+}
+
+/**
+ * @brief Apply permutation to a container type
+ *
+ * @tparam T1 Container type 1
+ * @tparam T2 Container type 2
+ * @tparam T3 Container type 3
+ * @param indices Set of indices that define the permutation
+ * @param t1 First collection to be permuted
+ * @param t2 Second collection to be permuted
+ * @param t3 Third collection to be permuted
+ */
+template <typename T1, typename T2, typename T3>
+void Permute(const Array<int>& indices, T1& t1, T2& t2, T3& t3)
+{
+   Permute(Array<int>(indices), t1, t2, t3);
+}
+
+/**
+ * @brief Apply permutation to a container type
+ * @details Sorts the indices variable in the process, thereby destroying the
+ * permutation.
+ *
+ * @tparam T1 Container type 1
+ * @tparam T2 Container type 2
+ * @tparam T3 Container type 3
+ * @param indices Set of indices that define the permutation
+ * @param t1 First collection to be permuted
+ * @param t2 Second collection to be permuted
+ * @param t3 Third collection to be permuted
+ */
+template <typename T1, typename T2, typename T3>
+void Permute(Array<int>&& indices, T1& t1, T2& t2, T3& t3)
+{
+   /*
+   TODO: In c++17 can replace this with a parameter pack expansion technique to
+   operate on arbitrary collections of reference accessible containers of
+   arbitrary type.
+   template <typename ...T> void Permute(Array<int>&&indices, T&... t)
+   {
+      for (int i = 0; i < indices.Size(); i++)
+      {
+         auto current = i;
+         while (i != indices[current])
+         {
+               auto next = indices[current];
+               // Lambda allows iteration over expansion in c++17
+               // https://stackoverflow.com/a/60136761
+               ([&]{std::swap(t[current], t[next]);} (), ...);
+               current = next;
+         }
+         indices[current] = current;
+      }
+   }
+   */
+
+   for (int i = 0; i < indices.Size(); i++)
+   {
+      auto current = i;
+      while (i != indices[current])
+      {
+         auto next = indices[current];
+         std::swap(t1[current], t1[next]);
+         std::swap(t2[current], t2[next]);
+         std::swap(t3[current], t3[next]);
+         indices[current] = current;
+         current = next;
+      }
+      indices[current] = current;
+   }
+}
+
+
 } // namespace SubMeshUtils
 } // namespace mfem
 

miniapps/adjoint/adjoint_advection_diffusion.cpp

@@ -323,11 +323,10 @@ int main(int argc, char *argv[])
    // Perform time-integration for the problem (looping over the time
    // iterations, ti, with a time-step dt).
    bool done = false;
-   for (int ti = 0; !done; )
+   for ( ; !done; )
    {
       real_t dt_real = max(dt, t_final - t);
       cvodes->Step(*U, t, dt_real);
-      ti++;
 
       done = (t >= t_final - 1e-8*dt);
 

miniapps/adjoint/cvsRoberts_ASAi_dns.cpp

@@ -221,11 +221,10 @@ int main(int argc, char *argv[])
    // Perform time-integration (looping over the time iterations, ti, with a
    // time-step dt).
    bool done = false;
-   for (int ti = 0; !done; )
+   while (!done)
    {
       real_t dt_real = max(dt, t_final - t);
       cvodes->Step(u, t, dt_real);
-      ti++;
 
       done = (t >= t_final - 1e-8*dt);
 

tests/unit/CMakeLists.txt

@@ -18,6 +18,7 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR})
 #    for d in general linalg mesh fem enzyme; do ls -1 $d/*.cpp; done
 set(UNIT_TESTS_SRCS
   general/test_array.cpp
+  general/test_mdspan.cpp
   general/test_arrays_by_name.cpp
   general/test_error.cpp
   general/test_mem.cpp

tests/unit/general/test_array.cpp

@@ -18,7 +18,7 @@ TEST_CASE("Array init-list construction", "[Array]")
 {
    int ContigData[6] = {6, 5, 4, 3, 2, 1};
    Array<int> a(ContigData, 6);
-   Array<int> b({6.0, 5.0, 4.0, 3.0, 2.0, 1.0});
+   Array<int> b{6, 5, 4, 3, 2, 1};
 
    for (int i = 0; i < a.Size(); i++)
    {
@@ -30,7 +30,7 @@ TEST_CASE("Array entry sorting", "[Array]")
 {
    int ContigData[6] = {6, 5, 4, 3, 2, 1};
    Array<int> a(ContigData, 6);
-   Array<int> b({1, 2, 3, 3, 2, 1});
+   Array<int> b{1, 2, 3, 3, 2, 1};
 
    a.Sort();
    b.Sort();
@@ -50,7 +50,7 @@ TEST_CASE("Array entry strict sorting", "[Array]")
 {
    int ContigData[6] = {6, 1, 4, 1, 2, 1};
    Array<int> a(ContigData, 6);
-   Array<int> b({1, 2, 3, 3, 2, 1});
+   Array<int> b{1, 2, 3, 3, 2, 1};
 
    a.Sort();
    b.Sort();

tests/unit/general/test_mdspan.cpp

@@ -0,0 +1,406 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifdef _WIN32
+#define _USE_MATH_DEFINES
+#include <cmath>
+#endif
+
+#include <list>
+#include <type_traits>
+
+#include "mfem.hpp"
+#include "unit_tests.hpp"
+
+#include "general/mdspan.hpp"
+#include "general/forall.hpp"
+
+#include "fem/mdgridfunc.hpp"
+#include "general/mdarray.hpp"
+#include "linalg/mdvector.hpp"
+
+using namespace mfem;
+
+static bool is_equal(const Vector &a, const Vector &b);
+
+TEST_CASE("MDArray", "[MDSpan][MDArray]")
+{
+   SECTION("Types")
+   {
+      MDArray<int,3> mda;
+      REQUIRE(mda.Size() == 0);
+      REQUIRE(std::is_same<decltype(mda.HostRead()), int const*>());
+      REQUIRE(std::is_same<decltype(mda.MDHostRead()), MDTensor<3, int const> const>());
+   }
+
+   SECTION("SetSize")
+   {
+      constexpr int NA = 11, NB = 22, NC = 33;
+      {
+
+         const int A = 7;
+         MDArray<int,3> abc;
+         abc.SetSize(NA, NB, NC);
+         abc = 7;
+         REQUIRE(abc.Size() == NA*NB*NC);
+         REQUIRE(abc.Read());
+         REQUIRE(abc.Write());
+         REQUIRE(abc.HostRead());
+         REQUIRE(abc.HostWrite());
+         REQUIRE(abc.MDRead()(0,0,0) == A);
+         REQUIRE(abc.MDWrite()(0,0,0) == A);
+         REQUIRE(abc.MDHostRead()(0,0,0) == A);
+         REQUIRE(abc.MDHostWrite()(0,0,0) == A);
+      }
+      {
+         MDArray<int,3> abc(NA, NB, NC);
+         REQUIRE(abc.Size() == NA*NB*NC);
+      }
+      {
+         const int A[6] = {0, 1, 2, 3, 4, 7};
+         MDArray<int,3,MDLayoutLeft<3>> abc_l(1,2,3);
+         MDArray<int,3,MDLayoutRight<3>> abc_r(1,2,3);
+
+         abc_l.Assign(A);
+         REQUIRE(abc_l.MDRead()(0,0,0) == 0);
+         REQUIRE(abc_l.MDRead()(0,1,2) == 7); // = 0 + 1( 1 + 2( 2)) = 5
+
+         abc_r.Assign(A);
+         REQUIRE(abc_r.MDRead()(0,0,0) == 0);
+         REQUIRE(abc_r.MDRead()(0,1,2) == 7); // = ((0)*2 + 1) * 3 + 2 = 5
+      }
+   }
+
+   SECTION("Offset")
+   {
+      constexpr int NA = 11, NB = 22, NC = 33;
+      constexpr int na =  0, nb =  1, nc =  2;
+      MDLayout<3> layout_012({na,nb,nc});
+
+      MDArray<int,3> abc(NA, NB, NC);
+      MDArray<int,3, MDLayout<3>> abc_ini(NA, NB, NC);
+      MDArray<int,3, MDLayout<3>> abc_set(NA, NB, NC);
+      abc_set.SetLayout(layout_012);
+
+      REQUIRE(abc_set.Offset(na,nb,nc) == abc.Offset(na,nb,nc));
+      REQUIRE(abc_set.Offset(na,nb,nc) == abc_ini.Offset(na,nb,nc));
+   }
+
+   SECTION("SetLayout")
+   {
+      constexpr int NA = 18, NB = 2, NC = 36;
+      // Fortran col major: (18, 2, 36)
+      //                    ( 0, 1,  2)
+      // = 0 + 18( 1 + 2( 2)) = 90
+      MDArray<int,3> left(NA,NB,NC); // default layout is LayoutLeft
+      REQUIRE(left.Offset(0,1,2) == 90);
+
+      // C/C++ row major: (18, 2, 36)
+      //                  ( 0, 1,  2)
+      // = 32( 2 + 36( 1 + 2(0))) = 38
+      // = ((0)*2 + 1) * 36 + 2
+      MDArray<int,3> right(NA, NB, NC);
+      right.SetLayout(MDLayout<3>({2,1,0}));
+      REQUIRE(right.Offset(0,1,2) == 38);
+
+      MDArray<int,3,MDLayoutRight<3>> right4(NA, NB, NC);
+      right4.SetLayout(MDLayoutRight<3>({2,1,0}));
+      REQUIRE(right4.Offset(0,1,2) == 38);
+   }
+}
+
+TEST_CASE("MDVector", "[MDSpan][MDVector]")
+{
+   SECTION("Types")
+   {
+      MDVector<3> mdv;
+      REQUIRE(mdv.Size() == 0);
+      REQUIRE(std::is_same<decltype(mdv.HostRead()), double const*>());
+      REQUIRE(std::is_same<decltype(mdv.MDHostRead()), MDTensor<3, double const> const>());
+   }
+
+   SECTION("SetSize")
+   {
+      constexpr int NA = 11, NB = 22, NC = 33;
+      {
+         MDVector<3> abc;
+         abc.SetSize(NA, NB, NC);
+         REQUIRE(abc.Size() == NA*NB*NC);
+         abc.HostRead();
+         abc.MDHostRead();
+      }
+      {
+         MDVector<3> abc(NA, NB, NC);
+         REQUIRE(abc.Size() == NA*NB*NC);
+      }
+   }
+
+   SECTION("Offset")
+   {
+      constexpr int NA = 11, NB = 22, NC = 33;
+      constexpr int na =  0, nb =  1, nc =  2;
+      MDLayout<3> layout_012({na,nb,nc});
+
+      MDVector<3> abc(NA, NB, NC);
+      MDVector<3, MDLayout<3>> abc_ini(NA, NB, NC);
+      MDVector<3, MDLayout<3>> abc_set(NA, NB, NC);
+      abc_set.SetLayout(layout_012);
+
+      REQUIRE(abc_set.Offset(na,nb,nc) == abc.Offset(na,nb,nc));
+      REQUIRE(abc_set.Offset(na,nb,nc) == abc_ini.Offset(na,nb,nc));
+   }
+
+   SECTION("SetLayout")
+   {
+      constexpr int NA = 18, NB = 2, NC = 36, ND = 32;
+      // Fortran col major: (N1:18, 2, 36, Nd:32)
+      //                    (    0, 1,  2,     3)
+      // = 0 + 18( 1 + 2( 2 + 36( 3))) = 3978
+      MDVector<4> left(NA,NB,NC,ND); // default layout is LayoutLeft
+      REQUIRE(left.Offset(0,1,2,3) == 3978);
+
+      // C/C++ row major: (N1:18, 2, 36, Nd:32)
+      //                  (    0, 1,  2,     3)
+      // = 3 + 32( 2 + 36( 1 + 2(0))) = 1219
+      // = (((0)*2 + 1) * 36 + 2) * 32 + 3
+      MDVector<4> right(NA, NB, NC, ND);
+      right.SetLayout(MDLayout<4>({3,2,1,0}));
+      REQUIRE(right.Offset(0,1,2,3) == 1219);
+
+      MDVector<4,MDLayoutRight<4>> right4(NA, NB, NC, ND);
+      right4.SetLayout(MDLayoutRight<4>({3,2,1,0}));
+      REQUIRE(right4.Offset(0,1,2,3) == 1219);
+   }
+}
+
+TEST_CASE("MDGridFunction layouts", "[MDSpan][MDGridFunction]")
+{
+   constexpr int NE = 7, NG = 3, NA = 5;
+
+   const bool all = launch_all_non_regression_tests;
+   auto p = all ? GENERATE(1,2) : 3;
+   auto nx = all ? GENERATE(3,5) : 2;
+   auto dim = all ? GENERATE(1,2,3) : 2;
+   CAPTURE(p, nx, dim);
+
+   auto MakeCartesian = [](int dim, int nx)
+   {
+      return dim == 2 ? Mesh::MakeCartesian2D(nx, nx, Element::QUADRILATERAL):
+             dim == 3 ? Mesh::MakeCartesian3D(nx, nx, nx, Element::HEXAHEDRON):
+             Mesh::MakeCartesian1D(nx);
+   };
+   Mesh mesh = MakeCartesian(dim, nx);
+
+   H1_FECollection fec(p, dim);
+   FiniteElementSpace fes(&mesh, &fec);
+   const int ND = fes.GetNDofs();
+
+   SECTION("Types")
+   {
+      MDGridFunction<4> mdgf(NE, NG, &fes, NA);
+      REQUIRE(mdgf.Size() == (NE * NG * fes.GetVSize() * NA));
+      REQUIRE(std::is_same<decltype(mdgf.HostRead()), double const*>());
+      REQUIRE(std::is_same<decltype(mdgf.MDHostRead()), MDTensor<4, double const> const>());
+   }
+
+   SECTION("LeftOffset")
+   {
+      MDGridFunction<4> gsa(NE, NG, &fes, NA);
+      const int gsa_0123 = gsa.Offset(0, 1, 2, 3);
+      REQUIRE(gsa_0123 == 0 + 1*(NE) + 2*(NE*NG) + 3*(NE*NG*ND));
+   }
+
+   SECTION("RightOffset")
+   {
+      MDGridFunction<4, MDLayoutRight<4>> gsa(NE, NG, &fes, NA);
+      const int gsa_0123 = gsa.Offset(0,1,2,3);
+      REQUIRE(gsa_0123 == 0*(NG*ND*NA) + 1*(ND*NA) + 2*(NA) + 3);
+   }
+
+   SECTION("Set/Get ScalarGridFunction")
+   {
+      MDGridFunction<3> egda(NG, &fes, NA);
+
+      GridFunction gf, rho(&fes);
+
+      BilinearForm M_ho(&fes);
+      M_ho.AddDomainIntegrator(new MassIntegrator);
+      M_ho.Assemble();
+      M_ho.Finalize();
+
+      auto compute_mass = [](GridFunction &gf)
+      {
+         FiniteElementSpace *fes = gf.FESpace();
+         ConstantCoefficient one(1.0);
+         BilinearForm ML2(fes);
+         ML2.AddDomainIntegrator(new MassIntegrator(one));
+         ML2.Assemble();
+         GridFunction ones(fes);
+         ones = 1.0;
+         return ML2.InnerProduct(gf, ones);
+      };
+
+      FunctionCoefficient rho_cft([](const Vector &x)
+      {
+         return x(1) + 0.25*cos(2*M_PI*x.Norml2());
+      });
+      rho.ProjectCoefficient(rho_cft);
+      const double rho_mass = compute_mass(rho);
+
+      const std::list<MDLayout<3>> layouts =
+      { {0,1,2}, {0,2,1}, {1,0,2}, {1,2,0}, {2,1,0}, {2,0,1} };
+
+      for (auto &layout: layouts)
+      {
+         egda = M_PI;
+         egda.SetLayout(layout);
+         for (int na = 0; na < NA; na++)
+         {
+            for (int ng = 0; ng < NG; ng++)
+            {
+               egda.GetScalarGridFunction(ng, gf, na);
+               REQUIRE(gf.Size() == fes.GetVSize());
+               REQUIRE(gf[0] == M_PI);
+               gf = rho;
+               egda.SetScalarGridFunction(ng, gf, na);
+               gf = 0.0;
+               egda.GetScalarGridFunction(ng, gf, na);
+               REQUIRE(is_equal((Vector&)gf, (Vector&)rho));
+               REQUIRE(compute_mass(gf) == MFEM_Approx(rho_mass));
+            }
+         }
+      }
+   }
+}
+
+TEST_CASE("MDGridFunction reshapes", "[MDSpan][MDReshapes]")
+{
+   SECTION("MDReshapes")
+   {
+      constexpr int p = 2;
+      constexpr int dim = 3;
+      constexpr int nx = 5, ny = 3, nz = 2;
+      Mesh mesh = Mesh::MakeCartesian3D(nx, ny, nz, Element::HEXAHEDRON);
+
+      H1_FECollection fec_mesh(p, dim);
+      FiniteElementSpace fes_mesh(&mesh, &fec_mesh, dim);
+      mesh.SetNodalFESpace(&fes_mesh);
+
+      L2_FECollection fec(p, dim);
+      FiniteElementSpace fes(&mesh, &fec);
+
+      const std::list<MDLayout<3>> layouts =
+      { {0,1,2}, {0,2,1}, {1,0,2}, {1,2,0}, {2,1,0}, {2,0,1} };
+
+      for (auto &layout: layouts)
+      {
+         constexpr int numGroups = 4, numAngles = 7;
+
+         MDGridFunction<3> psi(&fes, numGroups, numAngles);
+         psi.SetLayout(layout);
+
+         const GridFunction *nodes = mesh.GetNodes();
+         const FiniteElementSpace *mfes = mesh.GetNodalFESpace();
+         const int ng = numGroups, na = numAngles, ne = mfes->GetNE();
+         const ElementDofOrdering e_ordering = ElementDofOrdering::LEXICOGRAPHIC;
+         const Operator *R = mfes->GetElementRestriction(e_ordering);
+         REQUIRE(R);
+         const FiniteElement *mfe = mfes->GetFE(0);
+         const int nd = mfe->GetDof(), vdim = mfes->GetVDim();
+         Vector nodes_e(vdim*nd*ne); nodes_e.UseDevice(true);
+         constexpr int D1D = p + 1;
+         REQUIRE(fes.GetVSize() == D1D*D1D*D1D*ne);
+         nodes_e.Read();
+         REQUIRE(nodes);
+         R->Mult(*nodes, nodes_e);
+         const auto X = Reshape(nodes_e.Read(), D1D, D1D, D1D, vdim, ne);
+         auto dY = psi.MDWrite();
+
+         MDGridFunction<3> rY1(&fes, numGroups, numAngles);
+         rY1.SetLayout(MDLayout<3>(layout));
+         REQUIRE(rY1.Size() == psi.Size());
+         auto drY1 = rY1.MDWrite();
+
+         MDGridFunction<3> rY2(&fes, numGroups, numAngles);
+         rY2.SetLayout(MDLayout<3>(layout));
+         REQUIRE(ng%2 == 0);
+         auto drY2 = rY2.MDReshape<4>(rY2.Write(),
+                                      D1D*D1D*D1D*ne,
+                                      std::array<int,2> {2, ng/2},
+                                      na);
+
+         MDGridFunction<3> rY3(&fes, numGroups, numAngles);
+         rY3.SetLayout(MDLayout<3>(layout));
+         auto drY3 = rY3.MDReshape<6>(rY3.Write(),
+                                      std::array<int,4> {D1D, D1D, D1D, ne},
+                                      ng, na);
+
+         MDGridFunction<3> rY4(&fes, numGroups, numAngles);
+         rY4.SetLayout(MDLayout<3>(layout));
+         auto drY4 = rY4.MDReshape<7>(rY4.Write(),
+                                      std::array<int,4> {D1D, D1D, D1D, ne},
+                                      std::array<int,2> {1, ng},
+                                      na);
+
+         MDGridFunction<3> rY5(&fes, numGroups, numAngles);
+         rY5.SetLayout(MDLayout<3>(layout));
+         auto drY5 = rY5.MDReshape<7>(rY5.Write(),
+                                      std::array<int,4> {D1D, D1D, D1D, ne},
+                                      std::array<int,2> {2, ng/2},
+                                      na);
+
+         const double exp_m08 = exp(-0.8);
+
+         mfem::forall_3D(ne*ng*na, D1D,D1D,D1D, [=] MFEM_HOST_DEVICE(int ega)
+         {
+            const int e = ega/(ng*na), ga = ega%(ng*na), g = ga/na, a = ga%na;
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1D)
+               {
+                  MFEM_FOREACH_THREAD(dx,x,D1D)
+                  {
+                     const int xyze = dx + D1D*(dy + D1D*(dz + D1D*(e)));
+
+                     const double p0 = X(dx,dy,dz,0,e), p1 = X(dx,dy,dz,1,e);
+                     const double value = 1.0 - exp_m08*cos(M_PI*p0)*cos(M_PI*p1);
+
+                     dY(xyze,g,a) = value;
+                     drY1(xyze,g,a) = value;
+                     drY2(xyze,g%2,g/2,a) = value;
+                     drY3(dx,dy,dz,e, g, a) = value;
+                     drY4(dx,dy,dz,e, 0,g, a) = value;
+                     drY5(dx,dy,dz,e, g%2,g/2, a) = value;
+                  }
+               }
+            }
+         });
+         psi.MDHostRead(); rY1.HostRead();
+         REQUIRE(is_equal((Vector&)rY1, (Vector&)psi));
+         REQUIRE(is_equal((Vector&)rY2, (Vector&)psi));
+         REQUIRE(is_equal((Vector&)rY3, (Vector&)psi));
+         REQUIRE(is_equal((Vector&)rY4, (Vector&)psi));
+         REQUIRE(is_equal((Vector&)rY5, (Vector&)psi));
+      }
+   }
+}
+
+static bool is_equal(const Vector &a, const Vector &b)
+{
+   REQUIRE(a.Size() == b.Size());
+   for (int i = 0; i < a.Size(); i++)
+   {
+      const double va = a.GetData()[i], vb = b.GetData()[i];
+      REQUIRE(va == MFEM_Approx(vb));
+   };
+   return true;
+};
+

tests/unit/mesh/mesh_test_utils.cpp

@@ -11,9 +11,29 @@
 
 #include "mesh_test_utils.hpp"
 
+#include <numeric>
+
 namespace mfem
 {
 
+
+FiniteElementCollection *create_fec(FECType fectype, int p, int dim)
+{
+   switch (fectype)
+   {
+      case FECType::H1:
+         return new H1_FECollection(p, dim);
+      case FECType::ND:
+         return new ND_FECollection(p, dim);
+      case FECType::RT:
+         return new RT_FECollection(p - 1, dim);
+      case FECType::L2:
+         return new L2_FECollection(p, dim, BasisType::GaussLobatto);
+   }
+
+   return nullptr;
+}
+
 int CheckPoisson(Mesh &mesh, int order, int disabled_boundary_attribute)
 {
    constexpr int dim = 3;
@@ -204,7 +224,6 @@ Mesh DividingPlaneMesh(bool tet_mesh, bool split, bool three_dim)
    return mesh;
 }
 
-
 Mesh OrientedTriFaceMesh(int orientation, bool add_extbdr)
 {
    REQUIRE((orientation == 1 || orientation == 3 || orientation == 5));
@@ -407,6 +426,69 @@ Mesh CylinderMesh(Geometry::Type el_type, bool quadratic, int variant)
    return mesh;
 }
 
+void RefineSingleAttachedElement(Mesh &mesh, int vattr, int battr,
+                                 bool backwards)
+{
+   Array<Refinement> refs(1);
+   std::vector<int> ind(mesh.GetNBE());
+   if (backwards)
+   {
+      std::iota(ind.rbegin(), ind.rend(), 0);
+   }
+   else
+   {
+      std::iota(ind.begin(), ind.end(), 0);
+   }
+   for (int e : ind)
+   {
+      if (mesh.GetBdrAttribute(e) == battr)
+      {
+         int f, o, el1, el2;
+         mesh.GetBdrElementFace(e, &f, &o);
+         mesh.GetFaceElements(f, &el1, &el2);
+         if (mesh.GetAttribute(el1) == vattr)
+         { mesh.GeneralRefinement(Array<int> {el1}); return; }
+         if (mesh.GetAttribute(el2) == vattr)
+         { mesh.GeneralRefinement(Array<int> {el2}); return; }
+      }
+   }
+}
+
+void RefineSingleUnattachedElement(Mesh &mesh, int vattr, int battr,
+                                   bool backwards)
+{
+   std::set<int> attached_elements;
+   for (int e = 0; e < mesh.GetNBE(); e++)
+   {
+      if (mesh.GetBdrAttribute(e) == battr)
+      {
+         int f, o, el1, el2;
+         mesh.GetBdrElementFace(e, &f, &o);
+         mesh.GetFaceElements(f, &el1, &el2);
+         if (mesh.GetAttribute(el1) == vattr) { attached_elements.insert(el1); }
+         if (el2 >= 0 && mesh.GetAttribute(el2) == vattr) { attached_elements.insert(el2); }
+      }
+   }
+   if (backwards)
+   {
+      for (int i = mesh.GetNE() - 1; i >= 0; i--)
+         if (mesh.GetAttribute(i) == vattr && attached_elements.count(i) == 0)
+         {
+            mesh.GeneralRefinement(Array<int> {i});
+            return;
+         }
+   }
+   else
+   {
+      for (int i = 0; i < mesh.GetNE(); i++)
+         if (mesh.GetAttribute(i) == vattr && attached_elements.count(i) == 0)
+         {
+            mesh.GeneralRefinement(Array<int> {i});
+            return;
+         }
+   }
+}
+
 #ifdef MFEM_USE_MPI
 
 void TestVectorValueInVolume(Mesh &smesh, int nc_level, int skip, bool use_ND)

tests/unit/mesh/mesh_test_utils.hpp

@@ -22,6 +22,28 @@
 namespace mfem
 {
 
+/**
+ * @brief Which type of FiniteElementCollection to use
+ */
+enum class FECType
+{
+   H1,
+   ND,
+   RT,
+   L2
+};
+
+/**
+ * @brief Create a FiniteElementCollection
+ *
+ * @param fectype the type of FEC to create
+ * @param p The polynomial order
+ * @param dim The dimension
+ * @return FiniteElementCollection*
+ */
+FiniteElementCollection *create_fec(FECType fectype, int p, int dim);
+
+
 /**
  * @brief Helper function for performing an H1 Poisson solve on a serial mesh,
  * with homogeneous essential boundary conditions. Optionally can disable a
@@ -98,9 +120,33 @@ Mesh OrientedTriFaceMesh(int orientation, bool add_extbdr = false);
  */
 Mesh CylinderMesh(Geometry::Type el_type, bool quadratic, int variant = 0);
 
-#ifdef MFEM_USE_MPI
 
 
+/**
+ * @brief Helper to refine a single element attached to a boundary attribute
+ *
+ * @param mesh Mesh to refine
+ * @param vattr Volume attribute to check for elements
+ * @param battr Boundary attribute refined element should be attached to
+ * @param backwards Whether to iterate over the faces in reverse order
+ */
+void RefineSingleAttachedElement(Mesh &mesh, int vattr, int battr,
+                                 bool backwards = true);
+
+/**
+ * @brief Helper to refine a single element not attached to a boundary
+ *
+ * @param mesh Mesh to refine
+ * @param vattr Volume attribute to check for elements
+ * @param battr Boundary attribute refined element should not be attached to
+ * @param backwards Whether to iterate over the elements in reverse order
+ */
+void RefineSingleUnattachedElement(Mesh &mesh, int vattr, int battr,
+                                   bool backwards = true);
+
+
+#ifdef MFEM_USE_MPI
+
 /**
  * @brief Test GetVectorValue on face neighbor elements for nonconforming meshes
  *

tests/unit/mesh/test_ncmesh.cpp

@@ -2811,7 +2811,6 @@ TEST_CASE("RP=I", "[NCMesh]")
    }
 }
 
-
 TEST_CASE("InternalBoundaryProjectBdrCoefficient", "[NCMesh]")
 {
    auto test_project_H1 = [](Mesh &mesh, int order, double coef)
@@ -2887,6 +2886,4 @@ TEST_CASE("InternalBoundaryProjectBdrCoefficient", "[NCMesh]")
    }
 }
 
-
-
 } // namespace mfem

tests/unit/mesh/test_pmesh.cpp

@@ -54,6 +54,7 @@ TEST_CASE("ParMeshGlobalIndices",  "[Parallel], [ParMesh]")
             }
          }
 
+
          ParMesh pmesh(MPI_COMM_WORLD, mesh);
 
          int globalN = 0;
@@ -102,8 +103,10 @@ TEST_CASE("ParMeshGlobalIndices",  "[Parallel], [ParMesh]")
 
             // Verify that the global indices range from 0 to globalN-1.
             {
-               const HYPRE_BigInt localMin = gi.Min();
-               const HYPRE_BigInt localMax = gi.Max();
+               const HYPRE_BigInt localMin = gi.Size() > 0 ? gi.Min() :
+                                             std::numeric_limits<HYPRE_BigInt>::max();
+               const HYPRE_BigInt localMax = gi.Size() > 0 ? gi.Max() :
+                                             std::numeric_limits<HYPRE_BigInt>::min();
 
                HYPRE_BigInt globalMin, globalMax;
                MPI_Allreduce(&localMin, &globalMin, 1, HYPRE_MPI_BIG_INT, MPI_MIN,

tests/unit/mesh/test_psubmesh.cpp

@@ -11,6 +11,7 @@
 
 #include "mfem.hpp"
 #include "unit_tests.hpp"
+#include "mesh_test_utils.hpp"
 
 using namespace mfem;
 
@@ -18,57 +19,40 @@ using namespace mfem;
 
 namespace ParSubMeshTests
 {
-enum FECType
+
+void CHECK_GLOBAL_NORM(Vector &v, bool small = true)
 {
-   H1,
-   ND,
-   RT,
-   L2
+   real_t norm_local = v.Norml2(), norm_global = 0.0;
+   MPI_Allreduce(&norm_local, &norm_global, 1, MPITypeMap<real_t>::mpi_type,
+                 MPI_SUM, MPI_COMM_WORLD);
+   if (small)
+   {
+      REQUIRE(norm_global < 1e-8);
+   }
+   else
+   {
+      REQUIRE(norm_global > 1e-8);
+   }
 };
 
-FiniteElementCollection *create_fec(FECType fectype, int p, int dim)
-{
-   switch (fectype)
-   {
-      case H1:
-         return new H1_FECollection(p, dim);
-         break;
-      case ND:
-         return new ND_FECollection(p, dim);
-         break;
-      case RT:
-         return new RT_FECollection(p - 1, dim);
-         break;
-      case L2:
-         return new L2_FECollection(p, dim, BasisType::GaussLobatto);
-         break;
-   }
-
-   return nullptr;
-}
 
 FiniteElementCollection *create_surf_fec(FECType fectype, int p, int dim)
 {
    switch (fectype)
    {
-      case H1:
+      case FECType::H1:
          return new H1_FECollection(p, dim);
-         break;
-      case ND:
+      case FECType::ND:
          return new ND_FECollection(p, dim);
-         break;
-      case RT:
+      case FECType::RT:
          return new L2_FECollection(p - 1, dim, BasisType::GaussLegendre,
                                     FiniteElement::INTEGRAL);
-         break;
-      case L2:
+      case FECType::L2:
          return new L2_FECollection(p, dim, BasisType::GaussLobatto);
-         break;
    }
 
    return nullptr;
 }
-
 class SurfaceNormalCoef : public VectorCoefficient
 {
 public:
@@ -207,14 +191,14 @@ void multidomain_test_2d(FECType fec_type)
    SurfaceNormalCoef normalcoeff(dim);
    InnerProductCoefficient nvcoeff(normalcoeff, vcoeff);
 
-   if (fec_type == H1 || fec_type == L2)
+   if (fec_type == FECType::H1 || fec_type == FECType::L2)
    {
       parent_gf.ProjectCoefficient(coeff);
       parent_gf_ex.ProjectCoefficient(coeff);
       domain1_gf_ex.ProjectCoefficient(coeff);
       boundary1_gf_ex.ProjectCoefficient(coeff);
    }
-   else if (fec_type == ND)
+   else if (fec_type == FECType::ND)
    {
       parent_gf.ProjectCoefficient(vcoeff);
       parent_gf_ex.ProjectCoefficient(vcoeff);
@@ -231,13 +215,6 @@ void multidomain_test_2d(FECType fec_type)
 
    Vector tmp;
 
-   auto CHECK_GLOBAL_NORM = [](Vector &v)
-   {
-      real_t norm_local = v.Norml2(), norm_global = 0.0;
-      MPI_Allreduce(&norm_local, &norm_global, 1, MPITypeMap<real_t>::mpi_type,
-                    MPI_SUM, MPI_COMM_WORLD);
-      REQUIRE(norm_global < 1e-8);
-   };
 
    SECTION("ParentToSubMesh")
    {
@@ -260,7 +237,7 @@ void multidomain_test_2d(FECType fec_type)
    {
       SECTION("Volume to matching volume")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             parent_gf.ProjectCoefficient(coeff);
             domain1_gf.ProjectCoefficient(coeff);
@@ -277,11 +254,11 @@ void multidomain_test_2d(FECType fec_type)
       }
       SECTION("Surface to matching surface in volume")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             boundary1_gf.ProjectCoefficient(coeff);
          }
-         else if (fec_type == ND)
+         else if (fec_type == FECType::ND)
          {
             boundary1_gf.ProjectCoefficient(vcoeff);
          }
@@ -341,7 +318,8 @@ void multidomain_test_3d(FECType fec_type)
    MPI_Allreduce(&num_local_be, &num_global_be, 1, MPI_INT, MPI_SUM,
                  MPI_COMM_WORLD);
    REQUIRE(num_global_be == 16);
-   REQUIRE(cylinder_surface_submesh.bdr_attributes[0] == 900);
+   REQUIRE(cylinder_surface_submesh.bdr_attributes[0] ==
+           parent_mesh.bdr_attributes.Max() + 1);
 
    FiniteElementCollection *fec = create_fec(fec_type, p,
                                              parent_mesh.Dimension());
@@ -396,7 +374,7 @@ void multidomain_test_3d(FECType fec_type)
    SurfaceNormalCoef normalcoeff(dim);
    InnerProductCoefficient nvcoeff(normalcoeff, vcoeff);
 
-   if (fec_type == H1 || fec_type == L2)
+   if (fec_type == FECType::H1 || fec_type == FECType::L2)
    {
       parent_gf.ProjectCoefficient(coeff);
       parent_gf_ex.ProjectCoefficient(coeff);
@@ -404,7 +382,7 @@ void multidomain_test_3d(FECType fec_type)
       cylinder_surface_gf_ex.ProjectCoefficient(coeff);
       outer_gf_ex.ProjectCoefficient(coeff);
    }
-   else if (fec_type == ND)
+   else if (fec_type == FECType::ND)
    {
       parent_gf.ProjectCoefficient(vcoeff);
       parent_gf_ex.ProjectCoefficient(vcoeff);
@@ -423,14 +401,6 @@ void multidomain_test_3d(FECType fec_type)
 
    Vector tmp;
 
-   auto CHECK_GLOBAL_NORM = [](Vector &v)
-   {
-      real_t norm_local = v.Norml2(), norm_global = 0.0;
-      MPI_Allreduce(&norm_local, &norm_global, 1, MPITypeMap<real_t>::mpi_type,
-                    MPI_SUM, MPI_COMM_WORLD);
-      REQUIRE(norm_global < 1e-8);
-   };
-
    SECTION("ParentToSubMesh")
    {
       SECTION("Volume to matching volume")
@@ -452,7 +422,7 @@ void multidomain_test_3d(FECType fec_type)
    {
       SECTION("Volume to matching volume")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             parent_gf.ProjectCoefficient(coeff);
             cylinder_gf.ProjectCoefficient(coeff);
@@ -469,7 +439,7 @@ void multidomain_test_3d(FECType fec_type)
       }
       SECTION("Volume to matching volume")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             outer_gf.ProjectCoefficient(coeff);
          }
@@ -484,11 +454,11 @@ void multidomain_test_3d(FECType fec_type)
       }
       SECTION("Surface to matching surface in volume")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             cylinder_surface_gf.ProjectCoefficient(coeff);
          }
-         else if (fec_type == ND)
+         else if (fec_type == FECType::ND)
          {
             cylinder_surface_gf.ProjectCoefficient(vcoeff);
          }
@@ -506,7 +476,7 @@ void multidomain_test_3d(FECType fec_type)
    {
       SECTION("Volume to matching volume")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             cylinder_gf.ProjectCoefficient(coeff);
             outer_gf.ProjectCoefficient(coeff);
@@ -527,7 +497,7 @@ void multidomain_test_3d(FECType fec_type)
       }
       SECTION("Volume to matching volume (reversed)")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             cylinder_gf.ProjectCoefficient(coeff);
             outer_gf.ProjectCoefficient(coeff);
@@ -548,7 +518,7 @@ void multidomain_test_3d(FECType fec_type)
       }
       SECTION("Volume to matching surface on volume")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             cylinder_gf.ProjectCoefficient(coeff);
             outer_gf.ProjectCoefficient(coeff);
@@ -568,12 +538,12 @@ void multidomain_test_3d(FECType fec_type)
 
       SECTION("Volume to matching surface")
       {
-         if (fec_type == H1 || fec_type == L2)
+         if (fec_type == FECType::H1 || fec_type == FECType::L2)
          {
             cylinder_gf.ProjectCoefficient(coeff);
             cylinder_surface_gf_ex.ProjectCoefficient(coeff);
          }
-         else if (fec_type == ND)
+         else if (fec_type == FECType::ND)
          {
             cylinder_gf.ProjectCoefficient(vcoeff);
             cylinder_surface_gf_ex.ProjectCoefficient(vcoeff);
@@ -593,7 +563,7 @@ void multidomain_test_3d(FECType fec_type)
    delete fec;
 }
 
-TEST_CASE("ParSubMesh", "[Parallel],[ParSubMesh]")
+TEST_CASE("ParSubMesh", "[Parallel],[SubMesh]")
 {
    auto fec_type = GENERATE(FECType::H1, FECType::ND, FECType::RT, FECType::L2);
    multidomain_test_2d(fec_type);
@@ -621,10 +591,11 @@ Array<int> count_be(ParMesh &mesh)
    return glb_counts;
 }
 
-TEST_CASE("ParSubMesh Interior Boundaries", "[Parallel],[ParSubMesh]")
+TEST_CASE("ParSubMesh Interior Boundaries", "[Parallel],[SubMesh]")
 {
+   // whether to NC refine the attribute 1 elements
+   auto make_nc = GENERATE(false, true);
    int num_procs = Mpi::WorldSize();
-
    Mesh serial_mesh = Mesh::MakeCartesian3D(num_procs, num_procs, 1,
                                             Element::HEXAHEDRON,
                                             1.0, 1.0, 0.1, false);
@@ -636,7 +607,6 @@ TEST_CASE("ParSubMesh Interior Boundaries", "[Parallel],[ParSubMesh]")
       int attr = (i + (1 + num_procs % 2) * (i / num_procs)) % 2 + 1;
       serial_mesh.SetAttribute(i, attr);
    }
-
    int bdr_max = serial_mesh.bdr_attributes.Max();
 
    // Label all interior faces as boundary elements
@@ -656,55 +626,503 @@ TEST_CASE("ParSubMesh Interior Boundaries", "[Parallel],[ParSubMesh]")
    Array<int> partitioning(num_procs * num_procs);
    for (int i = 0; i < num_procs * num_procs; i++)
    {
-      // The following creates a shifting pattern where neighboring elements
-      // are never owned by the same processor
+      // The following creates a shifting pattern where neighboring elements are
+      // never owned by the same processor
       partitioning[i] = (2 * num_procs - 1 - (i % num_procs) -
                          i / num_procs) % num_procs;
    }
-
+   if (make_nc)
+   {
+      serial_mesh.EnsureNCMesh(true);
+   }
    ParMesh parent_mesh(MPI_COMM_WORLD, serial_mesh, partitioning);
 
+   if (make_nc)
+   {
+      // Refine after partitioning so that the checkerboard pattern persists.
+      Array<int> el_to_refine;
+      for (int i = 0; i < parent_mesh.GetNE(); i++)
+      {
+         if (parent_mesh.GetAttribute(i) == 1)
+         {
+            el_to_refine.Append(i);
+         }
+      }
+      parent_mesh.GeneralRefinement(el_to_refine);
+   }
+
    // Create a pair of domain-based sub meshes
    Array<int> domain1(1);
    domain1[0] = 1;
-
    Array<int> domain2(1);
    domain2[0] = 2;
 
    auto domain1_submesh = ParSubMesh::CreateFromDomain(parent_mesh,
                                                        domain1);
-
    auto domain2_submesh = ParSubMesh::CreateFromDomain(parent_mesh,
                                                        domain2);
 
    // Create histograms of boundary attributes in each sub-domain
    auto be1 = count_be(domain1_submesh);
    auto be2 = count_be(domain2_submesh);
+   REQUIRE(((be1.Size() >= 7) && (be2.Size() >= 7)));
 
    // Only the root process has valid histograms
    if (Mpi::Root())
    {
-      // Verify that all exterior boundary elements were accounted for
-      REQUIRE(be1[1] + be2[1] == num_procs * num_procs);
-      REQUIRE(be1[2] + be2[2] == num_procs);
-      REQUIRE(be1[3] + be2[3] == num_procs);
-      REQUIRE(be1[4] + be2[4] == num_procs);
-      REQUIRE(be1[5] + be2[5] == num_procs);
-      REQUIRE(be1[6] + be2[6] == num_procs * num_procs);
+      // Verify that all exterior boundary elements were accounted for. If an NC
+      // refine has occurred, there will be extra faces on half the checkerboard
+      const int num_top_refined = make_nc ? (num_procs/2)*(num_procs/2)
+                                  + ((num_procs+1)/2)*((num_procs+1)/2) : 0;
+      const int num_side_refined = make_nc ? (num_procs+1)/2 : 0;
+      CHECK(be1[1] + be2[1] == num_procs * num_procs + 3 * num_top_refined);
+      CHECK(be1[2] + be2[2] == num_procs + 3 * num_side_refined);
+      CHECK(be1[3] + be2[3] == num_procs + 3 * num_side_refined);
+      CHECK(be1[4] + be2[4] == num_procs + 3 * num_side_refined);
+      CHECK(be1[5] + be2[5] == num_procs + 3 * num_side_refined);
+      CHECK(be1[6] + be2[6] == num_procs * num_procs + 3 * num_top_refined);
 
-      // Verify that all interior boundary elements appear once in each submesh
+      // Verify that all interior boundary elements of serial mesh appear
+      // correct number of times in each submesh
       for (int i=0; i < serial_mesh.GetNumFaces(); i++)
       {
          if (serial_mesh.FaceIsInterior(i))
          {
             const int attr = bdr_max + i + 1;
-            REQUIRE(be1[attr] == 1);
-            REQUIRE(be2[attr] == 1);
+            REQUIRE(attr < be1.Size());
+            REQUIRE(attr < be2.Size());
+            CAPTURE(make_nc, i, attr, bdr_max, be1[attr], be2[attr]);
+            CHECK(be1[attr] == (make_nc ? 4 : 1));
+            CHECK(be2[attr] == 1);
+         }
+      }
+   }
+}
+/**
+ * @brief Helper class for testing a ParNCMesh
+ *
+ */
+struct ParNCMeshExposed : public ParNCMesh
+{
+   ParNCMeshExposed(const ParNCMesh &ncmesh) : ParNCMesh(ncmesh) {}
+   using ParNCMesh::elements;
+   using ParNCMesh::leaf_elements;
+   int CountUniqueLeafElements() const
+   {
+      int local = 0;
+      for (auto i : leaf_elements)
+      {
+         if (elements[i].rank == MyRank)
+         {
+            local++;
+         }
+      }
+      int global = 0;
+      MPI_Allreduce(&local, &global, 1, MPI_INT, MPI_SUM, GetGlobalMPI_Comm());
+      return global;
+   }
+};
+
+void CheckProjectMatch(ParMesh &mesh, ParSubMesh &submesh, FECType fec_type,
+                       bool check_pr = true)
+{
+   int p = 3;
+   CAPTURE(fec_type);
+   auto fec = std::unique_ptr<FiniteElementCollection>(create_fec(fec_type, p,
+                                                                  mesh.Dimension()));
+   auto sub_fec = std::unique_ptr<FiniteElementCollection>(create_fec(fec_type, p,
+                                                                      submesh.Dimension()));
+
+   ParFiniteElementSpace fes(&mesh, fec.get());
+   ParFiniteElementSpace sub_fes(&submesh, sub_fec.get());
+   ParGridFunction gf(&fes), gf_ext(&fes);
+   ParGridFunction sub_gf(&sub_fes), sub_gf_ext(&sub_fes);
+   auto coeff = FunctionCoefficient([](const Vector &coords)
+   {
+      real_t x = coords(0);
+      real_t y = coords(1);
+      real_t z = coords(2);
+      return 0.02 * sin(y * 5.0 * M_PI)
+             + 0.03 * sin(x * 5.0 * M_PI)
+             + 0.05 * sin(z * 5.0 * M_PI);
+   });
+
+   auto vcoeff = VectorFunctionCoefficient(mesh.SpaceDimension(),
+                                           [](const Vector &coords, Vector &V)
+   {
+      V.SetSize(3);
+      real_t x = coords(0);
+      real_t y = coords(1);
+      real_t z = coords(2);
+
+      V(0) = 0.02 * sin(y * 3.0 * M_PI)
+             + 0.03 * sin(x * 2.0 * M_PI)
+             + 0.05 * sin(z * 4.0 * M_PI);
+      V(1) = 0.02 * sin(z * 3.0 * M_PI)
+             + 0.03 * sin(y * 2.0 * M_PI)
+             + 0.05 * sin(x * 4.0 * M_PI);
+      V(2) = 0.02 * sin(x * 3.0 * M_PI)
+             + 0.03 * sin(y * 2.0 * M_PI)
+             + 0.05 * sin(z * 4.0 * M_PI);
+   });
+
+   if (fec_type == FECType::H1 || fec_type == FECType::L2)
+   {
+      gf.ProjectCoefficient(coeff);
+      sub_gf.ProjectCoefficient(coeff);
+   }
+   else
+   {
+      gf.ProjectCoefficient(vcoeff);
+      sub_gf.ProjectCoefficient(vcoeff);
+   }
+   gf_ext = gf;
+   sub_gf_ext = sub_gf;
+
+   SECTION("ParentToSubMesh")
+   {
+      // Direct transfer should be identical
+      ParSubMesh::Transfer(gf, sub_gf);
+      auto tmp = sub_gf_ext;
+      tmp -= sub_gf;
+      CHECK_GLOBAL_NORM(tmp);
+   }
+   SECTION("PRConstraint")
+   {
+      // Application of PR should be identical in mesh and submesh for an
+      // external boundary.
+      if (mesh.Nonconforming())
+      {
+         Vector tmp;
+         if (const auto *P = fes.GetProlongationMatrix())
+         {
+            const auto *R = fes.GetRestrictionMatrix();
+            tmp.SetSize(R->Height());
+            R->Mult(gf, tmp);
+            P->Mult(tmp, gf);
+         }
+         if (const auto *P = sub_fes.GetProlongationMatrix())
+         {
+            const auto *R = sub_fes.GetRestrictionMatrix();
+            tmp.SetSize(R->Height());
+            R->Mult(sub_gf_ext, tmp);
+            P->Mult(tmp, sub_gf_ext);
+         }
+         ParSubMesh::Transfer(gf, sub_gf);
+         tmp = sub_gf_ext;
+         tmp -= sub_gf;
+         CHECK_GLOBAL_NORM(tmp, check_pr);
+      }
+   }
+}
+
+TEST_CASE("VolumeParNCSubMesh", "[Parallel],[SubMesh]")
+{
+   bool use_tet = GENERATE(false,true);
+
+   auto mesh = use_tet ? OrientedTriFaceMesh(1, true) : DividingPlaneMesh(false,
+                                                                          true);
+   mesh.EnsureNCMesh(true);
+   SECTION("UniformRefinement2")
+   {
+      mesh.UniformRefinement();
+      mesh.UniformRefinement();
+      ParMesh pmesh(MPI_COMM_WORLD, mesh);
+      SECTION("SingleAttribute")
+      {
+         Array<int> subdomain_attributes(1);
+         subdomain_attributes[0] = GENERATE(range(1,2));
+         auto submesh = ParSubMesh::CreateFromDomain(pmesh, subdomain_attributes);
+
+         // Cast to an exposed variant to explore the internals.
+         auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+         CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+         CHECK(pncmesh_exposed.CountUniqueLeafElements() == 8*8);
+         for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+         {
+            CheckProjectMatch(pmesh, submesh, fec_type);
+         }
+      }
+
+      SECTION("UniformRefineTwoAttribute")
+      {
+         Array<int> subdomain_attributes(2);
+         subdomain_attributes[0] = 1;
+         subdomain_attributes[1] = 2;
+         auto submesh = ParSubMesh::CreateFromDomain(pmesh, subdomain_attributes);
+
+         // Cast to an exposed variant to explore the internals.
+         auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+         CHECK(pncmesh_exposed.GetNumRootElements() ==
+               pmesh.ncmesh->GetNumRootElements());
+         CHECK(pncmesh_exposed.CountUniqueLeafElements() == 2*8*8);
+         for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+         {
+            CheckProjectMatch(pmesh, submesh, fec_type);
+         }
+      }
+   }
+
+   SECTION("Nonconformal")
+   {
+      mesh.UniformRefinement();
+      Array<int> subdomain_attributes{GENERATE(1,2)};
+      auto backwards = GENERATE(false, true);
+      SECTION("ConsistentWithParent")
+      {
+         RefineSingleUnattachedElement(mesh, subdomain_attributes[0],
+                                       mesh.bdr_attributes.Max(), backwards);
+         {
+            ParMesh pmesh(MPI_COMM_WORLD, mesh);
+            auto submesh = ParSubMesh::CreateFromDomain(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 8 - 1 + 8);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type, true);
+            }
+         }
+         RefineSingleUnattachedElement(mesh, subdomain_attributes[0],
+                                       mesh.bdr_attributes.Max(), backwards);
+         {
+            ParMesh pmesh(MPI_COMM_WORLD, mesh);
+            auto submesh = ParSubMesh::CreateFromDomain(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 8 - 1 + 8 - 1 + 8);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type, true);
+            }
+         }
+      }
+
+      SECTION("InconsistentWithParent")
+      {
+         RefineSingleAttachedElement(mesh, subdomain_attributes[0],
+                                     mesh.bdr_attributes.Max(), backwards);
+         {
+            ParMesh pmesh(MPI_COMM_WORLD, mesh);
+            auto submesh = ParSubMesh::CreateFromDomain(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 8 - 1 + 8);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type, false);
+            }
+         }
+         RefineSingleAttachedElement(mesh, subdomain_attributes[0],
+                                     mesh.bdr_attributes.Max(), backwards);
+         {
+            ParMesh pmesh(MPI_COMM_WORLD, mesh);
+            auto submesh = ParSubMesh::CreateFromDomain(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 8 - 1 + 8 - 1 + 8);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type, false);
+            }
          }
       }
    }
 }
 
+TEST_CASE("ExteriorSurfaceParNCSubMesh", "[Parallel],[SubMesh]")
+{
+   SECTION("Hex")
+   {
+      auto mesh = Mesh("../../data/ref-cube.mesh", 1, 1);
+      mesh.EnsureNCMesh(true);
+      SECTION("UniformRefinement2")
+      {
+         mesh.UniformRefinement();
+         mesh.UniformRefinement();
+         ParMesh pmesh(MPI_COMM_WORLD, mesh);
+         SECTION("SingleAttribute")
+         {
+            Array<int> subdomain_attributes(1);
+            subdomain_attributes[0] = GENERATE(range(1,6));
+            auto submesh = ParSubMesh::CreateFromBoundary(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 4*4);
+            CHECK(submesh.bdr_attributes.Size() == 1);
+            CHECK(submesh.bdr_attributes[0] == mesh.bdr_attributes.Max() + 1);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type);
+            }
+         }
+
+         SECTION("UniformRefineTwoAttribute")
+         {
+            Array<int> subdomain_attributes(2);
+            subdomain_attributes[0] = GENERATE(range(1,6));
+            subdomain_attributes[1] = 1 + (subdomain_attributes[0] % 6);
+            auto submesh = ParSubMesh::CreateFromBoundary(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 2);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 2*4*4);
+            CHECK(submesh.bdr_attributes.Size() == 1);
+            CHECK(submesh.bdr_attributes[0] == mesh.bdr_attributes.Max() + 1);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type);
+            }
+         }
+      }
+
+      SECTION("NonconformalRefine")
+      {
+         Array<int> subdomain_attributes(1);
+         subdomain_attributes[0] = GENERATE(range(1,6));
+         mesh.UniformRefinement();
+         RefineSingleAttachedElement(mesh, 1, subdomain_attributes[0], true);
+         SECTION("Single")
+         {
+            ParMesh pmesh(MPI_COMM_WORLD, mesh);
+            auto submesh = ParSubMesh::CreateFromBoundary(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 4 - 1 + 4);
+            CHECK(submesh.bdr_attributes.Size() == 1);
+            CHECK(submesh.bdr_attributes[0] == mesh.bdr_attributes.Max() + 1);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type);
+            }
+
+         }
+         SECTION("Double")
+         {
+            RefineSingleAttachedElement(mesh, 1, subdomain_attributes[0], false);
+            ParMesh pmesh(MPI_COMM_WORLD, mesh);
+            auto submesh = ParSubMesh::CreateFromBoundary(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 4 - 1 + 4 - 1 + 4);
+            CHECK(submesh.bdr_attributes.Size() == 1);
+            CHECK(submesh.bdr_attributes[0] == mesh.bdr_attributes.Max() + 1);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type);
+            }
+         }
+      }
+   }
+
+   SECTION("Tet")
+   {
+      auto mesh = Mesh("../../data/ref-tetrahedron.mesh");
+      mesh.EnsureNCMesh(true);
+      SECTION("UniformRefinement2")
+      {
+         mesh.UniformRefinement();
+         mesh.UniformRefinement();
+         ParMesh pmesh(MPI_COMM_WORLD, mesh);
+         SECTION("SingleAttribute")
+         {
+            Array<int> subdomain_attributes(1);
+            subdomain_attributes[0] = GENERATE(range(1,4));
+            auto submesh = ParSubMesh::CreateFromBoundary(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 4*4);
+            CHECK(submesh.bdr_attributes.Size() == 1);
+            CHECK(submesh.bdr_attributes[0] == mesh.bdr_attributes.Max() + 1);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type);
+            }
+         }
+
+         SECTION("UniformRefineTwoAttribute")
+         {
+            Array<int> subdomain_attributes(2);
+            subdomain_attributes[0] = GENERATE(range(1,4));
+            subdomain_attributes[1] = 1 + (subdomain_attributes[0] % 4);
+            auto submesh = ParSubMesh::CreateFromBoundary(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 2);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 2*4*4);
+            CHECK(submesh.bdr_attributes.Size() == 1);
+            CHECK(submesh.bdr_attributes[0] == mesh.bdr_attributes.Max() + 1);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type);
+            }
+         }
+      }
+
+      SECTION("NonconformalRefine")
+      {
+         Array<int> subdomain_attributes(1);
+         subdomain_attributes[0] = GENERATE(range(1,4));
+         mesh.UniformRefinement();
+         RefineSingleAttachedElement(mesh, 1, subdomain_attributes[0], true);
+         SECTION("Single")
+         {
+            ParMesh pmesh(MPI_COMM_WORLD, mesh);
+            auto submesh = ParSubMesh::CreateFromBoundary(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 4 - 1 + 4);
+            CHECK(submesh.bdr_attributes.Size() == 1);
+            CHECK(submesh.bdr_attributes[0] == mesh.bdr_attributes.Max() + 1);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type);
+            }
+         }
+         SECTION("Double")
+         {
+            RefineSingleAttachedElement(mesh, 1, subdomain_attributes[0], false);
+            ParMesh pmesh(MPI_COMM_WORLD, mesh);
+            auto submesh = ParSubMesh::CreateFromBoundary(pmesh, subdomain_attributes);
+
+            // Cast to an exposed variant to explore the internals.
+            auto pncmesh_exposed = ParNCMeshExposed(*submesh.pncmesh);
+            CHECK(pncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(pncmesh_exposed.CountUniqueLeafElements() == 4 - 1 + 4 - 1 + 4);
+            CHECK(submesh.bdr_attributes.Size() == 1);
+            CHECK(submesh.bdr_attributes[0] == mesh.bdr_attributes.Max() + 1);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(pmesh, submesh, fec_type);
+            }
+         }
+      }
+   }
+
+
+}
+
+
 } // namespace ParSubMeshTests
 
 #endif // MFEM_USE_MPI

tests/unit/mesh/test_submesh.cpp

@@ -17,41 +17,17 @@
 
 using namespace mfem;
 
-enum FECType
-{
-   H1,
-   ND,
-   L2
-};
-enum FieldType
+enum class FieldType
 {
    SCALAR,
    VECTOR
 };
-enum TransferType
+enum class TransferType
 {
    ParentToSub,
    SubToParent
 };
 
-FiniteElementCollection *create_fec(FECType fec_type, int p, int dim)
-{
-   switch (fec_type)
-   {
-      case H1:
-         return new H1_FECollection(p, dim);
-         break;
-      case ND:
-         return new ND_FECollection(p, dim);
-         break;
-      case L2:
-         return new L2_FECollection(p, dim, BasisType::GaussLobatto);
-         break;
-   }
-
-   return nullptr;
-}
-
 void test_2d(Element::Type element_type,
              FECType fec_type,
              FieldType field_type,
@@ -62,7 +38,7 @@ void test_2d(Element::Type element_type,
 {
    constexpr int dim = 2;
    const int vdim = (field_type == FieldType::SCALAR ||
-                     fec_type == ND) ? 1 : dim;
+                     fec_type == FECType::ND) ? 1 : dim;
    real_t Hy = 1.0;
    Mesh mesh = Mesh::MakeCartesian2D(5, 5, element_type, true, 1.0, Hy, false);
 
@@ -178,11 +154,11 @@ void test_2d(Element::Type element_type,
    GridFunction sub_gf(&sub_fes);
    sub_gf = 0.0;
 
-   if (transfer_type == ParentToSub)
+   if (transfer_type == TransferType::ParentToSub)
    {
       GridFunction sub_ex_gf(&sub_fes);
 
-      if (vdim == 1 && (fec_type == H1 || fec_type == L2))
+      if (vdim == 1 && (fec_type == FECType::H1 || fec_type == FECType::L2))
       {
          parent_gf.ProjectCoefficient(coeff);
          sub_ex_gf.ProjectCoefficient(coeff);
@@ -199,11 +175,11 @@ void test_2d(Element::Type element_type,
       sub_gf -= sub_ex_gf;
       REQUIRE(sub_gf.Norml2() < 1e-10);
    }
-   else if (transfer_type == SubToParent)
+   else if (transfer_type == TransferType::SubToParent)
    {
       GridFunction parent_ex_gf(&parent_fes);
 
-      if (vdim == 1 && (fec_type == H1 || fec_type == L2))
+      if (vdim == 1 && (fec_type == FECType::H1 || fec_type == FECType::L2))
       {
          parent_gf.ProjectCoefficient(coeff);
          sub_gf.ProjectCoefficient(coeff);
@@ -238,7 +214,7 @@ void test_3d(Element::Type element_type,
 {
    constexpr int dim = 3;
    const int vdim = (field_type == FieldType::SCALAR ||
-                     fec_type == ND) ? 1 : dim;
+                     fec_type == FECType::ND) ? 1 : dim;
    real_t Hy = 1.0;
    Mesh mesh = Mesh::MakeCartesian3D(5, 5, 5, element_type, 1.0, Hy, 1.0, false);
 
@@ -358,11 +334,11 @@ void test_3d(Element::Type element_type,
    GridFunction sub_gf(&sub_fes);
    sub_gf = 0.0;
 
-   if (transfer_type == ParentToSub)
+   if (transfer_type == TransferType::ParentToSub)
    {
       GridFunction sub_ex_gf(&sub_fes);
 
-      if (vdim == 1 && (fec_type == H1 || fec_type == L2))
+      if (vdim == 1 && (fec_type == FECType::H1 || fec_type == FECType::L2))
       {
          parent_gf.ProjectCoefficient(coeff);
          sub_ex_gf.ProjectCoefficient(coeff);
@@ -379,11 +355,11 @@ void test_3d(Element::Type element_type,
       sub_gf -= sub_ex_gf;
       REQUIRE(sub_gf.Norml2() < 1e-10);
    }
-   else if (transfer_type == SubToParent)
+   else if (transfer_type == TransferType::SubToParent)
    {
       GridFunction parent_ex_gf(&parent_fes);
 
-      if (vdim == 1 && (fec_type == H1 || fec_type == L2))
+      if (vdim == 1 && (fec_type == FECType::H1 || fec_type == FECType::L2))
       {
          parent_gf.ProjectCoefficient(coeff);
          sub_gf.ProjectCoefficient(coeff);
@@ -564,3 +540,376 @@ TEST_CASE("InterfaceTransferSolve", "[SubMesh]")
 
    CHECK((x_sub.Norml2() / x_sub.Size()) == MFEM_Approx(0.0, 1e-7, 1e-7));
 }
+
+/**
+ * @brief Helper class for testing a NCMesh
+ *
+ */
+struct NCMeshExposed : public NCMesh
+{
+   NCMeshExposed(const NCMesh &ncmesh) : NCMesh(ncmesh) {}
+   using NCMesh::elements;
+   using NCMesh::leaf_elements;
+   int CountUniqueLeafElements() const
+   {
+      int local = 0;
+      for (auto i : leaf_elements)
+      {
+         if (elements[i].rank == MyRank)
+         {
+            ++local;
+         }
+      }
+      return local;
+   }
+};
+
+void CHECK_NORM(Vector &v, bool small = true)
+{
+   if (small)
+   {
+      REQUIRE(v.Norml2() < 1e-8);
+   }
+   else
+   {
+      REQUIRE(v.Norml2() > 1e-8);
+   }
+};
+
+void CheckProjectMatch(Mesh &mesh, SubMesh &submesh, FECType fec_type,
+                       bool check_pr = true)
+{
+   int p = 3;
+   auto fec = std::unique_ptr<FiniteElementCollection>(create_fec(fec_type, p,
+                                                                  mesh.Dimension()));
+   auto sub_fec = std::unique_ptr<FiniteElementCollection>(create_fec(fec_type, p,
+                                                                      submesh.Dimension()));
+
+   FiniteElementSpace fes(&mesh, fec.get());
+   FiniteElementSpace sub_fes(&submesh, sub_fec.get());
+   GridFunction gf(&fes), gf_ext(&fes);
+   GridFunction sub_gf(&sub_fes), sub_gf_ext(&sub_fes);
+   auto coeff = FunctionCoefficient([](const Vector &coords)
+   {
+      real_t x = coords(0);
+      real_t y = coords(1);
+      real_t z = coords(2);
+      return 0.02 * sin(y * 5.0 * M_PI)
+             + 0.03 * sin(x * 5.0 * M_PI)
+             + 0.05 * sin(z * 5.0 * M_PI);
+   });
+
+   auto vcoeff = VectorFunctionCoefficient(mesh.SpaceDimension(),
+                                           [](const Vector &coords, Vector &V)
+   {
+      V.SetSize(3);
+      real_t x = coords(0);
+      real_t y = coords(1);
+      real_t z = coords(2);
+
+      V(0) = 0.02 * sin(y * 3.0 * M_PI)
+             + 0.03 * sin(x * 2.0 * M_PI)
+             + 0.05 * sin(z * 4.0 * M_PI);
+      V(1) = 0.02 * sin(z * 3.0 * M_PI)
+             + 0.03 * sin(y * 2.0 * M_PI)
+             + 0.05 * sin(x * 4.0 * M_PI);
+      V(2) = 0.02 * sin(x * 3.0 * M_PI)
+             + 0.03 * sin(y * 2.0 * M_PI)
+             + 0.05 * sin(z * 4.0 * M_PI);
+   });
+
+   if (fec_type == FECType::H1 || fec_type == FECType::L2)
+   {
+      gf.ProjectCoefficient(coeff);
+      sub_gf.ProjectCoefficient(coeff);
+   }
+   else
+   {
+      gf.ProjectCoefficient(vcoeff);
+      sub_gf.ProjectCoefficient(vcoeff);
+   }
+   gf_ext = gf;
+   sub_gf_ext = sub_gf;
+
+   SECTION("ParentToSubMesh")
+   {
+      // Direct transfer should be identical
+      SubMesh::Transfer(gf, sub_gf);
+      auto tmp = sub_gf_ext;
+      tmp -= sub_gf;
+      CHECK_NORM(tmp);
+   }
+   SECTION("PRConstraint")
+   {
+      // Application of PR should be identical in mesh and submesh for an external boundary.
+      if (mesh.Nonconforming())
+      {
+         Vector tmp;
+         if (const auto *P = fes.GetProlongationMatrix())
+         {
+            const auto *R = fes.GetRestrictionMatrix();
+
+            tmp.SetSize(R->Height());
+            R->Mult(gf, tmp);
+            P->Mult(tmp, gf);
+         }
+         if (const auto *P = sub_fes.GetProlongationMatrix())
+         {
+            const auto *R = sub_fes.GetRestrictionMatrix();
+            tmp.SetSize(R->Height());
+            R->Mult(sub_gf_ext, tmp);
+            P->Mult(tmp, sub_gf_ext);
+         }
+         SubMesh::Transfer(gf, sub_gf);
+         tmp = sub_gf_ext;
+         tmp -= sub_gf;
+         CHECK_NORM(tmp, check_pr);
+      }
+   }
+}
+
+TEST_CASE("VolumeNCSubMesh", "[SubMesh]")
+{
+   bool use_tet = GENERATE(false,true);
+
+   auto mesh = use_tet ? OrientedTriFaceMesh(1, true) : DividingPlaneMesh(false,
+                                                                          true);
+   mesh.EnsureNCMesh(true);
+   SECTION("UniformRefinement2")
+   {
+      mesh.UniformRefinement();
+      mesh.UniformRefinement();
+      SECTION("SingleAttribute")
+      {
+         Array<int> subdomain_attributes(1);
+         subdomain_attributes[0] = GENERATE(range(1,2));
+         auto submesh = SubMesh::CreateFromDomain(mesh, subdomain_attributes);
+
+         // Cast to an exposed variant to explore the internals.
+         auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+         CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+         CHECK(ncmesh_exposed.CountUniqueLeafElements() == 8*8);
+         for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+         {
+            CheckProjectMatch(mesh, submesh, fec_type);
+         }
+      }
+      SECTION("UniformRefineTwoAttribute")
+      {
+         Array<int> subdomain_attributes(2);
+         subdomain_attributes[0] = 1;
+         subdomain_attributes[1] = 2;
+         auto submesh = SubMesh::CreateFromDomain(mesh, subdomain_attributes);
+
+         // Cast to an exposed variant to explore the internals.
+         auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+         CHECK(ncmesh_exposed.GetNumRootElements() == mesh.ncmesh->GetNumRootElements());
+         CHECK(ncmesh_exposed.CountUniqueLeafElements() == 2*8*8);
+         for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+         {
+            CheckProjectMatch(mesh, submesh, fec_type);
+         }
+      }
+   }
+
+   SECTION("Nonconformal")
+   {
+      mesh.UniformRefinement();
+      Array<int> subdomain_attributes{GENERATE(1,2)};
+      auto backwards = GENERATE(false, true);
+      SECTION("ConsistentWithParent")
+      {
+         RefineSingleUnattachedElement(mesh, subdomain_attributes[0],
+                                       mesh.bdr_attributes.Max(), backwards);
+         {
+            auto submesh = SubMesh::CreateFromDomain(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 8 - 1 + 8);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type, true);
+            }
+         }
+         RefineSingleUnattachedElement(mesh, subdomain_attributes[0],
+                                       mesh.bdr_attributes.Max(), backwards);
+         {
+            auto submesh = SubMesh::CreateFromDomain(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 8 - 1 + 8 - 1 + 8);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type, true);
+            }
+         }
+      }
+
+      SECTION("InconsistentWithParent")
+      {
+         RefineSingleAttachedElement(mesh, subdomain_attributes[0],
+                                     mesh.bdr_attributes.Max(), backwards);
+         {
+            auto submesh = SubMesh::CreateFromDomain(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 8 - 1 + 8);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type, false);
+            }
+         }
+         RefineSingleAttachedElement(mesh, subdomain_attributes[0],
+                                     mesh.bdr_attributes.Max(), backwards);
+         {
+            auto submesh = SubMesh::CreateFromDomain(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 8 - 1 + 8 - 1 + 8);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type, false);
+            }
+         }
+      }
+   }
+}
+
+TEST_CASE("ExteriorSurfaceNCSubMesh", "[SubMesh]")
+{
+   SECTION("Hex")
+   {
+      auto mesh = Mesh("../../data/ref-cube.mesh", 1, 1);
+      mesh.EnsureNCMesh(true);
+      SECTION("UniformRefinement2")
+      {
+         mesh.UniformRefinement();
+         mesh.UniformRefinement();
+         SECTION("SingleAttribute")
+         {
+            Array<int> subdomain_attributes{GENERATE(range(1,6))};
+            auto submesh = SubMesh::CreateFromBoundary(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 4*4);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type);
+            }
+         }
+         SECTION("UniformRefineTwoAttribute")
+         {
+            Array<int> subdomain_attributes(2);
+            subdomain_attributes[0] = GENERATE(range(1,6));
+            subdomain_attributes[1] = 1 + (subdomain_attributes[0] % 6);
+            auto submesh = SubMesh::CreateFromBoundary(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 2);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 2*4*4);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type);
+            }
+         }
+      }
+
+      SECTION("NonconformalRefine")
+      {
+         Array<int> subdomain_attributes{GENERATE(range(1,6))};
+         mesh.UniformRefinement();
+         RefineSingleAttachedElement(mesh, 1, subdomain_attributes[0], true);
+         SECTION("Single")
+         {
+            auto submesh = SubMesh::CreateFromBoundary(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 4 - 1 + 4);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type);
+            }
+         }
+         SECTION("Double")
+         {
+            RefineSingleAttachedElement(mesh, 1, subdomain_attributes[0], false);
+            auto submesh = SubMesh::CreateFromBoundary(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 4 - 1 + 4 - 1 + 4);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type);
+            }
+         }
+      }
+   }
+
+   SECTION("Tet")
+   {
+      auto mesh = Mesh("../../data/ref-tetrahedron.mesh");
+      mesh.EnsureNCMesh(true);
+      SECTION("UniformRefinement2")
+      {
+         mesh.UniformRefinement();
+         mesh.UniformRefinement();
+         SECTION("SingleAttribute")
+         {
+            Array<int> subdomain_attributes{GENERATE(range(1,4))};
+            auto submesh = SubMesh::CreateFromBoundary(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 4*4);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type);
+            }
+         }
+
+         SECTION("UniformRefineTwoAttribute")
+         {
+            Array<int> subdomain_attributes(2);
+            subdomain_attributes[0] = GENERATE(range(1,4));
+            subdomain_attributes[1] = 1 + (subdomain_attributes[0] % 4);
+            auto submesh = SubMesh::CreateFromBoundary(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 2);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 2*4*4);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type);
+            }
+         }
+      }
+
+      SECTION("NonconformalRefine")
+      {
+         Array<int> subdomain_attributes{GENERATE(range(1,4))};
+         mesh.UniformRefinement();
+         RefineSingleAttachedElement(mesh, 1, subdomain_attributes[0], true);
+         SECTION("Single")
+         {
+            auto submesh = SubMesh::CreateFromBoundary(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 4 - 1 + 4);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type);
+            }
+         }
+         SECTION("Double")
+         {
+            RefineSingleAttachedElement(mesh, 1, subdomain_attributes[0], false);
+            auto submesh = SubMesh::CreateFromBoundary(mesh, subdomain_attributes);
+            auto ncmesh_exposed = NCMeshExposed(*submesh.ncmesh);
+            CHECK(ncmesh_exposed.GetNumRootElements() == 1);
+            CHECK(ncmesh_exposed.CountUniqueLeafElements() == 4 - 1 + 4 - 1 + 4);
+            for (auto fec_type : {FECType::H1, FECType::L2, FECType::ND, FECType::RT})
+            {
+               CheckProjectMatch(mesh, submesh, fec_type);
+            }
+         }
+      }
+   }
+}
+