Added 1D example and optimized performance.

Parallel H1 HO <-> LOR Transfer Operator

.gitignore

@@ -265,11 +265,15 @@ miniapps/navier/*_output
 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/nurbs_patch_ex1
+miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
+miniapps/nurbs/sin-fit.mesh
+miniapps/nurbs/CurveInt
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -292,6 +296,7 @@ miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
+miniapps/tools/plor-transfer
 miniapps/tools/get-values
 miniapps/tools/check-tmop-metric
 miniapps/tools/tmop-metric-magnitude

CHANGELOG

@@ -10,6 +10,7 @@
 
 Version 4.5.3 (development)
 ===========================
+- Added curve interpolation method for NURBS.
 
 New and updated examples and miniapps
 -------------------------------------
@@ -32,6 +33,8 @@ New and updated examples and miniapps
 - Added new SubMesh examples demonstrating source terms and boundary conditions
   transferred from SubMesh objects.
 
+- Added a miniapp for interpolation of NURBS.
+
 - Added a new H(div) solvers miniapp in miniapps/hdiv-linear-solver,
   demonstrating the use of a matrix-free saddle-point solver methodology,
   suitable for high-order discretizations and for GPU acceleration. Examples
@@ -42,8 +45,20 @@ New and updated examples and miniapps
 
 - Moved the distance solver methods from miniapps/shifted to miniapps/common.
 
+- Added a new parallel LOR transfer miniapp, miniapps/tools/plor-transfer, which
+  mirrors the functionality of the serial LOR transfer miniapp,
+  miniapps/tools/lor-transfer
+
 Meshing improvements
 --------------------
+- Added support for free connectivity of NURBS patches allowing for more complex
+  patch configurations such as C-meshes. This is demonstrated in a new NURBS
+  miniapp.
+  
+- The edge to knot map for NURBS meshes can be determined automatically. It is no
+  longer needed to specify this in the NURBS mesh. A mesh in the NURBS miniapp
+  demonstrates this.
+
 - Added new methods in the Mesh class to set and get attributes on NURBS patches
   and patch boundaries.
 
@@ -63,8 +78,16 @@ Discretization improvements
 
 - VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
 
+- Added support for partial assembly on NURBS patches and NURBS patch sparse
+  matrix assembly. Patch matrix assembly includes the option to use reduced
+  approximate integration rules, computed by the newly implemented non-negative
+  least-squares (NNLS) solver.
+
 - Added support for p-refined meshes in FindPointsGSLIB.
 
+- Support for parallel transfer of H1 fields using the low-order refined (LOR)
+  transfer operators in L2ProjectionGridTransfer
+
 Linear and nonlinear solvers
 ----------------------------
 - Updated interface to MUMPS direct solver to support multiple right-hand

config/docker/Dockerfile

@@ -19,9 +19,7 @@ RUN apt-get update && \
     apt-get install -y libcurl4-openssl-dev libssl-dev
 
 ENV PATH=$PATH:/opt/mfem-view/bin
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/mfem-view/lib:/opt/mfem-view/lib64
 ENV DEBIAN_FRONTEND=noninteractive
 
 # The user will see the view on shell into the container
 WORKDIR /opt/mfem-view
-ENTRYPOINT ["/bin/bash"]

config/docker/Dockerfile.base

@@ -34,14 +34,14 @@ RUN cd /opt/mfem-env && \
     . /opt/spack/share/spack/setup-env.sh && \
     spack env activate . && \
     spack develop --path /code mfem@master+examples+miniapps && \
-    spack add mfem@master+examples+miniapps # && \
-#    spack install
+    spack add mfem@master+examples+miniapps && \
+    spack install
 
 # ensure mfem always on various paths
-#RUN cd /opt/mfem-env && \
-#    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+RUN cd /opt/mfem-env && \
+    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
 
 # Present the software install when we shell in
 # The view is at /opt/mfem-env/.spack-env/view
-#WORKDIR /opt/software
-#ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]
+WORKDIR /opt/software
+ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]

config/docker/README.md

@@ -7,21 +7,31 @@ You can use this image for a demo of using mfem! 🎉️
 Updated containers are built and deployed on merges to the main branch and releases.
 If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
 
-### Usage
+## Usage
 
-Here is how to build the container. Note that we build so it belongs to the same
-namespace as the repository here. "ghcr.io" means "GitHub Container Registry" and
+We provide two containers, which you can either build or use directly from
+[GitHub packages](https://github.com/orgs/mfem/packages?repo_name=mfem).
+
+ - `ghcr.io/mfem/mfem-ubuntu-base`: a "build from scratch" for mfem
+ - `ghcr.io/mfem/mfem-ubuntu`: a quick build that uses the base container
+
+In the above, "ghcr.io" means "GitHub Container Registry" and
 is the [GitHub packages](https://github.com/features/packages) registry that supports
- Docker images and other OCI artifacts. From the root of the repository:
+Docker images and other OCI artifacts. 
+
+### Ubuntu
+
+> Use or build this container for a multi-stage, slimmer base to develop on top of mfem
+
+Note that this container is provided on GitHub packages [here](https://github.com/mfem/mfem/pkgs/container/mfem-ubuntu)
+so you don't need to build it. However, if you want to, you can do the following:
 
 ```bash
 $ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu .
-$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
 ```
 
-### Shell Ubuntu
-
-To shell into the container:
+Note that this will pull the base image. If you want to rebuild it, see [ubuntu base](#ubuntu-base)
+below. Once you have built (or prefer to pull) you can shell into the container as follows:
 
 ```bash
 $ docker run -it ghcr.io/mfem/mfem-ubuntu
@@ -37,39 +47,13 @@ bin  etc  include  lib  libexec  sbin  share  var
  - Examples are in share/mfem/examples
  - Examples are in share/mfem/miniapps
 
-You can read more about interaction with these examples and miniapps below.
-
-### Shell Ubuntu Base
-
-To shell into the container:
+Using this container, if you want to develop a tool that _uses_ mfem, you can find the libraries / includes in:
 
 ```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
-```
-
-Off the bat, you can see mfem libraries are in your path so you can jump into development:
-
-```bash
-env | grep mfem
-```
-```bash
-PKG_CONFIG_PATH=/opt/mfem-env/.spack-env/view/lib/pkgconfig:/opt/mfem-env/.spack-env/view/share/pkgconfig:/opt/mfem-env/.spack-env/view/lib64/pkgconfig
-PWD=/opt/mfem-env
-MANPATH=/opt/mfem-env/.spack-env/view/share/man:/opt/mfem-env/.spack-env/view/man:
-CMAKE_PREFIX_PATH=/opt/mfem-env/.spack-env/view
-SPACK_ENV=/opt/mfem-env
-ACLOCAL_PATH=/opt/mfem-env/.spack-env/view/share/aclocal
-LD_LIBRARY_PATH=/opt/mfem-env/.spack-env/view/lib:/opt/mfem-env/.spack-env/view/lib64
-PATH=/opt/mfem-env/.spack-env/view/bin:/opt/view/bin:/opt/spack/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-```
-
-#### Examples and MiniApps
-
-If you want to develop a tool that _uses_ mfem, you can find the built libraries in:
-
-```
-$ ls /opt/mfem-env/.spack-env/view/
-bin  etc  include  lib  libexec  sbin  share  var
+$ ls include/ | grep mfem
+mfem
+mfem-performance.hpp
+mfem.hpp
 ```
 
 And yes, this is the working directory when you shell into the container!
@@ -79,6 +63,16 @@ You can find the examples here:
 ```bash
 cd share/mfem/examples
 ```
+
+Try quickly setting the `LD_LIBRARY_PATH` so we can see the shared libraries
+we need:
+
+```bash
+export LD_LIBRARY_PATH=/opt/mfem-view/lib:$LD_LIBRARY_PATH
+```
+
+And then run:
+
 ```bash
 $ ./ex0
 Options used:
@@ -97,7 +91,6 @@ Number of unknowns: 101
 Average reduction factor = 0.140201
 ```
 
-Try running a few, and look at the associated .cpp file for the source code!
 You can also explore the "mini apps," also in share/mfem, but under miniapps.
 
 ```bash
@@ -130,18 +123,87 @@ Rule:
 Applying rule...done.
 ```
 
-Have fun!
+Have fun! As a reminder, this container is ideal for developing your own
+applications that might use mfem, or having a nice environment to test out
+examples.
 
 
-#### Your own App
-If you want to develop with your own code base
-(and mfem as is in the container) you can bind to somewhere else in the container (e.g., src)
+### Ubuntu Base
+
+> Use this build for a development environment with spack and mfem
+
+This container is also [provided on GitHub packages](https://github.com/mfem/mfem/pkgs/container/mfem-ubuntu-base), 
+however you can build it locally too:
 
 ```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/src bash
+$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
+```
+
+To shell into the container:
+
+```bash
+$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
+```
+
+Change directory to the mfem environment, setup spack, and activate the environment:
+
+```bash
+source /opt/spack/share/spack/setup-env.sh
+cd /opt/mfem-env/
+spack env activate .
+```
+
+Note that this environment is installing to the view at `/opt/view`. Since the environment
+knows to install mfem from `/code` this means that you could make changes in the container (or bind
+`/code` to your container) and then update spack:
+
+```bash
+# Note that concretization takes a hot minute!
+$ spack install
+```
+
+And if you want to load mfem:
+
+```bash
+$ spack load mfem
+$ env | grep mfem
+```
+
+In this development container, you can find the examples and miniapps alongside
+mfem under `/code`:
+
+```bash
+cd /code/examples
+```
+```bash
+$ ./ex0
+```
+```console
+Options used:
+   --mesh ../data/star.mesh
+   --order 1
+Number of unknowns: 101
+   Iteration :   0  (B r, r) = 0.184259
+   Iteration :   1  (B r, r) = 0.102754
+   Iteration :   2  (B r, r) = 0.00558141
+   Iteration :   3  (B r, r) = 1.5247e-05
+   Iteration :   4  (B r, r) = 1.13807e-07
+   Iteration :   5  (B r, r) = 6.27231e-09
+   Iteration :   6  (B r, r) = 3.76268e-11
+   Iteration :   7  (B r, r) = 6.07423e-13
+   Iteration :   8  (B r, r) = 4.10615e-15
+Average reduction factor = 0.140201
+```
+
+This container is likely ideal for someone that wants to develop mfem itself.
+For other use cases, we recommend using the slimmer image. As an example,
+if you want to develop with your own code base (and mfem as is in the container) 
+you can bind to somewhere else in the container (e.g., src)
+
+```bash
+$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/code bash
 ```
 
 In the above, we can pretend your project is in the present working directory (PWD) and we are
 binding to source. You can then use the mfem in the container for development, and if you
 want to distribute your library or app in a container, you can use the mfem container as the base.
-

examples/ex0.cpp

@@ -22,7 +22,7 @@ using namespace mfem;
 int main(int argc, char *argv[])
 {
    // 1. Parse command line options.
-   const char *mesh_file = "../data/star.mesh";
+   string mesh_file = "../data/star.mesh";
    int order = 1;
 
    OptionsParser args(argc, argv);

examples/ex0p.cpp

@@ -26,7 +26,7 @@ int main(int argc, char *argv[])
    Hypre::Init();
 
    // 2. Parse command line options.
-   const char *mesh_file = "../data/star.mesh";
+   string mesh_file = "../data/star.mesh";
    int order = 1;
 
    OptionsParser args(argc, argv);

examples/ex37.cpp

@@ -0,0 +1,506 @@
+//                                MFEM Example 37
+//
+// Compile with: make ex37
+//
+// Sample runs:  ex37
+//               ex37 -i surface
+//               ex37 -i surface -o 0
+//               ex37 -i surface -r 1
+//               ex37 -i surface -o 4
+//               ex37 -i surface -o 4 -r 5
+//               ex37 -i volumetric
+//               ex37 -i volumetric -o 0
+//               ex37 -i volumetric -r 1
+//               ex37 -i volumetric -o 4
+//               ex37 -i volumetric -o 4 -r 5
+//               ex37 -i surface3d
+//               ex37 -i surface3d -o 0
+//               ex37 -i surface3d -r 1
+//               ex37 -i surface3d -o 4
+//               ex37 -i surface3d -o 4 -r 5
+//               ex37 -i volumetric3d
+//               ex37 -i volumetric3d -o 0
+//               ex37 -i volumetric3d -r 1
+//               ex37 -i volumetric3d -o 4
+//               ex37 -i volumetric3d -o 4 -r 5
+//
+// Description: This example code demonstrates the use of MFEM to integrate
+//              functions over implicit interfaces and subdomains bounded by
+//              implicit interfaces.
+//
+//              The quadrature rules are constructed by means of moment-fitting.
+//              The interface is given by the zero iso line of a level-set
+//              function ϕ and the subdomain is given as the domain where ϕ>0
+//              holds. The algorithm for construction of the quadrature rules
+//              was introduced by Mueller, Kummer and Oberlack [1].
+//
+//              There is an example for the integration of a quadratic function
+//              over the sphere in 2 dimensions and an example computong the
+//              arclength and area of an ellipse in 2 dimensions.
+//
+//              This example showcases how to set up integrators using the
+//              integration rules on surfaces and subdomains.
+//
+// [1] Mueller, B., Kummer, F. and Oberlack, M. (2013) Highly accurate surface
+//     and volume integration on implicit domains by means of moment-fitting.
+//     Int. J. Numer. Meth. Engng. (96) 512-528. DOI:10.1002/nme.4569
+
+#include "mfem.hpp"
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/// @brief Integration rule the example should demonstrate
+enum class IntegrationType { Volumetric1D, Surface2D, Volumetric2D,
+                             Surface3D, Volumetric3D
+                           };
+IntegrationType itype;
+
+/// @brief Level-set function defining the implicit interface
+double lvlset(const Vector& X)
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return .55 - X(0);
+      case IntegrationType::Surface2D:
+         return 1. - (pow(X(0), 2.) + pow(X(1), 2.));
+      case IntegrationType::Volumetric2D:
+         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.));
+      case IntegrationType::Surface3D:
+         return 1. - (pow(X(0), 2.) + pow(X(1), 2.) + pow(X(2), 2.));
+      case IntegrationType::Volumetric3D:
+         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.) + pow(X(2) / .5, 2.));
+      default:
+         return 1.;
+   }
+}
+
+/// @brief Function that should be integrated
+double integrand(const Vector& X)
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return 1.;
+      case IntegrationType::Surface2D:
+         return 3. * pow(X(0), 2.) - pow(X(1), 2.);
+      case IntegrationType::Volumetric2D:
+         return 1.;
+      case IntegrationType::Surface3D:
+         return 4. - 3. * pow(X(0), 2.) + 2. * pow(X(1), 2.) - pow(X(2), 2.);
+      case IntegrationType::Volumetric3D:
+         return 1.;
+      default:
+         return 0.;
+   }
+}
+
+/// @brief Analytic surface integral
+double Surface()
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return 1.;
+      case IntegrationType::Surface2D:
+         return 2. * M_PI;
+      case IntegrationType::Volumetric2D:
+         return 7.26633616541076;
+      case IntegrationType::Surface3D:
+         return 40. / 3. * M_PI;
+      case IntegrationType::Volumetric3D:
+         return 9.90182151329315;
+      default:
+         return 0.;
+   }
+}
+
+/// @brief Analyitc volume integral over subdomain with positiv level-set
+double Volume()
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return .55;
+      case IntegrationType::Surface2D:
+         return NAN;
+      case IntegrationType::Volumetric2D:
+         return 9. / 8. * M_PI;
+      case IntegrationType::Surface3D:
+         return NAN;
+      case IntegrationType::Volumetric3D:
+         return 3. / 4. * M_PI;
+      default:
+         return 0.;
+   }
+}
+
+#ifdef MFEM_USE_LAPACK
+/**
+ @brief Class for surface linearform integrator
+
+ Integrator to demonstrate the use of the surface integration rule on an
+ implicit surface defined by a level-set.
+ */
+class SurfaceLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   /// @brief vector to evaluate the basis functions
+   Vector shape;
+
+   /// @brief surface integration rule
+   SIntegrationRule* SIntRule;
+
+   /// @brief coefficient representing the level-set defining the interface
+   Coefficient &LevelSet;
+
+   /// @brief coefficient representing the integrand
+   Coefficient &Q;
+
+public:
+   /**
+    @brief Constructor for the surface linear form integrator
+
+    Constructor for the surface linear form integrator to demonstrate the use
+    of the surface integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    @param [in] ir surface integrtion rule to be used
+    */
+   SurfaceLFIntegrator(Coefficient &q, Coefficient &levelset,
+                       SIntegrationRule* ir)
+      : LinearFormIntegrator(), Q(q), LevelSet(levelset), SIntRule(ir) {}
+
+   /**
+    @brief Constructor for the surface linear form integrator
+
+    Constructor for the surface linear form integrator to demonstrate the use
+    of the surface integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    */
+   SurfaceLFIntegrator(Coefficient &q, Coefficient &levelset)
+      : LinearFormIntegrator(), Q(q), LevelSet(levelset), SIntRule(NULL) {}
+
+   /**
+    @brief Assembly of the element vector
+
+    Assemble the element vector of for the right hand side on the element given
+    by the FiniteElement and ElementTransformation.
+
+    @param [in] el finite Element the vector belongs to
+    @param [in] Tr transformation of finite element
+    @param [out] elvect vector containing the
+   */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.;
+
+      // Update the surface integration rule for the current element
+      SIntRule->SetElementWithSurfaceWeights(Tr.ElementNo);
+
+      for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
+      {
+         Tr.SetIntPoint((&(SIntRule->IntPoint(ip))));
+         double val = Tr.Weight() * Q.Eval(Tr, SIntRule->IntPoint(ip));
+         el.CalcShape(SIntRule->IntPoint(ip), shape);
+         add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
+      }
+   }
+
+   /// @brief Get the level-set defining the implicit interface
+   void SetSurface(Coefficient &levelset) { LevelSet = levelset; }
+
+   /// @brief Set the surface integration rule
+   void SetSIntRule(SIntegrationRule *ir) { SIntRule = ir; }
+
+   /// @brief Get the surface integration rule
+   const SIntegrationRule* GetSIntRule() { return SIntRule; }
+};
+
+/**
+ @brief Class for subdomain linearform integrator
+
+ Integrator to demonstrate the use of the subdomain integration rule within
+ an area defined by an implicit surface defined by a level-set.
+ */
+class SubdomainLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   /// @brief vector to evaluate the basis functions
+   Vector shape;
+
+   /// @brief surface integration rule
+   CutIntegrationRule* CutIntRule;
+
+   /// @brief coefficient representing the level-set defining the interface
+   Coefficient &LevelSet;
+
+   /// @brief coefficient representing the integrand
+   Coefficient &Q;
+
+public:
+   /**
+    @brief Constructor for the volumetric subdomain linear form integrator
+
+    Constructor for the subdomain linear form integrator to demonstrate the use
+    of the volumeric subdomain integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    @param [in] ir subdomain integrtion rule to be used
+    */
+   SubdomainLFIntegrator(Coefficient &q, Coefficient &levelset,
+                         CutIntegrationRule* ir)
+      : LinearFormIntegrator(), Q(q), LevelSet(levelset), CutIntRule(ir) {}
+
+   /**
+    @brief Constructor for the volumetric subdomain linear form integrator
+
+    Constructor for the subdomain linear form integrator to demonstrate the use
+    of the volumeric subdomain integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    */
+   SubdomainLFIntegrator(Coefficient &q, Coefficient &levelset)
+      : LinearFormIntegrator(), Q(q), LevelSet(levelset), CutIntRule(NULL) {}
+
+   /**
+    @brief Assembly of the element vector
+
+    Assemble the element vector of for the right hand side on the element given
+    by the FiniteElement and ElementTransformation.
+
+    @param [in] el finite Element the vector belongs to
+    @param [in] Tr transformation of finite element
+    @param [out] elvect vector containing the
+   */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.;
+
+      // Update the subdomain integration rule
+      CutIntRule->SetElement(Tr.ElementNo);
+
+      for (int ip = 0; ip < CutIntRule->GetNPoints(); ip++)
+      {
+         Tr.SetIntPoint((&(CutIntRule->IntPoint(ip))));
+         double val = Tr.Weight()
+                      * Q.Eval(Tr, CutIntRule->IntPoint(ip));
+         el.CalcPhysShape(Tr, shape);
+         add(elvect, CutIntRule->IntPoint(ip).weight * val, shape, elvect);
+      }
+   }
+
+   /// @brief Get the level-set defining the implicit interface
+   void SetSurface(Coefficient &levelset) { LevelSet = levelset; }
+
+   /// @brief Set the volumetric subdomain integration rule
+   void SetCutIntRule(CutIntegrationRule *ir) { CutIntRule = ir; }
+
+   /// @brief Get the volumetricsubdomain integration
+   const CutIntegrationRule* GetCutIntRule() { return CutIntRule; }
+};
+#endif //MFEM_USE_LAPACK
+
+int main(int argc, char *argv[])
+{
+#ifndef MFEM_USE_LAPACK
+   cout << "MFEM must be build with LAPACK for this example." << endl;
+   return EXIT_FAILURE;
+#else
+   // 1. Parse he command-line options.
+   int ref_levels = 3;
+   int order = 2;
+   const char *inttype = "surface2d";
+   itype = IntegrationType::Surface2D;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order", "Order of quadrature rule");
+   args.AddOption(&ref_levels, "-r", "--refine", "Number of meh refinements");
+   args.AddOption(&inttype, "-i", "--integrationtype",
+                  "IntegrationType to demonstrate");
+   args.ParseCheck();
+
+   if (strcmp(inttype, "volumetric1d") == 0
+       || strcmp(inttype, "Volumetric1D") == 0)
+   {
+      itype = IntegrationType::Volumetric1D;
+   }
+   else if (strcmp(inttype, "surface2d") == 0
+            || strcmp(inttype, "Surface2D") == 0)
+   {
+      itype = IntegrationType::Surface2D;
+   }
+   else if (strcmp(inttype, "volumetric2d") == 0
+            || strcmp(inttype, "Volumetric2D") == 0)
+   {
+      itype = IntegrationType::Volumetric2D;
+   }
+   else if (strcmp(inttype, "surface3d") == 0
+            || strcmp(inttype, "Surface3d") == 0)
+   {
+      itype = IntegrationType::Surface3D;
+   }
+   else if (strcmp(inttype, "volumetric3d") == 0
+            || strcmp(inttype, "Volumetric3d") == 0)
+   {
+      itype = IntegrationType::Volumetric3D;
+   }
+
+   // 2. Construct and refine the mesh.
+   Mesh *mesh;
+   if (itype == IntegrationType::Volumetric1D)
+   {
+      mesh = new Mesh("../data/inline-segment.mesh");
+   }
+   if (itype == IntegrationType::Surface2D
+       || itype == IntegrationType::Volumetric2D)
+   {
+      mesh = new Mesh(2, 4, 1, 0, 2);
+      mesh->AddVertex(-1.6,-1.6);
+      mesh->AddVertex(1.6,-1.6);
+      mesh->AddVertex(1.6,1.6);
+      mesh->AddVertex(-1.6,1.6);
+      mesh->AddQuad(0,1,2,3);
+      mesh->FinalizeQuadMesh(1, 0, 1);
+   }
+   else if (itype == IntegrationType::Surface3D
+            || itype == IntegrationType::Volumetric3D)
+   {
+      mesh = new Mesh(3, 8, 1, 0, 3);
+      mesh->AddVertex(-1.6,-1.6,-1.6);
+      mesh->AddVertex(1.6,-1.6,-1.6);
+      mesh->AddVertex(1.6,1.6,-1.6);
+      mesh->AddVertex(-1.6,1.6,-1.6);
+      mesh->AddVertex(-1.6,-1.6,1.6);
+      mesh->AddVertex(1.6,-1.6,1.6);
+      mesh->AddVertex(1.6,1.6,1.6);
+      mesh->AddVertex(-1.6,1.6,1.6);
+      mesh->AddHex(0,1,2,3,4,5,6,7);
+      mesh->FinalizeHexMesh(1, 0, 1);
+   }
+
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 3. Define the necessary finite element space on the mesh.
+   H1_FECollection fe_coll(1, mesh->Dimension());
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, &fe_coll);
+
+   // 4.
+   FunctionCoefficient levelset(lvlset);
+   FunctionCoefficient u(integrand);
+
+   // 5. Define the necessary Integration rules on element 0.
+   IsoparametricTransformation Tr;
+   mesh->GetElementTransformation(0, &Tr);
+   SIntegrationRule* sir = new SIntegrationRule(order, Tr, levelset);
+   CutIntegrationRule* cir = NULL;
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      cir = new CutIntegrationRule(order, Tr, levelset);
+   }
+
+   // 6. Define and assemble the linar forms on the finite element space.
+   LinearForm surface(fespace);
+   LinearForm volume(fespace);
+
+   surface.AddDomainIntegrator(new SurfaceLFIntegrator(u, levelset, sir));
+   surface.Assemble();
+
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      volume.AddDomainIntegrator(new SubdomainLFIntegrator(u, levelset, cir));
+      volume.Assemble();
+   }
+
+   // 7. Print information, computed values and errors to the console.
+   int qorder = 0;
+   int nbasis = 2 * (order + 1) + (int)(order * (order + 1) / 2);
+   IntegrationRules irs(0, Quadrature1D::GaussLegendre);
+   IntegrationRule ir = irs.Get(Geometry::SQUARE, qorder);
+   for (; ir.GetNPoints() <= nbasis; qorder++)
+   {
+      ir = irs.Get(Geometry::SQUARE, qorder);
+   }
+   cout << "============================================" << endl;
+   cout << "Mesh size dx:                       ";
+   if (itype != IntegrationType::Volumetric1D)
+   {
+      cout << 3.2 / pow(2., (double)ref_levels) << endl;
+   }
+   else
+   {
+      cout << .25 / pow(2., (double)ref_levels) << endl;
+   }
+   if (itype == IntegrationType::Surface2D
+       || itype == IntegrationType::Volumetric2D)
+   {
+      cout << "Number of div free basis functions: " << nbasis << endl;
+      cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
+   }
+   cout << scientific << setprecision(2);
+   cout << "============================================" << endl;
+   cout << "Computed value of surface integral: " << surface.Sum() << endl;
+   cout << "True value of surface integral:     " << Surface() << endl;
+   cout << "Absolut Error (Surface):            ";
+   cout << abs(surface.Sum() - Surface()) << endl;
+   cout << "Relative Error (Surface):           ";
+   cout << abs(surface.Sum() - Surface()) / Surface() << endl;
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      cout << "--------------------------------------------" << endl;
+      cout << "Computed value of volume integral:  " << volume.Sum() << endl;
+      cout << "True value of volume integral:      " << Volume() << endl;
+      cout << "Absolut Error (Volume):             ";
+      cout << abs(volume.Sum() - Volume()) << endl;
+      cout << "Relative Error (Volume):            ";
+      cout << abs(volume.Sum() - Volume()) / Volume() << endl;
+   }
+   cout << "============================================" << endl;
+
+   // 8. Plot the level-set function on a high order finite element space.
+   H1_FECollection fe_coll2(5, mesh->Dimension());
+   FiniteElementSpace fespace2(mesh, &fe_coll2);
+   FunctionCoefficient levelset_coeff(levelset);
+   GridFunction lgf(&fespace2);
+   lgf.ProjectCoefficient(levelset_coeff);
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock(vishost, visport);
+   sol_sock.precision(8);
+   sol_sock << "solution\n" << *mesh << lgf << flush;
+   sol_sock << "keys pppppppppppppppppppppppppppcmmlRj\n";
+   sol_sock << "levellines " << 0. << " " << 0. << " " << 1 << "\n" << flush;
+
+   delete sir;
+   delete cir;
+   delete fespace;
+   delete mesh;
+   return EXIT_SUCCESS;
+#endif //MFEM_USE_LAPACK
+}

examples/makefile

@@ -31,6 +31,9 @@ SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
   ex24p ex25p ex26p ex34p ex35p
 
+ifeq ($(MFEM_USE_LAPACK),YES)
+	SEQ_EXAMPLES += ex37
+endif
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else

fem/CMakeLists.txt

@@ -23,6 +23,7 @@ set(SRCS
   integ/bilininteg_diffusion_mf.cpp
   integ/bilininteg_diffusion_pa.cpp
   integ/bilininteg_diffusion_ea.cpp
+  integ/bilininteg_diffusion_patch.cpp
   integ/bilininteg_divdiv_pa.cpp
   integ/bilininteg_gradient_pa.cpp
   integ/bilininteg_interp_pa.cpp

fem/bilinearform.cpp

@@ -13,6 +13,7 @@
 
 #include "fem.hpp"
 #include "../general/device.hpp"
+#include "../mesh/nurbs.hpp"
 #include <cmath>
 
 namespace mfem
@@ -421,23 +422,30 @@ void BilinearForm::Assemble(int skip_zeros)
                         "invalid element marker for domain integrator #"
                         << k << ", counting from zero");
          }
+
+         if (domain_integs[k]->Patchwise())
+         {
+            MFEM_VERIFY(fes->GetNURBSext(), "Patchwise integration requires a "
+                        << "NURBS FE space");
+         }
       }
 
+      // Element-wise integration
       for (int i = 0; i < fes -> GetNE(); i++)
       {
-         int elem_attr = fes->GetMesh()->GetAttribute(i);
-         doftrans = fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
          {
             elmat_p = &(*element_matrices)(i);
          }
          else
          {
+            const int elem_attr = fes->GetMesh()->GetAttribute(i);
             elmat.SetSize(0);
             for (int k = 0; k < domain_integs.Size(); k++)
             {
-               if ( domain_integs_marker[k] == NULL ||
+               if ((domain_integs_marker[k] == NULL ||
                     (*(domain_integs_marker[k]))[elem_attr-1] == 1)
+                   && !domain_integs[k]->Patchwise())
                {
                   const FiniteElement &fe = *fes->GetFE(i);
                   eltrans = fes->GetElementTransformation(i);
@@ -460,6 +468,7 @@ void BilinearForm::Assemble(int skip_zeros)
             {
                elmat_p = &elmat;
             }
+            doftrans = fes->GetElementVDofs(i, vdofs);
             if (doftrans)
             {
                doftrans->TransformDual(elmat);
@@ -479,6 +488,43 @@ void BilinearForm::Assemble(int skip_zeros)
             }
          }
       }
+
+      // Patch-wise integration
+      if (fes->GetNURBSext())
+      {
+         for (int p=0; p<mesh->NURBSext->GetNP(); ++p)
+         {
+            bool vdofsSet = false;
+            for (int k = 0; k < domain_integs.Size(); k++)
+            {
+               if (domain_integs[k]->Patchwise())
+               {
+                  if (!vdofsSet)
+                  {
+                     fes->GetPatchVDofs(p, vdofs);
+                     vdofsSet = true;
+                  }
+
+                  SparseMatrix* spmat = nullptr;
+                  domain_integs[k]->AssemblePatchMatrix(p, *fes, spmat);
+                  Array<int> cols;
+                  Vector srow;
+
+                  for (int r=0; r<spmat->Height(); ++r)
+                  {
+                     spmat->GetRow(r, cols, srow);
+                     for (int i=0; i<cols.Size(); ++i)
+                     {
+                        cols[i] = vdofs[cols[i]];
+                     }
+                     mat->AddRow(vdofs[r], cols, srow);
+                  }
+
+                  delete spmat;
+               }
+            }
+         }
+      }
    }
 
    if (boundary_integs.Size())

fem/bilinearform_ext.cpp

@@ -299,7 +299,16 @@ void PABilinearFormExtension::Assemble()
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
    for (BilinearFormIntegrator *integ : integrators)
    {
-      integ->AssemblePA(*a->FESpace());
+      if (integ->Patchwise())
+      {
+         MFEM_VERIFY(a->FESpace()->GetNURBSext(),
+                     "Patchwise integration requires a NURBS FE space");
+         integ->AssembleNURBSPA(*a->FESpace());
+      }
+      else
+      {
+         integ->AssemblePA(*a->FESpace());
+      }
    }
 
    Array<BilinearFormIntegrator*> &bdr_integrators = *a->GetBBFI();
@@ -410,13 +419,39 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
 
    const int iSz = integrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
+
+   bool allPatchwise = true;
+   bool somePatchwise = false;
+
+   for (int i = 0; i < iSz; ++i)
+   {
+      if (integrators[i]->Patchwise())
+      {
+         somePatchwise = true;
+      }
+      else
+      {
+         allPatchwise = false;
+      }
+   }
+
+   MFEM_VERIFY(!(somePatchwise && !allPatchwise),
+               "All or none of the integrators should be patchwise");
+
+   if (DeviceCanUseCeed() || !elem_restrict || allPatchwise)
    {
       y.UseDevice(true); // typically this is a large vector, so store on device
       y = 0.0;
       for (int i = 0; i < iSz; ++i)
       {
-         integrators[i]->AddMultPA(x, y);
+         if (integrators[i]->Patchwise())
+         {
+            integrators[i]->AddMultNURBSPA(x, y);
+         }
+         else
+         {
+            integrators[i]->AddMultPA(x, y);
+         }
       }
    }
    else

fem/bilininteg.cpp

@@ -26,6 +26,12 @@ void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
               "   is not implemented for this class.");
 }
 
+void BilinearFormIntegrator::AssembleNURBSPA(const FiniteElementSpace&)
+{
+   mfem_error ("BilinearFormIntegrator::AssembleNURBSPA(fes)\n"
+               "   is not implemented for this class.");
+}
+
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                         const FiniteElementSpace&)
 {
@@ -92,7 +98,13 @@ void BilinearFormIntegrator::AssembleDiagonalPA_ADAt(const Vector &, Vector &)
 
 void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
 {
-   MFEM_ABORT("BilinearFormIntegrator::MultAssembled(...)\n"
+   MFEM_ABORT("BilinearFormIntegrator:AddMultPA:(...)\n"
+              "   is not implemented for this class.");
+}
+
+void BilinearFormIntegrator::AddMultNURBSPA(const Vector &, Vector &) const
+{
+   MFEM_ABORT("BilinearFormIntegrator::AddMultNURBSPA(...)\n"
               "   is not implemented for this class.");
 }
 
@@ -126,23 +138,30 @@ void BilinearFormIntegrator::AssembleDiagonalMF(Vector &)
               "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AssembleElementMatrix (
+void BilinearFormIntegrator::AssembleElementMatrix(
    const FiniteElement &el, ElementTransformation &Trans,
-   DenseMatrix &elmat )
+   DenseMatrix &elmat)
 {
    MFEM_ABORT("BilinearFormIntegrator::AssembleElementMatrix(...)\n"
               "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AssembleElementMatrix2 (
+void BilinearFormIntegrator::AssembleElementMatrix2(
    const FiniteElement &el1, const FiniteElement &el2,
-   ElementTransformation &Trans, DenseMatrix &elmat )
+   ElementTransformation &Trans, DenseMatrix &elmat)
 {
    MFEM_ABORT("BilinearFormIntegrator::AssembleElementMatrix2(...)\n"
               "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AssembleFaceMatrix (
+void BilinearFormIntegrator::AssemblePatchMatrix(
+   const int patch, const FiniteElementSpace &fes, SparseMatrix*& smat)
+{
+   mfem_error ("BilinearFormIntegrator::AssemblePatchMatrix(...)\n"
+               "   is not implemented for this class.");
+}
+
+void BilinearFormIntegrator::AssembleFaceMatrix(
    const FiniteElement &el1, const FiniteElement &el2,
    FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
@@ -848,6 +867,19 @@ void DiffusionIntegrator::AssembleElementMatrix
 
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
 
+   const NURBSFiniteElement *NURBSFE =
+      dynamic_cast<const NURBSFiniteElement *>(&el);
+
+   bool deleteRule = false;
+   if (NURBSFE && patchRules)
+   {
+      const int patch = NURBSFE->GetPatch();
+      const int* ijk = NURBSFE->GetIJK();
+      Array<const KnotVector*>& kv = NURBSFE->KnotVectors();
+      ir = &patchRules->GetElementRule(NURBSFE->GetElement(), patch, ijk, kv,
+                                       deleteRule);
+   }
+
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
@@ -882,6 +914,11 @@ void DiffusionIntegrator::AssembleElementMatrix
          AddMult_a_AAt(w, dshapedxt, elmat);
       }
    }
+
+   if (deleteRule)
+   {
+      delete ir;
+   }
 }
 
 void DiffusionIntegrator::AssembleElementMatrix2(

fem/bilininteg.hpp

@@ -61,6 +61,11 @@ public:
    virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                            const FiniteElementSpace &test_fes);
 
+   /// Method defining partial assembly on NURBS patches.
+   /** The result of the partial assembly is stored internally so that it can be
+       used later in the method AddMultNURBSPA(). */
+   virtual void AssembleNURBSPA(const FiniteElementSpace &fes);
+
    virtual void AssemblePABoundary(const FiniteElementSpace &fes);
 
    virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes);
@@ -82,6 +87,9 @@ public:
        called. */
    virtual void AddMultPA(const Vector &x, Vector &y) const;
 
+   /// Method for partially assembled action on NURBS patches.
+   virtual void AddMultNURBSPA(const Vector&x, Vector&y) const;
+
    /// Method for partially assembled transposed action.
    /** Perform the transpose action of integrator on the input @a x and add the
        result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
@@ -148,6 +156,13 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 
+   /** Given a particular NURBS patch, computes the patch matrix as a
+       SparseMatrix @a smat.
+    */
+   virtual void AssemblePatchMatrix(const int patch,
+                                    const FiniteElementSpace &fes,
+                                    SparseMatrix*& smat);
+
    virtual void AssembleFaceMatrix(const FiniteElement &el1,
                                    const FiniteElement &el2,
                                    FaceElementTransformations &Trans,
@@ -2111,6 +2126,59 @@ private:
    Vector pa_data;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
+   // Data for NURBS patch PA
+
+   // Type for a variable-row-length 2D array, used for data related to 1D
+   // quadrature rules in each dimension.
+   typedef std::vector<std::vector<int>> IntArrayVar2D;
+
+   int numPatches = 0;
+   static constexpr int numTypes = 2;  // Number of rule types
+
+   // In the case integrationMode == Mode::PATCHWISE_REDUCED, an approximate
+   // integration rule with sparse nonzero weights is computed by NNLSSolver,
+   // for each 1D basis function on each patch, in each spatial dimension. For a
+   // fixed 1D basis function b_i with DOF index i, in the tensor product basis
+   // of patch p, the prescribed exact 1D rule is of the form
+   // \sum_k a_{i,j,k} w_k for some integration points indexed by k, with
+   // weights w_k and coefficients a_{i,j,k} depending on Q(x), an element
+   // transformation, b_i, and b_j, for all 1D basis functions b_j whose support
+   // overlaps that of b_i. Define the constraint matrix G = [g_{j,k}] with
+   // g_{j,k} = a_{i,j,k} and the vector of exact weights w = [w_k]. A reduced
+   // rule should have different weights w_r, many of them zero, and should
+   // approximately satisfy Gw_r = Gw. A sparse approximate solution to this
+   // underdetermined system is computed by NNLSSolver, and its data is stored
+   // in the following members.
+
+   // For each patch p, spatial dimension d (total dim), and rule type t (total
+   // numTypes), an std::vector<Vector> of reduced quadrature weights for all
+   // basis functions is stored in reducedWeights[t + numTypes * (d + dim * p)],
+   // reshaped as rw(t,d,p). Note that nd may vary with respect to the patch and
+   // spatial dimension. Array reducedIDs is treated similarly.
+   std::vector<std::vector<Vector>> reducedWeights;
+   std::vector<IntArrayVar2D> reducedIDs;
+   std::vector<Array<int>> pQ1D, pD1D;
+   std::vector<std::vector<Array2D<double>>> pB, pG;
+   std::vector<IntArrayVar2D> pminD, pmaxD, pminQ, pmaxQ, pminDD, pmaxDD;
+
+   std::vector<Array<const IntegrationRule*>> pir1d;
+
+   void SetupPatchPA(const int patch, Mesh *mesh, bool unitWeights=false);
+
+   void SetupPatchBasisData(Mesh *mesh, unsigned int patch);
+
+   /** Called by AssemblePatchMatrix for sparse matrix assembly on a NURBS patch
+    with full 1D quadrature rules. */
+   void AssemblePatchMatrix_fullQuadrature(const int patch,
+                                           const FiniteElementSpace &fes,
+                                           SparseMatrix*& smat);
+
+   /** Called by AssemblePatchMatrix for sparse matrix assembly on a NURBS patch
+    with reduced 1D quadrature rules. */
+   void AssemblePatchMatrix_reducedQuadrature(const int patch,
+                                              const FiniteElementSpace &fes,
+                                              SparseMatrix*& smat);
+
 public:
    /// Construct a diffusion integrator with coefficient Q = 1
    DiffusionIntegrator(const IntegrationRule *ir = nullptr)
@@ -2146,6 +2214,14 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 
+   virtual void AssemblePatchMatrix(const int patch,
+                                    const FiniteElementSpace &fes,
+                                    SparseMatrix*& smat);
+
+   virtual void AssembleNURBSPA(const FiniteElementSpace &fes);
+
+   void AssemblePatchPA(const int patch, const FiniteElementSpace &fes);
+
    /// Perform the local action of the BilinearFormIntegrator
    virtual void AssembleElementVector(const FiniteElement &el,
                                       ElementTransformation &Tr,
@@ -2180,6 +2256,10 @@ public:
 
    virtual void AddMultTransposePA(const Vector&, Vector&) const;
 
+   virtual void AddMultNURBSPA(const Vector&, Vector&) const;
+
+   void AddMultPatchPA(const int patch, const Vector &x, Vector &y) const;
+
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe);
 

fem/datacollection.cpp

@@ -922,7 +922,7 @@ void ParaViewDataCollection::Save()
    {
       const std::string &field_name = qfield.first;
       std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid));
-      qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel());
+      qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel(), field_name);
    }
 
    // MPI rank 0 also creates a "PVTU" file that points to all of the separately

fem/fe/fe_nurbs.hpp

@@ -56,6 +56,10 @@ public:
    Vector              &Weights    ()         const { return weights; }
    /// Update the NURBSFiniteElement according to the currently set knot vectors
    virtual void         SetOrder   ()         const { }
+
+   /// Returns the indices (i,j) in 2D or (i,j,k) in 3D of this element in the
+   /// tensor product ordering of the patch.
+   const int* GetIJK() const { return ijk; }
 };
 
 

fem/fe_coll.cpp

@@ -1713,7 +1713,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          H1_Elements[Geometry::SEGMENT] = new H1_SegmentElement(p, btype);
       }
 
-      SegDofOrd[0] = new int[2*pm1];
+      SegDofOrd[0] = (pm1 > 0) ? new int[2*pm1] : nullptr;
       SegDofOrd[1] = SegDofOrd[0] + pm1;
       for (int i = 0; i < pm1; i++)
       {
@@ -1751,7 +1751,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
 
       const int &TriDof = H1_dof[Geometry::TRIANGLE];
       const int &QuadDof = H1_dof[Geometry::SQUARE];
-      TriDofOrd[0] = new int[6*TriDof];
+      TriDofOrd[0] = (TriDof > 0) ? new int[6*TriDof] : nullptr;
       for (int i = 1; i < 6; i++)
       {
          TriDofOrd[i] = TriDofOrd[i-1] + TriDof;
@@ -1772,7 +1772,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          }
       }
 
-      QuadDofOrd[0] = new int[8*QuadDof];
+      QuadDofOrd[0] = (QuadDof > 0) ? new int[8*QuadDof] : nullptr;
       for (int i = 1; i < 8; i++)
       {
          QuadDofOrd[i] = QuadDofOrd[i-1] + QuadDof;
@@ -1855,7 +1855,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          H1_Elements[Geometry::PYRAMID] = new LinearPyramidFiniteElement;
 
          const int &TetDof = H1_dof[Geometry::TETRAHEDRON];
-         TetDofOrd[0] = new int[24*TetDof];
+         TetDofOrd[0] = (TetDof > 0) ? new int[24*TetDof] : nullptr;
          for (int i = 1; i < 24; i++)
          {
             TetDofOrd[i] = TetDofOrd[i-1] + TetDof;
@@ -2127,7 +2127,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
       // No need to set the map_type for Tr_Elements.
 
       const int pp1 = p + 1;
-      SegDofOrd[0] = new int[2*pp1];
+      SegDofOrd[0] = (pp1 > 0) ? new int[2*pp1] : nullptr;
       SegDofOrd[1] = SegDofOrd[0] + pp1;
       for (int i = 0; i <= p; i++)
       {
@@ -2160,7 +2160,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
       }
 
       const int TriDof = L2_Elements[Geometry::TRIANGLE]->GetDof();
-      TriDofOrd[0] = new int[6*TriDof];
+      TriDofOrd[0] = (TriDof > 0) ? new int[6*TriDof] : nullptr;
       for (int i = 1; i < 6; i++)
       {
          TriDofOrd[i] = TriDofOrd[i-1] + TriDof;
@@ -2181,7 +2181,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          }
       }
       const int QuadDof = L2_Elements[Geometry::SQUARE]->GetDof();
-      OtherDofOrd = new int[QuadDof];
+      OtherDofOrd = (QuadDof > 0) ? new int[QuadDof] : nullptr;
       for (int j = 0; j < QuadDof; j++)
       {
          OtherDofOrd[j] = j; // for Or == 0
@@ -2225,7 +2225,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
       const int PriDof = L2_Elements[Geometry::PRISM]->GetDof();
       const int MaxDof = std::max(TetDof, std::max(PriDof, HexDof));
 
-      TetDofOrd[0] = new int[24*TetDof];
+      TetDofOrd[0] = (TetDof > 0) ? new int[24*TetDof] : nullptr;
       for (int i = 1; i < 24; i++)
       {
          TetDofOrd[i] = TetDofOrd[i-1] + TetDof;
@@ -2314,7 +2314,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
             }
          }
       }
-      OtherDofOrd = new int[MaxDof];
+      OtherDofOrd = (MaxDof > 0) ? new int[MaxDof] : nullptr;
       for (int j = 0; j < MaxDof; j++)
       {
          OtherDofOrd[j] = j; // for Or == 0
@@ -2502,7 +2502,7 @@ void RT_FECollection::InitFaces(const int p, const int dim_,
       RT_Elements[Geometry::SEGMENT] = l2_seg;
       RT_dof[Geometry::SEGMENT] = pp1;
 
-      SegDofOrd[0] = new int[2*pp1];
+      SegDofOrd[0] = (pp1 > 0) ? new int[2*pp1] : nullptr;
       SegDofOrd[1] = SegDofOrd[0] + pp1;
       for (int i = 0; i <= p; i++)
       {
@@ -2523,7 +2523,7 @@ void RT_FECollection::InitFaces(const int p, const int dim_,
       RT_dof[Geometry::SQUARE] = pp1*pp1;
 
       int TriDof = RT_dof[Geometry::TRIANGLE];
-      TriDofOrd[0] = new int[6*TriDof];
+      TriDofOrd[0] = (TriDof > 0) ? new int[6*TriDof] : nullptr;
       for (int i = 1; i < 6; i++)
       {
          TriDofOrd[i] = TriDofOrd[i-1] + TriDof;
@@ -2553,7 +2553,7 @@ void RT_FECollection::InitFaces(const int p, const int dim_,
       }
 
       int QuadDof = RT_dof[Geometry::SQUARE];
-      QuadDofOrd[0] = new int[8*QuadDof];
+      QuadDofOrd[0] = (QuadDof > 0) ? new int[8*QuadDof] : nullptr;
       for (int i = 1; i < 8; i++)
       {
          QuadDofOrd[i] = QuadDofOrd[i-1] + QuadDof;
@@ -2749,7 +2749,7 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       ND_Elements[Geometry::SEGMENT] = new ND_SegmentElement(p, ob_type);
       ND_dof[Geometry::SEGMENT] = p;
 
-      SegDofOrd[0] = new int[2*p];
+      SegDofOrd[0] = (p > 0) ? new int[2*p] : nullptr;
       SegDofOrd[1] = SegDofOrd[0] + p;
       for (int i = 0; i < p; i++)
       {
@@ -2769,7 +2769,7 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       ND_dof[Geometry::TRIANGLE] = p*pm1;
 
       int QuadDof = ND_dof[Geometry::SQUARE];
-      QuadDofOrd[0] = new int[8*QuadDof];
+      QuadDofOrd[0] = (QuadDof > 0) ? new int[8*QuadDof] : nullptr;
       for (int i = 1; i < 8; i++)
       {
          QuadDofOrd[i] = QuadDofOrd[i-1] + QuadDof;
@@ -2813,7 +2813,7 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       }
 
       int TriDof = ND_dof[Geometry::TRIANGLE];
-      TriDofOrd[0] = new int[6*TriDof];
+      TriDofOrd[0] = (TriDof > 0) ? new int[6*TriDof] : nullptr;
       for (int i = 1; i < 6; i++)
       {
          TriDofOrd[i] = TriDofOrd[i-1] + TriDof;
@@ -3163,7 +3163,7 @@ ND_R2D_FECollection::ND_R2D_FECollection(const int p, const int dim,
                                                                  ob_type);
       ND_dof[Geometry::SEGMENT] = 2 * p - 1;
 
-      SegDofOrd[0] = new int[4 * p - 2];
+      SegDofOrd[0] = (4*p > 2) ? new int[4 * p - 2] : nullptr;
       SegDofOrd[1] = SegDofOrd[0] + 2 * p - 1;
       for (int i = 0; i < p; i++)
       {
@@ -3347,7 +3347,7 @@ void RT_R2D_FECollection::InitFaces(const int p, const int dim,
       RT_Elements[Geometry::SEGMENT] = l2_seg;
       RT_dof[Geometry::SEGMENT] = pp1;
 
-      SegDofOrd[0] = new int[2*pp1];
+      SegDofOrd[0] = (pp1 > 0) ? new int[2*pp1] : nullptr;
       SegDofOrd[1] = SegDofOrd[0] + pp1;
       for (int i = 0; i <= p; i++)
       {

fem/fem.hpp

@@ -13,6 +13,7 @@
 #define MFEM_FEM_HPP
 
 #include "intrules.hpp"
+#include "intrules_cut.hpp"
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"

fem/fespace.cpp

@@ -309,6 +309,12 @@ FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
    }
 }
 
+void FiniteElementSpace::GetPatchVDofs(int i, Array<int> &vdofs) const
+{
+   GetPatchDofs(i, vdofs);
+   DofsToVDofs(vdofs);
+}
+
 void FiniteElementSpace::GetFaceVDofs(int i, Array<int> &vdofs) const
 {
    GetFaceDofs(i, vdofs);
@@ -2801,6 +2807,13 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
    return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }
 
+void FiniteElementSpace::GetPatchDofs(int patch, Array<int> &dofs) const
+{
+   MFEM_ASSERT(NURBSext,
+               "FiniteElementSpace::GetPatchDofs needs a NURBSExtension");
+   NURBSext->GetPatchDofs(patch, dofs);
+}
+
 const FiniteElement *FiniteElementSpace::GetFE(int i) const
 {
    if (i < 0 || !mesh->GetNE()) { return NULL; }

fem/fespace.hpp

@@ -811,6 +811,11 @@ public:
    virtual DofTransformation *GetBdrElementDofs(int bel,
                                                 Array<int> &dofs) const;
 
+   /** @brief Returns indices of degrees of freedom for NURBS patch index
+    @a patch. Cartesian ordering is used, for the tensor-product degrees of
+    freedom. */
+   void GetPatchDofs(int patch, Array<int> &dofs) const;
+
    /// @brief Returns the indices of the degrees of freedom for the specified
    /// face, including the DOFs for the edges and the vertices of the face.
    ///
@@ -995,7 +1000,7 @@ public:
 
    /// @brief Returns indices of degrees of freedom for the @a i'th element.
    /// The returned indices are offsets into an @ref ldof vector with @b vdim
-   /// not necessarily equal to 1. The returned indexes are always ordered
+   /// not necessarily equal to 1. The returned indices are always ordered
    /// byNODES, irrespective of whether the space is byNODES or byVDIM.
    /// See also GetElementDofs().
    ///
@@ -1024,6 +1029,9 @@ public:
    /// @note The returned object should NOT be deleted by the caller.
    DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;
 
+   /// Returns indices of degrees of freedom in @a vdofs for NURBS patch @a i.
+   void GetPatchVDofs(int i, Array<int> &vdofs) const;
+
    /// @brief Returns the indices of the degrees of freedom for the specified
    /// face, including the DOFs for the edges and the vertices of the face.
    ///

fem/gslib.cpp

@@ -1236,7 +1236,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
    gsl_ref.SetSize(points_cnt * dim);
    gsl_dist.SetSize(points_cnt);
 
-   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[], int dim)
+   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[])
    {
       for (int d = 0; d < dim; d++)
       {
@@ -1256,7 +1256,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
    {
       const double *xv_base[2];
       unsigned xv_stride[2];
-      xvFill(xv_base, xv_stride, dim);
+      xvFill(xv_base, xv_stride);
       findptsms_2(gsl_code.GetData(), sizeof(unsigned int),
                   gsl_proc.GetData(), sizeof(unsigned int),
                   gsl_elem.GetData(), sizeof(unsigned int),
@@ -1270,7 +1270,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
    {
       const double *xv_base[3];
       unsigned xv_stride[3];
-      xvFill(xv_base, xv_stride, dim);
+      xvFill(xv_base, xv_stride);
       findptsms_3(gsl_code.GetData(), sizeof(unsigned int),
                   gsl_proc.GetData(), sizeof(unsigned int),
                   gsl_elem.GetData(), sizeof(unsigned int),

fem/integ/bilininteg_diffusion_pa.cpp

@@ -12,6 +12,7 @@
 #include "../bilininteg.hpp"
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
+#include "../../mesh/nurbs.hpp"
 #include "../ceed/integrators/diffusion/diffusion.hpp"
 #include "bilininteg_diffusion_kernels.hpp"
 
@@ -74,6 +75,29 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
                               ir->GetWeights(), geom->J, coeff, pa_data);
 }
 
+void DiffusionIntegrator::AssembleNURBSPA(const FiniteElementSpace &fes)
+{
+   fespace = &fes;
+   Mesh *mesh = fes.GetMesh();
+   dim = mesh->Dimension();
+   MFEM_VERIFY(3 == dim, "Only 3D so far");
+
+   numPatches = mesh->NURBSext->GetNP();
+   for (int p=0; p<numPatches; ++p)
+   {
+      AssemblePatchPA(p, fes);
+   }
+}
+
+void DiffusionIntegrator::AssemblePatchPA(const int patch,
+                                          const FiniteElementSpace &fes)
+{
+   Mesh *mesh = fes.GetMesh();
+   SetupPatchBasisData(mesh, patch);
+
+   SetupPatchPA(patch, mesh);  // For full quadrature, unitWeights = false
+}
+
 void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
    if (DeviceCanUseCeed())
@@ -115,4 +139,221 @@ void DiffusionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
+// This version uses full 1D quadrature rules, taking into account the
+// minimum interaction between basis functions and integration points.
+void DiffusionIntegrator::AddMultPatchPA(const int patch, const Vector &x,
+                                         Vector &y) const
+{
+   MFEM_VERIFY(3 == dim, "Only 3D so far");
+
+   const Array<int>& Q1D = pQ1D[patch];
+   const Array<int>& D1D = pD1D[patch];
+
+   const std::vector<Array2D<double>>& B = pB[patch];
+   const std::vector<Array2D<double>>& G = pG[patch];
+
+   const IntArrayVar2D& minD = pminD[patch];
+   const IntArrayVar2D& maxD = pmaxD[patch];
+   const IntArrayVar2D& minQ = pminQ[patch];
+   const IntArrayVar2D& maxQ = pmaxQ[patch];
+
+   auto X = Reshape(x.Read(), D1D[0], D1D[1], D1D[2]);
+   auto Y = Reshape(y.ReadWrite(), D1D[0], D1D[1], D1D[2]);
+
+   const auto qd = Reshape(pa_data.Read(), Q1D[0]*Q1D[1]*Q1D[2],
+                           (symmetric ? 6 : 9));
+
+   // NOTE: the following is adapted from AssemblePatchMatrix_fullQuadrature
+   std::vector<Array3D<double>> grad(dim);
+   // TODO: Can an optimal order of dimensions be determined, for each patch?
+   Array3D<double> gradXY(3, std::max(Q1D[0], D1D[0]), std::max(Q1D[1], D1D[1]));
+   Array2D<double> gradX(3, std::max(Q1D[0], D1D[0]));
+
+   for (int d=0; d<dim; ++d)
+   {
+      grad[d].SetSize(Q1D[0], Q1D[1], Q1D[2]);
+
+      for (int qz = 0; qz < Q1D[2]; ++qz)
+      {
+         for (int qy = 0; qy < Q1D[1]; ++qy)
+         {
+            for (int qx = 0; qx < Q1D[0]; ++qx)
+            {
+               grad[d](qx,qy,qz) = 0.0;
+            }
+         }
+      }
+   }
+
+   for (int dz = 0; dz < D1D[2]; ++dz)
+   {
+      for (int qy = 0; qy < Q1D[1]; ++qy)
+      {
+         for (int qx = 0; qx < Q1D[0]; ++qx)
+         {
+            for (int d=0; d<dim; ++d)
+            {
+               gradXY(d,qx,qy) = 0.0;
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D[1]; ++dy)
+      {
+         for (int qx = 0; qx < Q1D[0]; ++qx)
+         {
+            gradX(0,qx) = 0.0;
+            gradX(1,qx) = 0.0;
+         }
+         for (int dx = 0; dx < D1D[0]; ++dx)
+         {
+            const double s = X(dx,dy,dz);
+            for (int qx = minD[0][dx]; qx <= maxD[0][dx]; ++qx)
+            {
+               gradX(0,qx) += s * B[0](qx,dx);
+               gradX(1,qx) += s * G[0](qx,dx);
+            }
+         }
+         for (int qy = minD[1][dy]; qy <= maxD[1][dy]; ++qy)
+         {
+            const double wy  = B[1](qy,dy);
+            const double wDy = G[1](qy,dy);
+            // This full range of qx values is generally necessary.
+            for (int qx = 0; qx < Q1D[0]; ++qx)
+            {
+               const double wx  = gradX(0,qx);
+               const double wDx = gradX(1,qx);
+               gradXY(0,qx,qy) += wDx * wy;
+               gradXY(1,qx,qy) += wx  * wDy;
+               gradXY(2,qx,qy) += wx  * wy;
+            }
+         }
+      }
+      for (int qz = minD[2][dz]; qz <= maxD[2][dz]; ++qz)
+      {
+         const double wz  = B[2](qz,dz);
+         const double wDz = G[2](qz,dz);
+         for (int qy = 0; qy < Q1D[1]; ++qy)
+         {
+            for (int qx = 0; qx < Q1D[0]; ++qx)
+            {
+               grad[0](qx,qy,qz) += gradXY(0,qx,qy) * wz;
+               grad[1](qx,qy,qz) += gradXY(1,qx,qy) * wz;
+               grad[2](qx,qy,qz) += gradXY(2,qx,qy) * wDz;
+            }
+         }
+      }
+   }
+
+   for (int qz = 0; qz < Q1D[2]; ++qz)
+   {
+      for (int qy = 0; qy < Q1D[1]; ++qy)
+      {
+         for (int qx = 0; qx < Q1D[0]; ++qx)
+         {
+            const int q = qx + ((qy + (qz * Q1D[1])) * Q1D[0]);
+            const double O00 = qd(q,0);
+            const double O01 = qd(q,1);
+            const double O02 = qd(q,2);
+            const double O10 = symmetric ? O01 : qd(q,3);
+            const double O11 = symmetric ? qd(q,3) : qd(q,4);
+            const double O12 = symmetric ? qd(q,4) : qd(q,5);
+            const double O20 = symmetric ? O02 : qd(q,6);
+            const double O21 = symmetric ? O12 : qd(q,7);
+            const double O22 = symmetric ? qd(q,5) : qd(q,8);
+
+            const double grad0 = grad[0](qx,qy,qz);
+            const double grad1 = grad[1](qx,qy,qz);
+            const double grad2 = grad[2](qx,qy,qz);
+
+            grad[0](qx,qy,qz) = (O00*grad0)+(O01*grad1)+(O02*grad2);
+            grad[1](qx,qy,qz) = (O10*grad0)+(O11*grad1)+(O12*grad2);
+            grad[2](qx,qy,qz) = (O20*grad0)+(O21*grad1)+(O22*grad2);
+         } // qx
+      } // qy
+   } // qz
+
+   for (int qz = 0; qz < Q1D[2]; ++qz)
+   {
+      for (int dy = 0; dy < D1D[1]; ++dy)
+      {
+         for (int dx = 0; dx < D1D[0]; ++dx)
+         {
+            for (int d=0; d<3; ++d)
+            {
+               gradXY(d,dx,dy) = 0.0;
+            }
+         }
+      }
+      for (int qy = 0; qy < Q1D[1]; ++qy)
+      {
+         for (int dx = 0; dx < D1D[0]; ++dx)
+         {
+            for (int d=0; d<3; ++d)
+            {
+               gradX(d,dx) = 0.0;
+            }
+         }
+         for (int qx = 0; qx < Q1D[0]; ++qx)
+         {
+            const double gX = grad[0](qx,qy,qz);
+            const double gY = grad[1](qx,qy,qz);
+            const double gZ = grad[2](qx,qy,qz);
+            for (int dx = minQ[0][qx]; dx <= maxQ[0][qx]; ++dx)
+            {
+               const double wx  = B[0](qx,dx);
+               const double wDx = G[0](qx,dx);
+               gradX(0,dx) += gX * wDx;
+               gradX(1,dx) += gY * wx;
+               gradX(2,dx) += gZ * wx;
+            }
+         }
+         for (int dy = minQ[1][qy]; dy <= maxQ[1][qy]; ++dy)
+         {
+            const double wy  = B[1](qy,dy);
+            const double wDy = G[1](qy,dy);
+            for (int dx = 0; dx < D1D[0]; ++dx)
+            {
+               gradXY(0,dx,dy) += gradX(0,dx) * wy;
+               gradXY(1,dx,dy) += gradX(1,dx) * wDy;
+               gradXY(2,dx,dy) += gradX(2,dx) * wy;
+            }
+         }
+      }
+      for (int dz = minQ[2][qz]; dz <= maxQ[2][qz]; ++dz)
+      {
+         const double wz  = B[2](qz,dz);
+         const double wDz = G[2](qz,dz);
+         for (int dy = 0; dy < D1D[1]; ++dy)
+         {
+            for (int dx = 0; dx < D1D[0]; ++dx)
+            {
+               Y(dx,dy,dz) +=
+                  ((gradXY(0,dx,dy) * wz) +
+                   (gradXY(1,dx,dy) * wz) +
+                   (gradXY(2,dx,dy) * wDz));
+            }
+         }
+      } // dz
+   } // qz
+}
+
+void DiffusionIntegrator::AddMultNURBSPA(const Vector &x, Vector &y) const
+{
+   Vector xp, yp;
+
+   for (int p=0; p<numPatches; ++p)
+   {
+      Array<int> vdofs;
+      fespace->GetPatchVDofs(p, vdofs);
+
+      x.GetSubVector(vdofs, xp);
+      yp.SetSize(vdofs.Size());
+      yp = 0.0;
+
+      AddMultPatchPA(p, xp, yp);
+
+      y.AddElementVector(vdofs, yp);
+   }
+}
+
 } // namespace mfem

fem/intrules.cpp

@@ -16,6 +16,7 @@
 // Formulas at http://nines.cs.kuleuven.be/research/ecf/ecf.html
 
 #include "fem.hpp"
+#include "../mesh/nurbs.hpp"
 #include <cmath>
 
 #ifdef MFEM_USE_MPFR
@@ -173,6 +174,51 @@ void IntegrationRule::GrundmannMollerSimplexRule(int s, int n)
    }
 }
 
+IntegrationRule*
+IntegrationRule::ApplyToKnotIntervals(KnotVector const& kv) const
+{
+   const int np = this->GetNPoints();
+   const int ne = kv.GetNE();
+
+   IntegrationRule *kvir = new IntegrationRule(ne * np);
+
+   double x0 = kv[0];
+   double x1 = x0;
+
+   int id = 0;
+   for (int e=0; e<ne; ++e)
+   {
+      x0 = x1;
+
+      if (e == ne-1)
+      {
+         x1 = kv[kv.Size() - 1];
+      }
+      else
+      {
+         // Find the next unique knot
+         while (id < kv.Size() - 1)
+         {
+            id++;
+            if (kv[id] != x0)
+            {
+               x1 = kv[id];
+               break;
+            }
+         }
+      }
+
+      const double s = x1 - x0;
+
+      for (int j=0; j<this->GetNPoints(); ++j)
+      {
+         const double x = x0 + (s * (*this)[j].x);
+         (*kvir)[(e * np) + j].Set1w(x, (*this)[j].weight);
+      }
+   }
+
+   return kvir;
+}
 
 #ifdef MFEM_USE_MPFR
 
@@ -1725,4 +1771,235 @@ IntegrationRule *IntegrationRules::CubeIntegrationRule(int Order)
    return CubeIntRules[Order];
 }
 
+IntegrationRule& NURBSMeshRules::GetElementRule(const int elem,
+                                                const int patch, const int *ijk,
+                                                Array<const KnotVector*> const& kv,
+                                                bool & deleteRule) const
+{
+   deleteRule = false;
+
+   // First check whether a rule has been assigned to element index elem.
+   auto search = elementToRule.find(elem);
+   if (search != elementToRule.end())
+   {
+      return *elementRule[search->second];
+   }
+
+   MFEM_VERIFY(patchRules1D.NumRows(),
+               "Undefined rule in NURBSMeshRules::GetElementRule");
+
+   // Use a tensor product of rules on the patch.
+   MFEM_VERIFY(kv.Size() == dim, "");
+
+   int np = 1;
+   std::vector<std::vector<double>> el(dim);
+
+   std::vector<int> npd;
+   npd.assign(3, 0);
+
+   for (int d=0; d<dim; ++d)
+   {
+      const int order = kv[d]->GetOrder();
+
+      const double kv0 = (*kv[d])[order + ijk[d]];
+      const double kv1 = (*kv[d])[order + ijk[d] + 1];
+
+      const bool rightEnd = (order + ijk[d] + 1) == (kv[d]->Size() - 1);
+
+      for (int i=0; i<patchRules1D(patch,d)->Size(); ++i)
+      {
+         const IntegrationPoint& ip = (*patchRules1D(patch,d))[i];
+         if (kv0 <= ip.x && (ip.x < kv1 || rightEnd))
+         {
+            const double x = (ip.x - kv0) / (kv1 - kv0);
+            el[d].push_back(x);
+            el[d].push_back(ip.weight);
+         }
+      }
+
+      npd[d] = el[d].size() / 2;
+      np *= npd[d];
+   }
+
+   IntegrationRule *irp = new IntegrationRule(np);
+   deleteRule = true;
+
+   // Set (*irp)[i + j*npd[0] + k*npd[0]*npd[1]] =
+   //     (el[0][2*i], el[1][2*j], el[2][2*k])
+
+   MFEM_VERIFY(npd[0] > 0 && npd[1] > 0, "Assuming 2D or 3D");
+
+   for (int i = 0; i < npd[0]; ++i)
+   {
+      for (int j = 0; j < npd[1]; ++j)
+      {
+         for (int k = 0; k < std::max(npd[2], 1); ++k)
+         {
+            const int id = i + j*npd[0] + k*npd[0]*npd[1];
+            (*irp)[id].x = el[0][2*i];
+            (*irp)[id].y = el[1][2*j];
+
+            (*irp)[id].weight = el[0][(2*i)+1];
+            (*irp)[id].weight *= el[1][(2*j)+1];
+
+            if (npd[2] > 0)
+            {
+               (*irp)[id].z = el[2][2*k];
+               (*irp)[id].weight *= el[2][(2*k)+1];
+            }
+         }
+      }
+   }
+
+   return *irp;
+}
+
+void NURBSMeshRules::GetIntegrationPointFrom1D(const int patch, int i, int j,
+                                               int k, IntegrationPoint & ip)
+{
+   MFEM_VERIFY(patchRules1D.NumRows() > 0,
+               "Assuming patchRules1D is set.");
+
+   ip.weight = (*patchRules1D(patch,0))[i].weight;
+   ip.x = (*patchRules1D(patch,0))[i].x;
+
+   if (dim > 1)
+   {
+      ip.weight *= (*patchRules1D(patch,1))[j].weight;
+      ip.y = (*patchRules1D(patch,1))[j].x;  // 1D rule only has x
+   }
+
+   if (dim > 2)
+   {
+      ip.weight *= (*patchRules1D(patch,2))[k].weight;
+      ip.z = (*patchRules1D(patch,2))[k].x;  // 1D rule only has x
+   }
+}
+
+void NURBSMeshRules::Finalize(Mesh const& mesh)
+{
+   if ((int) pointToElem.size() == npatches) { return; }  // Already set
+
+   MFEM_VERIFY(elementToRule.empty() && patchRules1D.NumRows() > 0
+               && npatches > 0, "Assuming patchRules1D is set.");
+   MFEM_VERIFY(mesh.NURBSext, "");
+   MFEM_VERIFY(mesh.Dimension() == dim, "");
+
+   pointToElem.resize(npatches);
+   patchRules1D_KnotSpan.resize(npatches);
+
+   // First, find all the elements in each patch.
+   std::vector<std::vector<int>> patchElements(npatches);
+
+   for (int e=0; e<mesh.GetNE(); ++e)
+   {
+      patchElements[mesh.NURBSext->GetElementPatch(e)].push_back(e);
+   }
+
+   Array<int> ijk(3);
+   Array<int> maxijk(3);
+   Array<int> np(3);  // Number of points in each dimension
+   ijk = 0;
+
+   Array<const KnotVector*> pkv;
+
+   for (int p=0; p<npatches; ++p)
+   {
+      patchRules1D_KnotSpan[p].resize(dim);
+
+      // For each patch, get the range of ijk.
+      mesh.NURBSext->GetPatchKnotVectors(p, pkv);
+      MFEM_VERIFY((int) pkv.Size() == dim, "");
+
+      maxijk = 1;
+      np = 1;
+      for (int d=0; d<dim; ++d)
+      {
+         maxijk[d] = pkv[d]->GetNKS();
+         np[d] = patchRules1D(p,d)->Size();
+      }
+
+      // For each patch, set a map from ijk to element index.
+      Array3D<int> ijk2elem(maxijk[0], maxijk[1], maxijk[2]);
+      ijk2elem = -1;
+
+      for (auto elem : patchElements[p])
+      {
+         mesh.NURBSext->GetElementIJK(elem, ijk);
+         MFEM_VERIFY(ijk2elem(ijk[0], ijk[1], ijk[2]) == -1, "");
+         ijk2elem(ijk[0], ijk[1], ijk[2]) = elem;
+      }
+
+      // For each point, find its ijk and from that its element index.
+      // It is assumed here that the NURBSFiniteElement kv the same as the
+      // patch kv.
+
+      for (int d=0; d<dim; ++d)
+      {
+         patchRules1D_KnotSpan[p][d].SetSize(patchRules1D(p,d)->Size());
+
+         for (int r=0; r<patchRules1D(p,d)->Size(); ++r)
+         {
+            const IntegrationPoint& ip = (*patchRules1D(p,d))[r];
+
+            const int order = pkv[d]->GetOrder();
+
+            // Find ijk_d such that ip.x is in the corresponding knot-span.
+            int ijk_d = 0;
+            bool found = false;
+            while (!found)
+            {
+               const double kv0 = (*pkv[d])[order + ijk_d];
+               const double kv1 = (*pkv[d])[order + ijk_d + 1];
+
+               const bool rightEnd = (order + ijk_d + 1) == (pkv[d]->Size() - 1);
+
+               if (kv0 <= ip.x && (ip.x < kv1 || rightEnd))
+               {
+                  found = true;
+               }
+               else
+               {
+                  ijk_d++;
+               }
+            }
+
+            patchRules1D_KnotSpan[p][d][r] = ijk_d;
+         }
+      }
+
+      pointToElem[p].SetSize(np[0], np[1], np[2]);
+      for (int i=0; i<np[0]; ++i)
+         for (int j=0; j<np[1]; ++j)
+            for (int k=0; k<np[2]; ++k)
+            {
+               const int elem = ijk2elem(patchRules1D_KnotSpan[p][0][i],
+                                         patchRules1D_KnotSpan[p][1][j],
+                                         patchRules1D_KnotSpan[p][2][k]);
+               MFEM_VERIFY(elem >= 0, "");
+               pointToElem[p](i,j,k) = elem;
+            }
+   } // Loop (p) over patches
+}
+
+void NURBSMeshRules::SetPatchRules1D(const int patch,
+                                     std::vector<const IntegrationRule*> & ir1D)
+{
+   MFEM_VERIFY((int) ir1D.size() == dim, "Wrong dimension");
+
+   for (int i=0; i<dim; ++i)
+   {
+      patchRules1D(patch,i) = ir1D[i];
+   }
+}
+
+NURBSMeshRules::~NURBSMeshRules()
+{
+   for (int i=0; i<patchRules1D.NumRows(); ++i)
+      for (int j=0; j<patchRules1D.NumCols(); ++j)
+      {
+         delete patchRules1D(i, j);
+      }
+}
+
 }

fem/intrules.hpp

@@ -15,9 +15,15 @@
 #include "../config/config.hpp"
 #include "../general/array.hpp"
 
+#include <vector>
+#include <map>
+
 namespace mfem
 {
 
+class KnotVector;
+class Mesh;
+
 /* Classes for IntegrationPoint, IntegrationRule, and container class
    IntegrationRules.  Declares the global variable IntRules */
 
@@ -257,10 +263,105 @@ public:
        a call like this: `IntPoint(i).weight`. */
    const Array<double> &GetWeights() const;
 
+   /// @brief Return an integration rule for KnotVector @a kv, defined by
+   /// applying this rule on each knot interval.
+   IntegrationRule* ApplyToKnotIntervals(KnotVector const& kv) const;
+
    /// Destroys an IntegrationRule object
    ~IntegrationRule() { }
 };
 
+/// Class for defining different integration rules on each NURBS patch.
+class NURBSMeshRules
+{
+public:
+   /// Construct a rule for each patch, using SetPatchRules1D.
+   NURBSMeshRules(const int numPatches, const int dim_) :
+      patchRules1D(numPatches, dim_),
+      npatches(numPatches), dim(dim_) { }
+
+   /// Returns a rule for the element.
+   IntegrationRule &GetElementRule(const int elem, const int patch,
+                                   const int *ijk,
+                                   Array<const KnotVector*> const& kv,
+                                   bool & deleteRule) const;
+
+   /// Add a rule to be used for individual elements. Returns the rule index.
+   std::size_t AddElementRule(IntegrationRule *ir_element)
+   {
+      elementRule.push_back(ir_element);
+      return elementRule.size() - 1;
+   }
+
+   /// @brief Set the integration rule for the element of the given index. This
+   /// rule is used instead of the rule for the patch containing the element.
+   void SetElementRule(const std::size_t element,
+                       const std::size_t elementRuleIndex)
+   {
+      elementToRule[element] = elementRuleIndex;
+   }
+
+   /// @brief Set 1D integration rules to be used as a tensor product rule on
+   /// the patch with index @a patch. This class takes ownership of these rules.
+   void SetPatchRules1D(const int patch,
+                        std::vector<const IntegrationRule*> & ir1D);
+
+   /// @brief For tensor product rules defined on each patch by
+   /// SetPatchRules1D(), return a pointer to the 1D rule in the specified
+   /// @a dimension.
+   const IntegrationRule* GetPatchRule1D(const int patch,
+                                         const int dimension) const
+   {
+      return patchRules1D(patch, dimension);
+   }
+
+   /// @brief For tensor product rules defined on each patch by
+   /// SetPatchRules1D(), return the integration point with index (i,j,k).
+   void GetIntegrationPointFrom1D(const int patch, int i, int j, int k,
+                                  IntegrationPoint & ip);
+
+   /// @brief Finalize() must be called before this class can be used for
+   /// assembly. In particular, it defines data used by GetPointElement().
+   void Finalize(Mesh const& mesh);
+
+   /// @brief For tensor product rules defined on each patch by
+   /// SetPatchRules1D(), returns the index of the element containing
+   /// integration point (i,j,k) for patch index @a patch. Finalize() must be
+   /// called first.
+   int GetPointElement(int patch, int i, int j, int k) const
+   {
+      return pointToElem[patch](i,j,k);
+   }
+
+   int GetDim() const { return dim; }
+
+   /// @brief For tensor product rules defined on each patch by
+   /// SetPatchRules1D(), returns an array of knot span indices for each
+   /// integration point in the specified @a dimension.
+   const Array<int>& GetPatchRule1D_KnotSpan(const int patch,
+                                             const int dimension) const
+   {
+      return patchRules1D_KnotSpan[patch][dimension];
+   }
+
+   ~NURBSMeshRules();
+
+private:
+   /// Tensor-product rules defined on all patches independently.
+   Array2D<const IntegrationRule*> patchRules1D;
+
+   /// Integration rules defined on elements.
+   std::vector<IntegrationRule*> elementRule;
+
+   std::map<std::size_t, std::size_t> elementToRule;
+
+   std::vector<Array3D<int>> pointToElem;
+   std::vector<std::vector<Array<int>>> patchRules1D_KnotSpan;
+
+   const int npatches;
+   const int dim;
+};
+
 /// A Class that defines 1-D numerical quadrature rules on [0,1].
 class QuadratureFunctions1D
 {

fem/nonlininteg.hpp

@@ -26,9 +26,23 @@ namespace mfem
     assemble the local gradient operator and to compute the local energy. */
 class NonlinearFormIntegrator
 {
+public:
+   enum Mode
+   {
+      ELEMENTWISE = 0,       /**< Element-wise integration (default) */
+      PATCHWISE = 1,         /**< Patch-wise integration (NURBS meshes) */
+      PATCHWISE_REDUCED = 2, /**< Patch-wise integration (NURBS meshes) with
+                                  reduced integration rules. */
+   };
+
 protected:
    const IntegrationRule *IntRule;
 
+   Mode integrationMode = Mode::ELEMENTWISE;
+
+   // Prescribed integration rules (not reduced approximate rules).
+   NURBSMeshRules *patchRules = nullptr;
+
    // CEED extension
    ceed::Operator* ceedOp;
 
@@ -42,6 +56,14 @@ public:
        let the integrator choose (when @a ir == NULL). */
    virtual void SetIntRule(const IntegrationRule *ir) { IntRule = ir; }
 
+   void SetIntegrationMode(Mode m) { integrationMode = m; }
+
+   /// For patchwise integration, SetNURBSPatchIntRule must be called.
+   void SetNURBSPatchIntRule(NURBSMeshRules *pr) { patchRules = pr; }
+   bool HasNURBSPatchIntRule() const { return patchRules != nullptr; }
+
+   bool Patchwise() const { return integrationMode != Mode::ELEMENTWISE; }
+
    /// Prescribe a fixed IntegrationRule to use.
    void SetIntegrationRule(const IntegrationRule &ir) { SetIntRule(&ir); }
 

fem/pfespace.cpp

@@ -163,18 +163,18 @@ void ParFiniteElementSpace::Construct()
 
       // calculate number of ghost DOFs
       ngvdofs = pncmesh->GetNGhostVertices()
-                * fec->DofForGeometry(Geometry::POINT);
+                * fec->DofForGeometry(Geometry::Type::POINT);
 
       if (pmesh->Dimension() > 1)
       {
          ngedofs = pncmesh->GetNGhostEdges()
-                   * fec->DofForGeometry(Geometry::SEGMENT);
+                   * fec->DofForGeometry(Geometry::Type::SEGMENT);
       }
 
       if (pmesh->Dimension() > 2)
       {
-         int stride = fec->DofForGeometry(Geometry::SQUARE);
-         ngfdofs = pncmesh->GetNGhostFaces() * stride;
+         ngfdofs = pncmesh->GetNGhostFaces()
+                   * fec->DofForGeometry(Geometry::Type::SQUARE);
       }
 
       // total number of ghost DOFs. Ghost DOFs start at index 'ndofs', i.e.,
@@ -1842,9 +1842,13 @@ int ParFiniteElementSpace::PackDof(int entity, int index, int edof) const
 static int bisect(const int* array, int size, int value)
 {
    const int* end = array + size;
-   const int* pos = std::lower_bound(array, end, value);
-   MFEM_VERIFY(pos != end, "value not found");
-   return pos - array;
+   const int* pos = std::upper_bound(array, end, value);
+   MFEM_VERIFY(pos != array, "value not found");
+   if (pos == end)
+   {
+      MFEM_VERIFY(*(array+size - 1) == value, "Last entry must be exact")
+   }
+   return pos - array - 1;
 }
 
 /** Dissect a DOF number to obtain the entity type (0=vertex, 1=edge, 2=face),
@@ -1880,7 +1884,8 @@ void ParFiniteElementSpace::UnpackDof(int dof,
          else // mixed faces or var-order space
          {
             const Table &table = var_face_dofs;
-            MFEM_ASSERT(table.Size(), "");
+
+            MFEM_ASSERT(table.Size() > 0, "");
             int jpos = bisect(table.GetJ(), table.Size_of_connections(), dof);
             index = bisect(table.GetI(), table.Size(), jpos);
             edof = dof - table.GetRow(index)[0];
@@ -2010,7 +2015,6 @@ class NeighborRowMessage : public VarMessage<314>
 public:
    typedef NCMesh::MeshId MeshId;
    typedef ParNCMesh::GroupId GroupId;
-
    struct RowInfo
    {
       int entity, index, edof;
@@ -2022,8 +2026,6 @@ public:
 
       RowInfo(int ent, int idx, int edof, GroupId grp)
          : entity(ent), index(idx), edof(edof), group(grp) {}
-
-      typedef std::vector<RowInfo> List;
    };
 
    NeighborRowMessage() : pncmesh(NULL) {}
@@ -2034,7 +2036,7 @@ public:
       rows.push_back(RowInfo(entity, index, edof, group, row));
    }
 
-   const RowInfo::List& GetRows() const { return rows; }
+   const std::vector<RowInfo>& GetRows() const { return rows; }
 
    void SetNCMesh(ParNCMesh* pnc) { pncmesh = pnc; }
    void SetFEC(const FiniteElementCollection* fec_) { this->fec = fec_; }
@@ -2042,7 +2044,7 @@ public:
    typedef std::map<int, NeighborRowMessage> Map;
 
 protected:
-   RowInfo::List rows;
+   std::vector<RowInfo> rows;
 
    ParNCMesh *pncmesh;
    const FiniteElementCollection* fec;
@@ -2051,7 +2053,6 @@ protected:
    virtual void Decode(int);
 };
 
-
 void NeighborRowMessage::Encode(int rank)
 {
    std::ostringstream stream;
@@ -2161,11 +2162,21 @@ void NeighborRowMessage::Decode(int rank)
             ind = fec->DofOrderForOrientation(geom, fo);
          }
 
-         double s = 1.0;
+#ifdef MFEM_DEBUG_PMATRIX
+         mfem::out << "Rank " << pncmesh->MyRank << " receiving from " << rank
+                   << ": ent " << ent << ", index " << id.index
+                   << ", edof " << edof << " (id " << id.element << "/"
+                   << int(id.local) << ")" << std::endl;
+#endif
+
+         // If edof arrived with a negative index, flip it, and the scaling.
+         double s = (edof < 0) ? -1.0 : 1.0;
+         edof = (edof < 0) ? -1 - edof : edof;
+
          if (ind && (edof = ind[edof]) < 0)
          {
             edof = -1 - edof;
-            s = -1.0;
+            s *= -1.0;
          }
 
          rows.push_back(RowInfo(ent, id.index, edof, group_ids[gi++]));
@@ -2189,10 +2200,8 @@ ParFiniteElementSpace::ScheduleSendRow(const PMatrixRow &row, int dof,
    int ent, idx, edof;
    UnpackDof(dof, ent, idx, edof);
 
-   const ParNCMesh::CommGroup &group = pncmesh->GetGroup(group_id);
-   for (unsigned i = 0; i < group.size(); i++)
+   for (const auto &rank : pncmesh->GetGroup(group_id))
    {
-      int rank = group[i];
       if (rank != MyRank)
       {
          NeighborRowMessage &msg = send_msg[rank];
@@ -2312,7 +2321,7 @@ int ParFiniteElementSpace
                  && fec->GetContType() == FiniteElementCollection::TANGENTIAL),
                "Nedelec NC tets of order >= 2 are not supported yet.");
 
-   bool dg = (nvdofs == 0 && nedofs == 0 && nfdofs == 0);
+   const bool dg = (nvdofs == 0 && nedofs == 0 && nfdofs == 0);
 
 #ifdef MFEM_PMATRIX_STATS
    n_msgs_sent = n_msgs_recv = 0;
@@ -2321,7 +2330,7 @@ int ParFiniteElementSpace
 
    // *** STEP 1: build master-slave dependency lists ***
 
-   int total_dofs = ndofs + ngdofs;
+   const int total_dofs = ndofs + ngdofs;
    SparseMatrix deps(ndofs, total_dofs);
 
    if (!dg && !partial)
@@ -2332,16 +2341,14 @@ int ParFiniteElementSpace
       for (int entity = 0; entity <= 2; entity++)
       {
          const NCMesh::NCList &list = pncmesh->GetNCList(entity);
-         if (!list.masters.Size()) { continue; }
+         if (list.masters.Size() == 0) { continue; }
 
          IsoparametricTransformation T;
          DenseMatrix I;
 
          // process masters that we own or that affect our edges/faces
-         for (int mi = 0; mi < list.masters.Size(); mi++)
+         for (const auto &mf : list.masters)
          {
-            const NCMesh::Master &mf = list.masters[mi];
-
             // get master DOFs
             if (pncmesh->IsGhost(entity, mf.index))
             {
@@ -2352,10 +2359,10 @@ int ParFiniteElementSpace
                GetEntityDofs(entity, mf.index, master_dofs, mf.Geom());
             }
 
-            if (!master_dofs.Size()) { continue; }
+            if (master_dofs.Size() == 0) { continue; }
 
             const FiniteElement* fe = fec->FiniteElementForGeometry(mf.Geom());
-            if (!fe) { continue; }
+            if (fe == nullptr) { continue; }
 
             switch (mf.Geom())
             {
@@ -2371,7 +2378,7 @@ int ParFiniteElementSpace
                const NCMesh::Slave &sf = list.slaves[si];
                if (pncmesh->IsGhost(entity, sf.index)) { continue; }
 
-               const int variant = 0; // TODO parallel var-order
+               constexpr int variant = 0; // TODO parallel var-order
                GetEntityDofs(entity, sf.index, slave_dofs, mf.Geom(), variant);
                if (!slave_dofs.Size()) { continue; }
 
@@ -2398,37 +2405,37 @@ int ParFiniteElementSpace
    {
       Array<int> dofs;
 
-      // initialize dof_group[], dof_owner[]
-      for (int entity = 0; entity <= 2; entity++)
+      auto initialize_group_and_owner = [&dof_group, &dof_owner, &dofs,
+                                                     this](int entity, const MeshId &id)
       {
-         const NCMesh::NCList &list = pncmesh->GetNCList(entity);
+         if (id.index < 0) { return; }
 
-         int lsize[3] =
-         { list.conforming.Size(), list.masters.Size(), list.slaves.Size() };
+         GroupId owner = pncmesh->GetEntityOwnerId(entity, id.index);
+         GroupId group = pncmesh->GetEntityGroupId(entity, id.index);
 
-         for (int l = 0; l < 3; l++)
+         GetBareDofs(entity, id.index, dofs);
+
+         for (auto dof : dofs)
          {
-            for (int i = 0; i < lsize[l]; i++)
-            {
-               const MeshId &id =
-                  (l == 0) ? list.conforming[i] :
-                  (l == 1) ? (const MeshId&) list.masters[i]
-                  /*    */ : (const MeshId&) list.slaves[i];
+            dof_owner[dof] = owner;
+            dof_group[dof] = group;
+         }
+      };
 
-               if (id.index < 0) { continue; }
-
-               GroupId owner = pncmesh->GetEntityOwnerId(entity, id.index);
-               GroupId group = pncmesh->GetEntityGroupId(entity, id.index);
-
-               GetBareDofs(entity, id.index, dofs);
-
-               for (int j = 0; j < dofs.Size(); j++)
-               {
-                  int dof = dofs[j];
-                  dof_owner[dof] = owner;
-                  dof_group[dof] = group;
-               }
-            }
+      // initialize dof_group[], dof_owner[] in sequence
+      for (int entity : {0,1,2})
+      {
+         for (const auto &id : pncmesh->GetNCList(entity).conforming)
+         {
+            initialize_group_and_owner(entity, id);
+         }
+         for (const auto &id : pncmesh->GetNCList(entity).masters)
+         {
+            initialize_group_and_owner(entity, id);
+         }
+         for (const auto &id : pncmesh->GetNCList(entity).slaves)
+         {
+            initialize_group_and_owner(entity, id);
          }
       }
    }
@@ -2440,15 +2447,50 @@ int ParFiniteElementSpace
 
    // DOFs that stayed independent and are ours are true DOFs
    int num_true_dofs = 0;
-   for (int i = 0; i < ndofs; i++)
+   for (int i = 0; i < ndofs; ++i)
    {
       if (dof_owner[i] == 0 && deps.RowSize(i) == 0)
       {
-         num_true_dofs++;
+         ++num_true_dofs;
          finalized[i] = true;
       }
    }
 
+#ifdef MFEM_DEBUG_PMATRIX
+   // Helper for dumping diagnostics on one dof
+   auto dof_diagnostics = [&](int dof, bool print_diagnostic)
+   {
+      const auto &comm_group = pncmesh->GetGroup(dof_group[dof]);
+      std::stringstream msg;
+      msg << std::boolalpha;
+      msg << "R" << Mpi::WorldRank() << " dof " << dof
+          << " owner_rank " << pncmesh->GetGroup(dof_owner[dof])[0] << " CommGroup {";
+      for (const auto &x : comm_group)
+      {
+         msg << x << ' ';
+      }
+      msg << "} finalized " << finalized[dof];
+
+      Array<int> cols;
+      if (dof < ndofs)
+      {
+         Vector row;
+         deps.GetRow(dof, cols, row);
+         msg << " deps cols {";
+         for (const auto &x : cols)
+         {
+            msg << x << ' ';
+         }
+         msg << '}';
+      }
+
+      int entity, index, edof;
+      UnpackDof(dof, entity, index, edof);
+      msg << " entity " << entity << " index " << index << " edof " << edof;
+      return msg.str();
+   };
+#endif
+
    // calculate global offsets
    HYPRE_BigInt loc_sizes[2] = { ndofs*vdim, num_true_dofs*vdim };
    Array<HYPRE_BigInt>* offsets[2] = { &dof_offs, &tdof_offs };
@@ -2470,10 +2512,10 @@ int ParFiniteElementSpace
 
    std::vector<PMatrixRow> pmatrix(total_dofs);
 
-   bool bynodes = (ordering == Ordering::byNODES);
-   int vdim_factor = bynodes ? 1 : vdim;
-   int dof_stride = bynodes ? ndofs : 1;
-   int tdof_stride = bynodes ? num_true_dofs : 1;
+   const bool bynodes = (ordering == Ordering::byNODES);
+   const int vdim_factor = bynodes ? 1 : vdim;
+   const int dof_stride = bynodes ? ndofs : 1;
+   const int tdof_stride = bynodes ? num_true_dofs : 1;
 
    // big container for all messages we send (the list is for iterations)
    std::list<NeighborRowMessage::Map> send_msg;
@@ -2495,13 +2537,13 @@ int ParFiniteElementSpace
 
          for (int vd = 0; vd < vdim; vd++)
          {
-            int vdof = dof*vdim_factor + vd*dof_stride;
-            int vtdof = tdof*vdim_factor + vd*tdof_stride;
+            const int vdof = dof*vdim_factor + vd*dof_stride;
+            const int vtdof = tdof*vdim_factor + vd*tdof_stride;
 
             if (R_) { (*R_)->Add(vtdof, vdof, 1.0); }
             if (dof_tdof) { (*dof_tdof)[vdof] = vtdof; }
          }
-         tdof++;
+         ++tdof;
       }
    }
 
@@ -2542,14 +2584,12 @@ int ParFiniteElementSpace
          n_rows_recv += recv_msg.GetRows().size();
 #endif
 
-         const NeighborRowMessage::RowInfo::List &rows = recv_msg.GetRows();
-         for (unsigned i = 0; i < rows.size(); i++)
+         for (const auto &ri : recv_msg.GetRows())
          {
-            const NeighborRowMessage::RowInfo &ri = rows[i];
-            int dof = PackDof(ri.entity, ri.index, ri.edof);
+            const int dof = PackDof(ri.entity, ri.index, ri.edof);
             pmatrix[dof] = ri.row;
 
-            if (dof < ndofs && !finalized[dof]) { num_finalized++; }
+            if (dof < ndofs && !finalized[dof]) { ++num_finalized; }
             finalized[dof] = true;
 
             if (ri.group >= 0 && dof_group[dof] != ri.group)
@@ -2567,13 +2607,14 @@ int ParFiniteElementSpace
          done = true;
          for (int dof = 0; dof < ndofs; dof++)
          {
-            if (finalized[dof]) { continue; }
-
-            bool owned = (dof_owner[dof] == 0);
-            bool shared = (dof_group[dof] != 0);
-
-            if (owned && DofFinalizable(dof, finalized, deps))
+            const bool owned = (dof_owner[dof] == 0);
+            if (!finalized[dof]
+                && owned
+                && DofFinalizable(dof, finalized, deps))
             {
+               int ent, idx, edof;
+               UnpackDof(dof, ent, idx, edof);
+
                const int* dep_col = deps.GetRowColumns(dof);
                const double* dep_coef = deps.GetRowEntries(dof);
                int num_dep = deps.RowSize(dof);
@@ -2588,10 +2629,11 @@ int ParFiniteElementSpace
                pmatrix[dof] = buffer;
 
                finalized[dof] = true;
-               num_finalized++;
+               ++num_finalized;
                done = false;
 
                // send row to neighbors who need it
+               const bool shared = (dof_group[dof] != 0);
                if (shared)
                {
                   ScheduleSendRow(pmatrix[dof], dof, dof_group[dof],
@@ -2602,7 +2644,7 @@ int ParFiniteElementSpace
       }
 
 #ifdef MFEM_DEBUG_PMATRIX
-      /*static int dump = 0;
+      static int dump = 0;
       if (dump < 10)
       {
          char fname[100];
@@ -2610,7 +2652,7 @@ int ParFiniteElementSpace
          std::ofstream f(fname);
          DebugDumpDOFs(f, deps, dof_group, dof_owner, finalized);
          dump++;
-      }*/
+      }
 #endif
 
       // send current batch of messages
@@ -2635,10 +2677,9 @@ int ParFiniteElementSpace
    }
 
    // make sure we can discard all send buffers
-   for (std::list<NeighborRowMessage::Map>::iterator
-        it = send_msg.begin(); it != send_msg.end(); ++it)
+   for (auto &msg : send_msg)
    {
-      NeighborRowMessage::WaitAllSent(*it);
+      NeighborRowMessage::WaitAllSent(msg);
    }
 
 #ifdef MFEM_PMATRIX_STATS

fem/qfunction.cpp

@@ -115,7 +115,8 @@ std::ostream &operator<<(std::ostream &os, const QuadratureFunction &qf)
 }
 
 void QuadratureFunction::SaveVTU(std::ostream &os, VTKFormat format,
-                                 int compression_level) const
+                                 int compression_level,
+                                 const std::string &field_name) const
 {
    os << R"(<VTKFile type="UnstructuredGrid" version="0.1")";
    if (compression_level != 0)
@@ -129,11 +130,9 @@ void QuadratureFunction::SaveVTU(std::ostream &os, VTKFormat format,
    const char *type_str = (format != VTKFormat::BINARY32) ? "Float64" : "Float32";
    std::vector<char> buf;
 
-   Mesh &mesh = *qspace->GetMesh();
-
-   int np = qspace->GetSize();
-   int ne = mesh.GetNE();
-   int sdim = mesh.SpaceDimension();
+   const int np = qspace->GetSize();
+   const int ne = qspace->GetNE();
+   const int sdim = qspace->GetMesh()->SpaceDimension();
 
    // For quadrature functions, each point is a vertex cell, so number of cells
    // is equal to number of points
@@ -148,7 +147,7 @@ void QuadratureFunction::SaveVTU(std::ostream &os, VTKFormat format,
    Vector pt(sdim);
    for (int i = 0; i < ne; i++)
    {
-      ElementTransformation &T = *mesh.GetElementTransformation(i);
+      ElementTransformation &T = *qspace->GetTransformation(i);
       const IntegrationRule &ir = GetIntRule(i);
       for (int j = 0; j < ir.Size(); j++)
       {
@@ -205,8 +204,9 @@ void QuadratureFunction::SaveVTU(std::ostream &os, VTKFormat format,
    os << "</Cells>\n";
 
    os << "<PointData>\n";
-   os << "<DataArray type=\"" << type_str << "\" Name=\"u\" format=\""
-      << fmt_str << "\" NumberOfComponents=\"" << vdim << "\">\n";
+   os << "<DataArray type=\"" << type_str << "\" Name=\"" << field_name
+      << "\" format=\"" << fmt_str << "\" NumberOfComponents=\"" << vdim
+      << "\">\n";
    for (int i = 0; i < ne; i++)
    {
       DenseMatrix vals;
@@ -233,10 +233,11 @@ void QuadratureFunction::SaveVTU(std::ostream &os, VTKFormat format,
 }
 
 void QuadratureFunction::SaveVTU(const std::string &filename, VTKFormat format,
-                                 int compression_level) const
+                                 int compression_level,
+                                 const std::string &field_name) const
 {
    std::ofstream f(filename + ".vtu");
-   SaveVTU(f, format, compression_level);
+   SaveVTU(f, format, compression_level, field_name);
 }
 
 }

fem/qfunction.hpp

@@ -185,7 +185,7 @@ public:
    /// format is VTKFormat::ASCII. Otherwise, zlib compression will be used for
    /// binary data.
    void SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
-                int compression_level=0) const;
+                int compression_level=0, const std::string &field_name="u") const;
 
    /// @brief Save the QuadratureFunction to a VTU (ParaView) file.
    ///
@@ -193,7 +193,7 @@ public:
    /// @sa SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
    ///             int compression_level=0)
    void SaveVTU(const std::string &filename, VTKFormat format=VTKFormat::ASCII,
-                int compression_level=0) const;
+                int compression_level=0, const std::string &field_name="u") const;
 
    virtual ~QuadratureFunction()
    {

fem/transfer.cpp

@@ -546,6 +546,234 @@ L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
    const FiniteElementSpace& fes_ho_, const FiniteElementSpace& fes_lor_)
    : L2Projection(fes_ho_, fes_lor_)
 {
+   std::unique_ptr<SparseMatrix> R_mat, M_LH_mat;
+   std::tie(R_mat, M_LH_mat) = ComputeSparseRAndM_LH();
+
+   FiniteElementSpace fes_ho_scalar(fes_ho.GetMesh(), fes_ho.FEColl(), 1);
+   FiniteElementSpace fes_lor_scalar(fes_lor.GetMesh(), fes_lor.FEColl(), 1);
+
+   const SparseMatrix *P_ho = fes_ho_scalar.GetConformingProlongation();
+   const SparseMatrix *P_lor = fes_lor_scalar.GetConformingProlongation();
+
+   if (P_ho || P_lor)
+   {
+      if (P_ho && P_lor)
+      {
+         R_mat.reset(RAP(*P_lor, *R_mat, *P_ho));
+         M_LH_mat.reset(RAP(*P_lor, *M_LH_mat, *P_ho));
+      }
+      else if (P_ho)
+      {
+         R_mat.reset(mfem::Mult(*R_mat, *P_ho));
+         M_LH_mat.reset(mfem::Mult(*M_LH_mat, *P_ho));
+      }
+      else // P_lor != nullptr
+      {
+         R_mat.reset(mfem::Mult(*P_lor, *R_mat));
+         M_LH_mat.reset(mfem::Mult(*P_lor, *M_LH_mat));
+      }
+   }
+
+   SparseMatrix *RTxM_LH_mat = TransposeMult(*R_mat, *M_LH_mat);
+   precon.reset(new DSmoother(*RTxM_LH_mat));
+
+   // Set ownership
+   RTxM_LH.reset(RTxM_LH_mat);
+   R = std::move(R_mat);
+   M_LH = std::move(M_LH_mat);
+
+   SetupPCG();
+}
+
+#ifdef MFEM_USE_MPI
+
+L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
+   const ParFiniteElementSpace& pfes_ho, const ParFiniteElementSpace& pfes_lor)
+   : L2Projection(pfes_ho, pfes_lor),
+     pcg(pfes_ho.GetComm())
+{
+   std::tie(R, M_LH) = ComputeSparseRAndM_LH();
+
+   ParFiniteElementSpace pfes_ho_scalar(pfes_ho.GetParMesh(),
+                                        pfes_ho.FEColl(), 1);
+   ParFiniteElementSpace pfes_lor_scalar(pfes_lor.GetParMesh(),
+                                         pfes_lor.FEColl(), 1);
+
+   HypreParMatrix R_local = HypreParMatrix(pfes_ho.GetComm(),
+                                           pfes_lor_scalar.GlobalVSize(),
+                                           pfes_ho_scalar.GlobalVSize(),
+                                           pfes_lor_scalar.GetDofOffsets(),
+                                           pfes_ho_scalar.GetDofOffsets(),
+                                           static_cast<SparseMatrix*>(R.get()));
+   HypreParMatrix M_LH_local = HypreParMatrix(pfes_ho.GetComm(),
+                                              pfes_lor_scalar.GlobalVSize(),
+                                              pfes_ho_scalar.GlobalVSize(),
+                                              pfes_lor_scalar.GetDofOffsets(),
+                                              pfes_ho_scalar.GetDofOffsets(),
+                                              static_cast<SparseMatrix*>(M_LH.get()));
+
+   HypreParMatrix *R_mat = RAP(pfes_lor_scalar.Dof_TrueDof_Matrix(),
+                               &R_local, pfes_ho_scalar.Dof_TrueDof_Matrix());
+   HypreParMatrix *M_LH_mat = RAP(pfes_lor_scalar.Dof_TrueDof_Matrix(),
+                                  &M_LH_local, pfes_ho_scalar.Dof_TrueDof_Matrix());
+
+   std::unique_ptr<HypreParMatrix> R_T(R_mat->Transpose());
+   HypreParMatrix *RTxM_LH_mat = ParMult(R_T.get(), M_LH_mat, true);
+
+   HypreBoomerAMG *amg = new HypreBoomerAMG(*RTxM_LH_mat);
+   amg->SetPrintLevel(0);
+
+   R.reset(R_mat);
+   M_LH.reset(M_LH_mat);
+   RTxM_LH.reset(RTxM_LH_mat);
+   precon.reset(amg);
+
+   SetupPCG();
+   pcg.SetPreconditioner(*precon);
+   pcg.SetOperator(*RTxM_LH);
+}
+
+#endif
+
+void L2ProjectionGridTransfer::L2ProjectionH1Space::SetupPCG()
+{
+   // Basic PCG solver setup
+   pcg.SetPrintLevel(0);
+   // pcg.SetPrintLevel(IterativeSolver::PrintLevel().Summary());
+   pcg.SetMaxIter(1000);
+   // initial values for relative and absolute tolerance
+   pcg.SetRelTol(1e-13);
+   pcg.SetAbsTol(1e-13);
+   pcg.SetPreconditioner(*precon);
+   pcg.SetOperator(*RTxM_LH);
+}
+
+void L2ProjectionGridTransfer::L2ProjectionH1Space::Mult(
+   const Vector& x, Vector& y) const
+{
+   Vector X(fes_ho.GetTrueVSize());
+   Vector X_dim(R->Width());
+
+   Vector Y_dim(R->Height());
+   Vector Y(fes_lor.GetTrueVSize());
+
+   Array<int> vdofs_list;
+
+   GetTDofs(fes_ho, x, X);
+
+   for (int d = 0; d < fes_ho.GetVDim(); ++d)
+   {
+      TDofsListByVDim(fes_ho, d, vdofs_list);
+      X.GetSubVector(vdofs_list, X_dim);
+      R->Mult(X_dim, Y_dim);
+      TDofsListByVDim(fes_lor, d, vdofs_list);
+      Y.SetSubVector(vdofs_list, Y_dim);
+   }
+
+   SetFromTDofs(fes_lor, Y, y);
+}
+
+void L2ProjectionGridTransfer::L2ProjectionH1Space::MultTranspose(
+   const Vector& x, Vector& y) const
+{
+   Vector X(fes_lor.GetTrueVSize());
+   Vector X_dim(R->Height());
+
+   Vector Y_dim(R->Width());
+   Vector Y(fes_ho.GetTrueVSize());
+
+   Array<int> vdofs_list;
+
+   GetTDofsTranspose(fes_lor, x, X);
+
+   for (int d = 0; d < fes_ho.GetVDim(); ++d)
+   {
+      TDofsListByVDim(fes_lor, d, vdofs_list);
+      X.GetSubVector(vdofs_list, X_dim);
+      R->MultTranspose(X_dim, Y_dim);
+      TDofsListByVDim(fes_ho, d, vdofs_list);
+      Y.SetSubVector(vdofs_list, Y_dim);
+   }
+
+   SetFromTDofsTranspose(fes_ho, Y, y);
+}
+
+void L2ProjectionGridTransfer::L2ProjectionH1Space::Prolongate(
+   const Vector& x, Vector& y) const
+{
+   Vector X(fes_lor.GetTrueVSize());
+   Vector X_dim(M_LH->Height());
+   Vector Xbar(pcg.Width());
+
+   Vector Y_dim(pcg.Height());
+   Vector Y(fes_ho.GetTrueVSize());
+
+   Array<int> vdofs_list;
+
+   GetTDofs(fes_lor, x, X);
+
+   for (int d = 0; d < fes_ho.GetVDim(); ++d)
+   {
+      TDofsListByVDim(fes_lor, d, vdofs_list);
+      X.GetSubVector(vdofs_list, X_dim);
+      // Compute y = P x = (R^T M_LH)^(-1) M_LH^T X = (R^T M_LH)^(-1) Xbar
+      M_LH->MultTranspose(X_dim, Xbar);
+      Y_dim = 0.0;
+      pcg.Mult(Xbar, Y_dim);
+      TDofsListByVDim(fes_ho, d, vdofs_list);
+      Y.SetSubVector(vdofs_list, Y_dim);
+   }
+
+   SetFromTDofs(fes_ho, Y, y);
+}
+
+void L2ProjectionGridTransfer::L2ProjectionH1Space::ProlongateTranspose(
+   const Vector& x, Vector& y) const
+{
+   Vector X(fes_ho.GetTrueVSize());
+   Vector X_dim(pcg.Width());
+   Vector Xbar(pcg.Height());
+
+   Vector Y_dim(M_LH->Height());
+   Vector Y(fes_lor.GetTrueVSize());
+
+   Array<int> vdofs_list;
+
+   GetTDofsTranspose(fes_ho, x, X);
+
+   for (int d = 0; d < fes_ho.GetVDim(); ++d)
+   {
+      TDofsListByVDim(fes_ho, d, vdofs_list);
+      X.GetSubVector(vdofs_list, X_dim);
+      // Compute y = P^T x = M_LH (R^T M_LH)^(-1) X = M_LH Xbar
+      Xbar = 0.0;
+      pcg.Mult(X_dim, Xbar);
+      M_LH->Mult(Xbar, Y_dim);
+      TDofsListByVDim(fes_lor, d, vdofs_list);
+      Y.SetSubVector(vdofs_list, Y_dim);
+   }
+
+   SetFromTDofsTranspose(fes_lor, Y, y);
+}
+
+void L2ProjectionGridTransfer::L2ProjectionH1Space::SetRelTol(double p_rtol_)
+{
+   pcg.SetRelTol(p_rtol_);
+}
+
+void L2ProjectionGridTransfer::L2ProjectionH1Space::SetAbsTol(double p_atol_)
+{
+   pcg.SetAbsTol(p_atol_);
+}
+
+std::pair<
+std::unique_ptr<SparseMatrix>,
+std::unique_ptr<SparseMatrix>>
+                            L2ProjectionGridTransfer::L2ProjectionH1Space::ComputeSparseRAndM_LH()
+{
+   std::pair<std::unique_ptr<SparseMatrix>,
+       std::unique_ptr<SparseMatrix>> r_and_mlh;
+
    Mesh* mesh_ho = fes_ho.GetMesh();
    Mesh* mesh_lor = fes_lor.GetMesh();
    int nel_ho = mesh_ho->GetNE();
@@ -553,7 +781,7 @@ L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
    int ndof_lor = fes_lor.GetNDofs();
 
    // If the local mesh is empty, skip all computations
-   if (nel_ho == 0) { return; }
+   if (nel_ho == 0) { return {nullptr, nullptr}; }
 
    const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
 
@@ -611,18 +839,26 @@ L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
       }
    }
    // DOF by DOF inverse of non-zero entries
-   for (int i = 0; i < ndof_lor; ++i)
-   {
-      ML_inv[i] = 1.0 / ML_inv[i];
-   }
+   LumpedMassInverse(ML_inv);
 
    // Compute sparsity pattern for R = M_L^(-1) M_LH and allocate
-   AllocR();
+   r_and_mlh.first = AllocR();
    // Allocate M_LH (same sparsity pattern as R)
    // L refers to the low-order refined mesh (DOFs correspond to rows)
    // H refers to the higher-order mesh (DOFs correspond to columns)
-   M_LH = SparseMatrix(R.GetI(), R.GetJ(), NULL,
-                       R.Height(), R.Width(), false, true, true);
+   Memory<int> I(r_and_mlh.first->Height() + 1);
+   for (int icol = 0; icol < r_and_mlh.first->Height() + 1; ++icol)
+   {
+      I[icol] = r_and_mlh.first->GetI()[icol];
+   }
+   Memory<int> J(r_and_mlh.first->NumNonZeroElems());
+   for (int jcol = 0; jcol < r_and_mlh.first->NumNonZeroElems(); ++jcol)
+   {
+      J[jcol] = r_and_mlh.first->GetJ()[jcol];
+   }
+   r_and_mlh.second = std::unique_ptr<SparseMatrix>(new SparseMatrix(
+                                                       I, J, NULL,
+                                                       r_and_mlh.first->Height(), r_and_mlh.first->Width(), true, true, true));
 
    IntegrationPointTransformation ip_tr;
    IsoparametricTransformation& emb_tr = ip_tr.Transf;
@@ -667,131 +903,118 @@ L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
          }
          Array<int> dofs_ho(nedof_ho);
          fes_ho.GetElementDofs(iho, dofs_ho);
-         M_LH.AddSubMatrix(dofs_lor, dofs_ho, M_LH_el);
-         R.AddSubMatrix(dofs_lor, dofs_ho, R_el);
+         r_and_mlh.second->AddSubMatrix(dofs_lor, dofs_ho, M_LH_el);
+         r_and_mlh.first->AddSubMatrix(dofs_lor, dofs_ho, R_el);
       }
    }
 
-   // Create PCG solver
-   RTxM_LH = TransposeMult(R, M_LH);
-   pcg.SetPrintLevel(0);
-   pcg.SetMaxIter(1000);
-   // initial values for relative and absolute tolerance
-   SetRelTol(1e-13);
-   SetAbsTol(1e-13);
-   Ds = DSmoother(*RTxM_LH);
-   pcg.SetPreconditioner(Ds);
-   pcg.SetOperator(*RTxM_LH);
+   return r_and_mlh;
 }
 
-L2ProjectionGridTransfer::L2ProjectionH1Space::~L2ProjectionH1Space()
+void L2ProjectionGridTransfer::L2ProjectionH1Space::GetTDofs(
+   const FiniteElementSpace& fes, const Vector& x, Vector& X) const
 {
-   delete RTxM_LH;
-}
-
-void L2ProjectionGridTransfer::L2ProjectionH1Space::Mult(
-   const Vector& x, Vector& y) const
-{
-   int vdim = fes_ho.GetVDim();
-   const int ndof_ho = fes_ho.GetNDofs();
-   const int ndof_lor = fes_lor.GetNDofs();
-   Array<int> dofs_ho(ndof_ho);
-   Array<int> dofs_lor(ndof_lor);
-   Vector x_dim(ndof_ho);
-   Vector y_dim(ndof_lor);
-
-   for (int d = 0; d < vdim; ++d)
+   const Operator* res = fes.GetRestrictionOperator();
+   if (res)
    {
-      fes_ho.GetVDofs(d, dofs_ho);
-      fes_lor.GetVDofs(d, dofs_lor);
-      x.GetSubVector(dofs_ho, x_dim);
-      R.Mult(x_dim, y_dim);
-      y.SetSubVector(dofs_lor, y_dim);
+      res->Mult(x, X);
+   }
+   else
+   {
+      X = x;
    }
 }
 
-void L2ProjectionGridTransfer::L2ProjectionH1Space::MultTranspose(
-   const Vector& x, Vector& y) const
+void L2ProjectionGridTransfer::L2ProjectionH1Space::SetFromTDofs(
+   const FiniteElementSpace& fes, const Vector &X, Vector& x) const
 {
-   int vdim = fes_ho.GetVDim();
-   const int ndof_ho = fes_ho.GetNDofs();
-   const int ndof_lor = fes_lor.GetNDofs();
-   Array<int> dofs_ho(ndof_ho);
-   Array<int> dofs_lor(ndof_lor);
-   Vector x_dim(ndof_lor);
-   Vector y_dim(ndof_ho);
-
-   for (int d = 0; d < vdim; ++d)
+   const Operator* P = fes.GetProlongationMatrix();
+   if (P)
    {
-      fes_ho.GetVDofs(d, dofs_ho);
-      fes_lor.GetVDofs(d, dofs_lor);
-      x.GetSubVector(dofs_lor, x_dim);
-      R.MultTranspose(x_dim, y_dim);
-      y.SetSubVector(dofs_ho, y_dim);
+      P->Mult(X, x);
+   }
+   else
+   {
+      x = X;
    }
 }
 
-void L2ProjectionGridTransfer::L2ProjectionH1Space::Prolongate(
-   const Vector& x, Vector& y) const
+void L2ProjectionGridTransfer::L2ProjectionH1Space::GetTDofsTranspose(
+   const FiniteElementSpace& fes, const Vector& x, Vector& X) const
 {
-   int vdim = fes_ho.GetVDim();
-   const int ndof_ho = fes_ho.GetNDofs();
-   const int ndof_lor = fes_lor.GetNDofs();
-   Array<int> dofs_ho(ndof_ho);
-   Array<int> dofs_lor(ndof_lor);
-   Vector x_dim(ndof_lor);
-   Vector y_dim(ndof_ho);
-   Vector xbar(ndof_ho);
-
-   for (int d = 0; d < vdim; ++d)
+   const Operator* P = fes.GetProlongationMatrix();
+   if (P)
    {
-      fes_lor.GetVDofs(d, dofs_lor);
-      x.GetSubVector(dofs_lor, x_dim);
-      // Compute y = P x = (R^T M_LH)^(-1) M_LH^T x = (R^T M_LH)^(-1) xbar
-      M_LH.MultTranspose(x_dim, xbar);
-      y_dim = 0.0;
-      pcg.Mult(xbar, y_dim);
-      fes_ho.GetVDofs(d, dofs_ho);
-      y.SetSubVector(dofs_ho, y_dim);
+      P->MultTranspose(x, X);
+   }
+   else
+   {
+      X = x;
    }
 }
 
-void L2ProjectionGridTransfer::L2ProjectionH1Space::ProlongateTranspose(
-   const Vector& x, Vector& y) const
+void L2ProjectionGridTransfer::L2ProjectionH1Space::SetFromTDofsTranspose(
+   const FiniteElementSpace& fes, const Vector &X, Vector& x) const
 {
-   int vdim = fes_ho.GetVDim();
-   const int ndof_ho = fes_ho.GetNDofs();
-   const int ndof_lor = fes_lor.GetNDofs();
-   Array<int> dofs_ho(ndof_ho);
-   Array<int> dofs_lor(ndof_lor);
-   Vector x_dim(ndof_ho);
-   Vector y_dim(ndof_lor);
-   Vector xbar(ndof_ho);
-
-   for (int d = 0; d < vdim; ++d)
+   const Operator *R_op = fes.GetRestrictionOperator();
+   if (R_op)
    {
-      fes_ho.GetVDofs(d, dofs_ho);
-      x.GetSubVector(dofs_ho, x_dim);
-      // Compute y = P^T x = M_LH (R^T M_LH)^(-1) x = M_LH xbar
-      xbar = 0.0;
-      pcg.Mult(x_dim, xbar);
-      M_LH.Mult(xbar, y_dim);
-      fes_lor.GetVDofs(d, dofs_lor);
-      y.SetSubVector(dofs_lor, y_dim);
+      R_op->MultTranspose(X, x);
+   }
+   else
+   {
+      x = X;
    }
 }
 
-void L2ProjectionGridTransfer::L2ProjectionH1Space::SetRelTol(double p_rtol_)
+void L2ProjectionGridTransfer::L2ProjectionH1Space::TDofsListByVDim(
+   const FiniteElementSpace& fes, int vdim, Array<int>& vdofs_list) const
 {
-   pcg.SetRelTol(p_rtol_);
+   const SparseMatrix *R_mat = fes.GetRestrictionMatrix();
+   if (R_mat)
+   {
+      Array<int> x_vdofs_list(fes.GetNDofs());
+      Array<int> x_vdofs_marker(fes.GetVSize());
+      Array<int> X_vdofs_marker(fes.GetTrueVSize());
+      fes.GetVDofs(vdim, x_vdofs_list);
+      FiniteElementSpace::ListToMarker(x_vdofs_list, fes.GetVSize(), x_vdofs_marker);
+      R_mat->BooleanMult(x_vdofs_marker, X_vdofs_marker);
+      FiniteElementSpace::MarkerToList(X_vdofs_marker, vdofs_list);
+   }
+   else
+   {
+      vdofs_list.SetSize(fes.GetNDofs());
+      fes.GetVDofs(vdim, vdofs_list);
+   }
 }
 
-void L2ProjectionGridTransfer::L2ProjectionH1Space::SetAbsTol(double p_atol_)
+void L2ProjectionGridTransfer::L2ProjectionH1Space::LumpedMassInverse(
+   Vector& ML_inv) const
 {
-   pcg.SetAbsTol(p_atol_);
+   Vector ML_inv_full(fes_lor.GetVSize());
+   // set ML_inv on dofs for vdim = 0
+   Array<int> vdofs_list(fes_lor.GetNDofs());
+   fes_lor.GetVDofs(0, vdofs_list);
+   ML_inv_full.SetSubVector(vdofs_list, ML_inv);
+
+   Vector ML_inv_true(fes_lor.GetTrueVSize());
+   const Operator *P = fes_lor.GetProlongationMatrix();
+   if (P) { P->MultTranspose(ML_inv_full, ML_inv_true); }
+   else { ML_inv_true = ML_inv_full; }
+
+   for (int i = 0; i < ML_inv_true.Size(); ++i)
+   {
+      ML_inv_true[i] = 1.0 / ML_inv_true[i];
+   }
+
+   if (P) { P->Mult(ML_inv_true, ML_inv_full); }
+   else { ML_inv_full = ML_inv_true; }
+
+   ML_inv_full.GetSubVector(vdofs_list, ML_inv);
 }
 
-void L2ProjectionGridTransfer::L2ProjectionH1Space::AllocR()
+std::unique_ptr<SparseMatrix>
+L2ProjectionGridTransfer::L2ProjectionH1Space::AllocR()
 {
    const Table& elem_dof_ho = fes_ho.GetElementToDofTable();
    const Table& elem_dof_lor = fes_lor.GetElementToDofTable();
@@ -871,11 +1094,13 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::AllocR()
    dof_lor_dof_ho.SortRows();
    double* data = Memory<double>(dof_dofI[ndof_lor]);
 
-   R = SparseMatrix(dof_dofI, dof_dofJ, data, ndof_lor, ndof_ho,
-                    true, true, true);
-   R = 0.0;
+   std::unique_ptr<SparseMatrix> R_local(new SparseMatrix(
+                                            dof_dofI, dof_dofJ, data, ndof_lor, ndof_ho, true, true, true));
+   (*R_local) = 0.0;
 
    dof_lor_dof_ho.LoseData();
+
+   return R_local;
 }
 
 L2ProjectionGridTransfer::~L2ProjectionGridTransfer()
@@ -905,7 +1130,20 @@ void L2ProjectionGridTransfer::BuildF()
    if (!force_l2_space &&
        dom_fes.FEColl()->GetContType() == FiniteElementCollection::CONTINUOUS)
    {
-      F = new L2ProjectionH1Space(dom_fes, ran_fes);
+      if (!Parallel())
+      {
+         F = new L2ProjectionH1Space(dom_fes, ran_fes);
+      }
+      else
+      {
+#ifdef MFEM_USE_MPI
+         const mfem::ParFiniteElementSpace& dom_pfes =
+            static_cast<mfem::ParFiniteElementSpace&>(dom_fes);
+         const mfem::ParFiniteElementSpace& ran_pfes =
+            static_cast<mfem::ParFiniteElementSpace&>(ran_fes);
+         F = new L2ProjectionH1Space(dom_pfes, ran_pfes);
+#endif
+      }
    }
    else
    {

fem/transfer.hpp

@@ -180,9 +180,15 @@ protected:
    public:
       virtual void Prolongate(const Vector& x, Vector& y) const = 0;
       virtual void ProlongateTranspose(const Vector& x, Vector& y) const = 0;
-      /// Sets relative tolerance and absolute tolerance in preconditioned
-      /// conjugate gradient solver. Only used for H1 spaces.
+      /// @brief Sets relative tolerance in preconditioned conjugate gradient
+      /// solver.
+      ///
+      /// Only used for H1 spaces.
       virtual void SetRelTol(double p_rtol_) = 0;
+      /// @brief Sets absolute tolerance in preconditioned conjugate gradient
+      /// solver.
+      ///
+      /// Only used for H1 spaces.
       virtual void SetAbsTol(double p_atol_) = 0;
    protected:
       const FiniteElementSpace& fes_ho;
@@ -249,29 +255,22 @@ protected:
       /// conservative left-inverse prolongation operation. This functionality
       /// is also provided as an Operator by L2Prolongation.
       virtual void ProlongateTranspose(const Vector& x, Vector& y) const;
-      virtual void SetRelTol(double p_rtol_) {}
-      virtual void SetAbsTol(double p_atol_) {}
+      virtual void SetRelTol(double p_rtol_) { } ///< No-op.
+      virtual void SetAbsTol(double p_atol_) { } ///< No-op.
    };
 
-   /** Class for projection operator between a H1 high-order finite element
-       space on a coarse mesh, and a H1 low-order finite element space on a
-       refined mesh (LOR). */
+   /** Projection operator between a H1 high-order finite element space on a
+       coarse mesh, and a H1 low-order finite element space on a refined mesh
+       (LOR). */
    class L2ProjectionH1Space : public L2Projection
    {
-      // The restriction operator is represented as a SparseMatrix R. The
-      // prolongation operator is a dense matrix computed as the inverse of (R^T
-      // M_L R), and hence, is not stored.
-      SparseMatrix R;
-      // Used to compute P = (RTxM_LH)^(-1) M_LH^T
-      SparseMatrix M_LH;
-      SparseMatrix* RTxM_LH;
-      CGSolver pcg;
-      DSmoother Ds;
-
    public:
-      L2ProjectionH1Space(const FiniteElementSpace& fes_ho_,
-                          const FiniteElementSpace& fes_lor_);
-      virtual ~L2ProjectionH1Space();
+      L2ProjectionH1Space(const FiniteElementSpace &fes_ho_,
+                          const FiniteElementSpace &fes_lor_);
+#ifdef MFEM_USE_MPI
+      L2ProjectionH1Space(const ParFiniteElementSpace &pfes_ho_,
+                          const ParFiniteElementSpace &pfes_lor_);
+#endif
       /// Maps <tt>x</tt>, primal field coefficients defined on a coarse mesh
       /// with a higher order H1 finite element space, to <tt>y</tt>, primal
       /// field coefficients defined on a refined mesh with a low order H1
@@ -305,11 +304,51 @@ protected:
       virtual void ProlongateTranspose(const Vector& x, Vector& y) const;
       virtual void SetRelTol(double p_rtol_);
       virtual void SetAbsTol(double p_atol_);
-   private:
-      /// Computes sparsity pattern and initializes R matrix. Based on
-      /// BilinearForm::AllocMat() except maps between HO elements and LOR
-      /// elements.
-      void AllocR();
+   protected:
+      /// Sets up the PCG solver (sets parameters, operator, and preconditioner)
+      void SetupPCG();
+      /// Computes on-rank R and M_LH matrices.
+      std::pair<std::unique_ptr<SparseMatrix>,
+          std::unique_ptr<SparseMatrix>> ComputeSparseRAndM_LH();
+      /// @brief Recovers vector of tdofs given a vector of dofs and a finite
+      /// element space
+      void GetTDofs(const FiniteElementSpace& fes, const Vector& x, Vector& X) const;
+      /// Sets dof values given a vector of tdofs and a finite element space
+      void SetFromTDofs(const FiniteElementSpace& fes,
+                        const Vector& X,
+                        Vector& x) const;
+      /// @brief Recovers a vector of dual field coefficients on the tdofs given
+      /// a vector of dual coefficients and a finite element space
+      void GetTDofsTranspose(const FiniteElementSpace& fes,
+                             const Vector& x,
+                             Vector& X) const;
+      /// @brief Sets dual field coefficients given a vector of dual field
+      /// coefficients on the tdofs and a finite element space
+      void SetFromTDofsTranspose(const FiniteElementSpace& fes,
+                                 const Vector& X,
+                                 Vector& x) const;
+      /// @brief Fills the vdofs_list array with a list of vdofs for a given
+      /// vdim and a given finite element space
+      void TDofsListByVDim(const FiniteElementSpace& fes,
+                           int vdim,
+                           Array<int>& vdofs_list) const;
+      /// Returns the inverse of an on-rank lumped mass matrix
+      void LumpedMassInverse(Vector& ML_inv) const;
+      /// @brief Computes sparsity pattern and initializes R matrix.
+      ///
+      /// Based on BilinearForm::AllocMat(), except maps between coarse HO
+      /// elements and refined LOR elements.
+      std::unique_ptr<SparseMatrix> AllocR();
+
+      CGSolver pcg;
+      std::unique_ptr<Solver> precon;
+      // The restriction operator is represented as an Operator R. The
+      // prolongation operator is a dense matrix computed as the inverse of (R^T
+      // M_L R), and hence, is not stored.
+      std::unique_ptr<Operator> R;
+      // Used to compute P = (RT*M_LH)^(-1) M_LH^T
+      std::unique_ptr<Operator> M_LH;
+      std::unique_ptr<Operator> RTxM_LH;
    };
 
    /** Mass-conservative prolongation operator going in the opposite direction

general/array.hpp

@@ -465,6 +465,9 @@ public:
 
    inline const T &operator()(int i, int j, int k) const;
    inline       T &operator()(int i, int j, int k);
+
+   inline void operator=(const T &a)
+   { array1d = a; }
 };
 
 

general/optparser.cpp

@@ -207,6 +207,9 @@ void OptionsParser::Parse()
                case STRING:
                   *(const char **)(options[j].var_ptr) = argv[i++];
                   break;
+               case STD_STRING:
+                  *(std::string *)(options[j].var_ptr) = argv[i++];
+                  break;
                case ENABLE:
                   *(bool *)(options[j].var_ptr) = true;
                   option_check[j+1] = 1;  // Do not allow the DISABLE Option
@@ -284,6 +287,10 @@ void OptionsParser::WriteValue(const Option &opt, std::ostream &os)
          os << *(const char **)(opt.var_ptr);
          break;
 
+      case STD_STRING:
+         out << *(std::string *)(opt.var_ptr);
+         break;
+
       case ARRAY:
       {
          Array<int> &list = *(Array<int>*)(opt.var_ptr);
@@ -401,8 +408,9 @@ void OptionsParser::PrintHelp(ostream &os) const
    static const char *seprtr = ", ";
    static const char *descr_sep = "\n\t";
    static const char *line_sep = "";
-   static const char *types[] = { " <int>", " <double>", " <string>", "", "",
-                                  " '<int>...'", " '<double>...'"
+   static const char *types[] = { " <int>", " <double>", " <string>",
+                                  " <string>", "", "", " '<int>...'",
+                                  " '<double>...'"
                                 };
 
    os << indent << "-h" << seprtr << "--help" << descr_sep

general/optparser.hpp

@@ -31,7 +31,7 @@ class Vector;
 class OptionsParser
 {
 public:
-   enum OptionType { INT, DOUBLE, STRING, ENABLE, DISABLE, ARRAY, VECTOR };
+   enum OptionType { INT, DOUBLE, STRING, STD_STRING, ENABLE, DISABLE, ARRAY, VECTOR };
 
 private:
    struct Option
@@ -115,6 +115,15 @@ public:
                             required));
    }
 
+   /// Add a string (std::string) option and set 'var' to receive the value.
+   void AddOption(std::string *var, const char *short_name,
+                  const char *long_name, const char *description,
+                  bool required = false)
+   {
+      options.Append(Option(STD_STRING, var, short_name, long_name, description,
+                            required));
+   }
+
    /** Add an integer array (separated by spaces) option and set 'var' to
        receive the values. */
    void AddOption(Array<int> * var, const char *short_name,

linalg/densemat.cpp

@@ -4173,6 +4173,28 @@ DenseMatrixSVD::DenseMatrixSVD(int h, int w,
    Init();
 }
 
+DenseMatrixSVD::DenseMatrixSVD(DenseMatrix &M,
+                               char left_singular_vectors,
+                               char right_singular_vectors)
+{
+   m = M.Height();
+   n = M.Width();
+   jobu = left_singular_vectors;
+   jobvt = right_singular_vectors;
+   Init();
+}
+
+DenseMatrixSVD::DenseMatrixSVD(int h, int w,
+                               char left_singular_vectors,
+                               char right_singular_vectors)
+{
+   m = h;
+   n = w;
+   jobu = left_singular_vectors;
+   jobvt = right_singular_vectors;
+   Init();
+}
+
 void DenseMatrixSVD::Init()
 {
    sv.SetSize(min(m, n));
@@ -4195,12 +4217,22 @@ void DenseMatrixSVD::Eval(DenseMatrix &M)
 #endif
    double * datau = nullptr;
    double * datavt = nullptr;
-   if (jobu == 'S')
+   if (jobu == 'A')
+   {
+      U.SetSize(m,m);
+      datau = U.Data();
+   }
+   else if (jobu == 'S')
    {
       U.SetSize(m,min(m,n));
       datau = U.Data();
    }
-   if (jobvt == 'S')
+   if (jobvt == 'A')
+   {
+      Vt.SetSize(n,n);
+      datavt = Vt.Data();
+   }
+   else if (jobvt == 'S')
    {
       Vt.SetSize(min(m,n),n);
       datavt = Vt.Data();

linalg/densemat.hpp

@@ -939,7 +939,12 @@ public:
    ~DenseMatrixGeneralizedEigensystem();
 };
 
+/**
+ @brief Class for Singular Value Decomposition of a DenseMatrix
 
+ Singular Value Decomposition (SVD) of a DenseMatrix with the use of the DGESVD
+ driver from LAPACK.
+ */
 class DenseMatrixSVD
 {
    DenseMatrix Mc;
@@ -955,16 +960,129 @@ class DenseMatrixSVD
 
    void Init();
 public:
+
+   /**
+    @brief Constructor for the DenseMatrixSVD
+
+    Constructor for the DenseMatrixSVD with LAPACK. The parameters for the left
+    and right singular vectors can be choosen according to the parameters for
+    the LAPACK DGESVD.
+
+    @param [in] M matrix to set the size to n=M.Height(), m=M.Width()
+    @param [in] left_singular_vectors optional parameter to define if first
+    left singular vectors should be computed
+    @param [in] right_singular_vectors optional parameter to define if first
+    right singular vectors should be computed
+    */
+   MFEM_DEPRECATED DenseMatrixSVD(DenseMatrix &M,
+                                  bool left_singular_vectors=false,
+                                  bool right_singular_vectors=false);
+
+   /**
+    @brief Constructor for the DenseMatrixSVD
+
+    Constructor for the DenseMatrixSVD with LAPACK. The parameters for the left
+    and right singular
+    vectors can be choosen according to the parameters for the LAPACK DGESVD.
+
+    @param [in] h height of the matrix
+    @param [in] w width of the matrix
+    @param [in] left_singular_vectors optional parameter to define if first
+    left singular vectors should be computed
+    @param [in] right_singular_vectors optional parameter to define if first
+    right singular vectors should be computed
+    */
+   MFEM_DEPRECATED DenseMatrixSVD(int h, int w,
+                                  bool left_singular_vectors=false,
+                                  bool right_singular_vectors=false);
+
+   /**
+    @brief Constructor for the DenseMatrixSVD
+
+    Constructor for the DenseMatrixSVD with LAPACK. The parameters for the left
+    and right singular vectors can be choosen according to the parameters for
+    the LAPACK DGESVD.
+
+    @param [in] M matrix to set the size to n=M.Height(), m=M.Width()
+    @param [in] left_singular_vectors optional parameter to define which left
+    singular vectors should be computed
+    @param [in] right_singular_vectors optional parameter to define which right
+    singular vectors should be computed
+
+    Options for computation of singular vectors:
+
+    'A': All singular vectors are computed (default)
+
+    'S': The first min(n,m) singular vectors are computed
+
+    'N': No singular vectors are computed
+    */
    DenseMatrixSVD(DenseMatrix &M,
-                  bool left_singular_vectors=false,
-                  bool right_singlular_vectors=false);
+                  char left_singular_vectors='A',
+                  char right_singular_vectors='A');
+
+   /**
+    @brief Constructor for the DenseMatrixSVD
+
+    Constructor for the DenseMatrixSVD with LAPACK. The parameters for the left
+    and right singular vectors can be choosen according to the
+    parameters for the LAPACK DGESVD.
+
+    @param [in] h height of the matrix
+    @param [in] w width of the matrix
+    @param [in] left_singular_vectors optional parameter to define which left
+    singular vectors should be computed
+    @param [in] right_singular_vectors optional parameter to define which right
+    singular vectors should be computed
+
+    Options for computation of singular vectors:
+
+    'A': All singular vectors are computed (default)
+
+    'S': The first min(n,m) singular vectors are computed
+
+    'N': No singular vectors are computed
+    */
    DenseMatrixSVD(int h, int w,
-                  bool left_singular_vectors=false,
-                  bool right_singlular_vectors=false);
+                  char left_singular_vectors='A',
+                  char right_singular_vectors='A');
+
+   /**
+    @brief Evaluate the SVD
+
+    Call of the DGESVD driver from LAPACK for the DenseMatrix M. The singular
+    vectors are computed according to the setup in the call of the constructor.
+
+    @param [in] M DenseMatrix the SVD should be evaluated for
+    */
    void Eval(DenseMatrix &M);
+
+   /**
+    @brief Return singular values
+
+    @return sv Vector containing all singular values
+    */
    Vector &Singularvalues() { return sv; }
+
+   /**
+    @brief Return specific singular value
+
+    @return sv(i) i-th singular value
+    */
    double Singularvalue(int i) { return sv(i); }
+
+   /**
+    @brief Return left singular vectors
+
+    @return U DenseMatrix containing left singular vectors
+    */
    DenseMatrix &LeftSingularvectors() { return U; }
+
+   /**
+    @brief Return right singular vectors
+
+    @return Vt DenseMatrix containing right singular vectors
+    */
    DenseMatrix &RightSingularvectors() { return Vt; }
    ~DenseMatrixSVD();
 };

linalg/solvers.cpp

@@ -3544,4 +3544,829 @@ void AuxSpaceSmoother::Mult(const Vector &x, Vector &y, bool transpose) const
 }
 #endif // MFEM_USE_MPI
 
+#ifdef MFEM_USE_LAPACK
+// LAPACK routines for NNLSSolver
+extern "C" void
+dormqr_(char *, char *, int *, int *, int *, double *, int*, double *,
+        double *, int *, double *, int*, int*);
+
+extern "C" void
+dgeqrf_(int *, int *, double *, int *, double *, double *, int *, int *);
+
+extern "C" void
+dgemv_(char *, int *, int *, double *, double *, int *, double *, int *,
+       double *, double *, int *);
+
+extern "C" void
+dtrsm_(char *side, char *uplo, char *transa, char *diag, int *m, int *n,
+       double *alpha, double *a, int *lda, double *b, int *ldb);
+
+NNLSSolver::NNLSSolver()
+   : Solver(0), mat(nullptr), const_tol_(1.0e-14), min_nnz_(0),
+     max_nnz_(0), verbosity_(0), res_change_termination_tol_(1.0e-4),
+     zero_tol_(1.0e-14), rhs_delta_(1.0e-11), n_outer_(100000),
+     n_inner_(100000), nStallCheck_(100), normalize_(true),
+     NNLS_qrres_on_(false), qr_residual_mode_(QRresidualMode::hybrid)
+{}
+
+void NNLSSolver::SetOperator(const Operator &op)
+{
+   mat = dynamic_cast<const DenseMatrix*>(&op);
+   MFEM_VERIFY(mat, "NNLSSolver operator must be of type DenseMatrix");
+
+   // The size of this operator is that of the transpose of op.
+   height = op.Width();
+   width = op.Height();
+
+   row_scaling_.SetSize(mat->NumRows());
+   row_scaling_ = 1.0;
+}
+
+void NNLSSolver::SetQRResidualMode(const QRresidualMode qr_residual_mode)
+{
+   qr_residual_mode_ = qr_residual_mode;
+   if (qr_residual_mode_ == QRresidualMode::on)
+   {
+      NNLS_qrres_on_ = true;
+   }
+}
+
+void NNLSSolver::NormalizeConstraints(Vector& rhs_lb, Vector& rhs_ub) const
+{
+   // Scale everything so that rescaled half gap is the same for all constraints
+   const int m = mat->NumRows();
+
+   MFEM_VERIFY(rhs_lb.Size() == m && rhs_ub.Size() == m, "");
+
+   Vector rhs_avg = rhs_ub;
+   rhs_avg += rhs_lb;
+   rhs_avg *= 0.5;
+
+   Vector rhs_halfgap = rhs_ub;
+   rhs_halfgap -= rhs_lb;
+   rhs_halfgap *= 0.5;
+
+   Vector rhs_avg_glob = rhs_avg;
+   Vector rhs_halfgap_glob = rhs_halfgap;
+   Vector halfgap_target(m);
+   halfgap_target = 1.0e3 * const_tol_;
+
+   row_scaling_.SetSize(m);
+
+   for (int i=0; i<m; ++i)
+   {
+      const double s = halfgap_target(i) / rhs_halfgap_glob(i);
+      row_scaling_[i] = s;
+
+      rhs_lb(i) = (rhs_avg(i) * s) - halfgap_target(i);
+      rhs_ub(i) = (rhs_avg(i) * s) + halfgap_target(i);
+   }
+}
+
+void NNLSSolver::Mult(const Vector &w, Vector &sol) const
+{
+   MFEM_VERIFY(mat, "NNLSSolver operator must be of type DenseMatrix");
+   Vector rhs_ub(mat->NumRows());
+   mat->Mult(w, rhs_ub);
+   rhs_ub *= row_scaling_;
+
+   Vector rhs_lb(rhs_ub);
+   Vector rhs_Gw(rhs_ub);
+
+   for (int i=0; i<rhs_ub.Size(); ++i)
+   {
+      rhs_lb(i) -= rhs_delta_;
+      rhs_ub(i) += rhs_delta_;
+   }
+
+   if (normalize_) { NormalizeConstraints(rhs_lb, rhs_ub); }
+   Solve(rhs_lb, rhs_ub, sol);
+
+   if (verbosity_ > 1)
+   {
+      int nnz = 0;
+      for (int i=0; i<sol.Size(); ++i)
+      {
+         if (sol(i) != 0.0)
+         {
+            nnz++;
+         }
+      }
+
+      mfem::out << "Number of nonzeros in NNLSSolver solution: " << nnz
+                << ", out of " << sol.Size() << endl;
+
+      // Check residual of NNLS solution
+      Vector res(mat->NumRows());
+      mat->Mult(sol, res);
+      res *= row_scaling_;
+
+      const double normGsol = res.Norml2();
+      const double normRHS = rhs_Gw.Norml2();
+
+      res -= rhs_Gw;
+      const double relNorm = res.Norml2() / std::max(normGsol, normRHS);
+      mfem::out << "Relative residual norm for NNLSSolver solution of Gs = Gw: "
+                << relNorm << endl;
+   }
+}
+
+void NNLSSolver::Solve(const Vector& rhs_lb, const Vector& rhs_ub,
+                       Vector& soln) const
+{
+   int m = mat->NumRows();
+   int n = mat->NumCols();
+
+   MFEM_VERIFY(rhs_lb.Size() == m && rhs_lb.Size() == m && soln.Size() == n, "");
+   MFEM_VERIFY(n >= m, "NNLSSolver system cannot be over-determined.");
+
+   if (max_nnz_ == 0)
+   {
+      max_nnz_ = mat->NumCols();
+   }
+
+   // Prepare right hand side
+   Vector rhs_avg(rhs_ub);
+   rhs_avg += rhs_lb;
+   rhs_avg *= 0.5;
+
+   Vector rhs_halfgap(rhs_ub);
+   rhs_halfgap -= rhs_lb;
+   rhs_halfgap *= 0.5;
+
+   Vector rhs_avg_glob(rhs_avg);
+   Vector rhs_halfgap_glob(rhs_halfgap);
+
+   int ione = 1;
+   double fone = 1.0;
+
+   char lside = 'L';
+   char trans = 'T';
+   char notrans = 'N';
+
+   std::vector<unsigned int> nz_ind(m);
+   Vector res_glob(m);
+   Vector mu(n);
+   Vector mu2(n);
+   int n_nz_ind = 0;
+   int n_glob = 0;
+   int m_update;
+   int min_nnz_cap = std::min(static_cast<int>(min_nnz_), std::min(m,n));
+   int info;
+   std::vector<double> l2_res_hist;
+   std::vector<unsigned int> stalled_indices;
+   int stalledFlag = 0;
+   int num_stalled = 0;
+   int nz_ind_zero = 0;
+
+   Vector soln_nz_glob(m);
+   Vector soln_nz_glob_up(m);
+
+   // The following matrices are stored in column-major format as Vectors
+   Vector mat_0_data(m * n);
+   Vector mat_qr_data(m * n);
+   Vector submat_data(m * n);
+
+   Vector tau(n);
+   Vector sub_tau = tau;
+   Vector vec1(m);
+
+   // Temporary work arrays
+   int lwork;
+   std::vector<double> work;
+   int n_outer_iter = 0;
+   int n_total_inner_iter = 0;
+   int i_qr_start;
+   int n_update;
+   // 0 = converged; 1 = maximum iterations reached;
+   // 2 = NNLS stalled (no change in residual for many iterations)
+   int exit_flag = 1;
+
+   res_glob = rhs_avg_glob;
+   Vector qt_rhs_glob = rhs_avg_glob;
+   Vector qqt_rhs_glob = qt_rhs_glob;
+   Vector sub_qt = rhs_avg_glob;
+
+   // Compute threshold tolerance for the Lagrange multiplier mu
+   double mu_tol = 0.0;
+
+   {
+      Vector rhs_scaled(rhs_halfgap_glob);
+      Vector tmp(n);
+      rhs_scaled *= row_scaling_;
+      mat->MultTranspose(rhs_scaled, tmp);
+
+      mu_tol = 1.0e-15 * tmp.Max();
+   }
+
+   double rmax = 0.0;
+   double mumax = 0.0;
+
+   for (int oiter = 0; oiter < n_outer_; ++oiter)
+   {
+      stalledFlag = 0;
+
+      rmax = fabs(res_glob(0)) - rhs_halfgap_glob(0);
+      for (int i=1; i<m; ++i)
+      {
+         rmax = std::max(rmax, fabs(res_glob(i)) - rhs_halfgap_glob(i));
+      }
+
+      l2_res_hist.push_back(res_glob.Norml2());
+
+      if (verbosity_ > 1)
+      {
+         mfem::out << "NNLS " << oiter << " " << n_total_inner_iter << " " << m
+                   << " " << n << " " << n_glob << " " << rmax << " "
+                   << l2_res_hist[oiter] << endl;
+      }
+      if (rmax <= const_tol_ && n_glob >= min_nnz_cap)
+      {
+         if (verbosity_ > 1)
+         {
+            mfem::out << "NNLS target tolerance met" << endl;
+         }
+         exit_flag = 0;
+         break;
+      }
+
+      if (n_glob >= max_nnz_)
+      {
+         if (verbosity_ > 1)
+         {
+            mfem::out << "NNLS target nnz met" << endl;
+         }
+         exit_flag = 0;
+         break;
+      }
+
+      if (n_glob >= m)
+      {
+         if (verbosity_ > 1)
+         {
+            mfem::out << "NNLS system is square... exiting" << endl;
+         }
+         exit_flag = 3;
+         break;
+      }
+
+      // Check for stall after the first nStallCheck iterations
+      if (oiter > nStallCheck_)
+      {
+         double mean0 = 0.0;
+         double mean1 = 0.0;
+         for (int i=0; i<nStallCheck_/2; ++i)
+         {
+            mean0 += l2_res_hist[oiter - i];
+            mean1 += l2_res_hist[oiter - (nStallCheck_) - i];
+         }
+
+         double mean_res_change = (mean1 / mean0) - 1.0;
+         if (std::abs(mean_res_change) < res_change_termination_tol_)
+         {
+            if (verbosity_ > 1)
+            {
+               mfem::out << "NNLSSolver stall detected... exiting" << endl;
+            }
+            exit_flag = 2;
+            break;
+         }
+      }
+
+      // Find the next index
+      res_glob *= row_scaling_;
+      mat->MultTranspose(res_glob, mu);
+
+      for (int i = 0; i < n_nz_ind; ++i)
+      {
+         mu(nz_ind[i]) = 0.0;
+      }
+      for (unsigned int i = 0; i < stalled_indices.size(); ++i)
+      {
+         mu(stalled_indices[i]) = 0.0;
+      }
+
+      mumax = mu.Max();
+
+      if (mumax < mu_tol)
+      {
+         num_stalled = stalled_indices.size();
+         if (num_stalled > 0)
+         {
+            if (verbosity_ > 0)
+            {
+               mfem::out << "NNLS Lagrange multiplier is below the minimum "
+                         << "threshold: mumax = " << mumax << ", mutol = "
+                         << mu_tol << "\n" << " Resetting stalled indices "
+                         << "vector of size " << num_stalled << "\n";
+            }
+            stalled_indices.resize(0);
+
+            mat->MultTranspose(res_glob, mu);
+
+            for (int i = 0; i < n_nz_ind; ++i)
+            {
+               mu(nz_ind[i]) = 0.0;
+            }
+
+            mumax = mu.Max();
+         }
+      }
+
+      int imax = 0;
+      {
+         double tmax = mu(0);
+         for (int i=1; i<n; ++i)
+         {
+            if (mu(i) > tmax)
+            {
+               tmax = mu(i);
+               imax = i;
+            }
+         }
+      }
+
+      // Record the local value of the next index
+      nz_ind[n_nz_ind] = imax;
+      ++n_nz_ind;
+
+      if (verbosity_ > 2)
+      {
+         mfem::out << "Found next index: " << imax << " " << mumax << endl;
+      }
+
+      for (int i=0; i<m; ++i)
+      {
+         mat_0_data(i + (n_glob*m)) = (*mat)(i,imax) * row_scaling_[i];
+         mat_qr_data(i + (n_glob*m)) = mat_0_data(i + (n_glob*m));
+      }
+
+      i_qr_start = n_glob;
+      ++n_glob; // Increment the size of the global matrix
+
+      if (verbosity_ > 2)
+      {
+         mfem::out << "Updated matrix with new index" << endl;
+      }
+
+      for (int iiter = 0; iiter < n_inner_; ++iiter)
+      {
+         ++n_total_inner_iter;
+
+         // Initialize
+         const bool incremental_update = true;
+         n_update = n_glob - i_qr_start;
+         m_update = m - i_qr_start;
+         if (incremental_update)
+         {
+            // Apply Householder reflectors to compute Q^T new_cols
+            lwork = -1;
+            work.resize(10);
+
+            dormqr_(&lside, &trans, &m, &n_update, &i_qr_start,
+                    mat_qr_data.GetData(), &m, tau.GetData(),
+                    mat_qr_data.GetData() + (i_qr_start * m), &m,
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // Q^T A update work calculation failed
+            lwork = static_cast<int>(work[0]);
+            work.resize(lwork);
+            dormqr_(&lside, &trans, &m, &n_update, &i_qr_start,
+                    mat_qr_data.GetData(), &m, tau.GetData(),
+                    mat_qr_data.GetData() + (i_qr_start * m), &m,
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // Q^T A update failed
+            // Compute QR factorization of the submatrix
+            lwork = -1;
+            work.resize(10);
+
+            // Copy m_update-by-n_update submatrix of mat_qr_data,
+            // starting at (i_qr_start, i_qr_start)
+            for (int i=0; i<m_update; ++i)
+               for (int j=0; j<n_update; ++j)
+               {
+                  submat_data[i + (j * m_update)] =
+                     mat_qr_data[i + i_qr_start + ((j + i_qr_start) * m)];
+               }
+
+            // Copy tau subvector of length n_update, starting at i_qr_start
+            for (int j=0; j<n_update; ++j)
+            {
+               sub_tau[j] = tau[i_qr_start + j];
+            }
+
+            dgeqrf_(&m_update, &n_update,
+                    submat_data.GetData(), &m_update, sub_tau.GetData(),
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // QR update factorization work calc
+            lwork = static_cast<int>(work[0]);
+            if (lwork == 0) { lwork = 1; }
+            work.resize(lwork);
+            dgeqrf_(&m_update, &n_update,
+                    submat_data.GetData(), &m_update, sub_tau.GetData(),
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // QR update factorization failed
+
+            // Copy result back
+            for (int i=0; i<m_update; ++i)
+               for (int j=0; j<n_update; ++j)
+               {
+                  mat_qr_data[i + i_qr_start + ((j + i_qr_start)* m)] =
+                     submat_data[i + (j * m_update)];
+               }
+
+            for (int j=0; j<n_update; ++j)
+            {
+               tau[i_qr_start + j] = sub_tau[j];
+            }
+         }
+         else
+         {
+            // Copy everything to mat_qr then do full QR
+            for (int i=0; i<m; ++i)
+               for (int j=0; j<n_glob; ++j)
+               {
+                  mat_qr_data(i + (j*m)) = mat_0_data(i + (j*m));
+               }
+
+            // Compute qr factorization (first find the size of work and then
+            // perform qr)
+            lwork = -1;
+            work.resize(10);
+            dgeqrf_(&m, &n_glob,
+                    mat_qr_data.GetData(), &m, tau.GetData(),
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // QR factorization work calculation
+            lwork = static_cast<int>(work[0]);
+            work.resize(lwork);
+            dgeqrf_(&m, &n_glob,
+                    mat_qr_data.GetData(), &m, tau.GetData(),
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // QR factorization failed
+         }
+
+         if (verbosity_ > 2)
+         {
+            mfem::out << "Updated QR " << iiter << endl;
+         }
+
+         // Apply Householder reflectors to compute Q^T b
+         if (incremental_update && iiter == 0)
+         {
+            lwork = -1;
+            work.resize(10);
+
+            // Copy submatrix of mat_qr_data starting at
+            //   (i_qr_start, i_qr_start), of size m_update-by-1
+            // Copy submatrix of qt_rhs_glob starting at (i_qr_start, 0),
+            //   of size m_update-by-1
+
+            for (int i=0; i<m_update; ++i)
+            {
+               submat_data[i] = mat_qr_data[i + i_qr_start + (i_qr_start * m)];
+               sub_qt[i] = qt_rhs_glob[i + i_qr_start];
+            }
+
+            sub_tau[0] = tau[i_qr_start];
+
+            dormqr_(&lside, &trans, &m_update, &ione, &ione,
+                    submat_data.GetData(), &m_update, sub_tau.GetData(),
+                    sub_qt.GetData(), &m_update,
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // H_last y work calculation failed
+            lwork = static_cast<int>(work[0]);
+            work.resize(lwork);
+            dormqr_(&lside, &trans, &m_update, &ione, &ione,
+                    submat_data.GetData(), &m_update, sub_tau.GetData(),
+                    sub_qt.GetData(), &m_update,
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // H_last y failed
+            // Copy result back
+            for (int i=0; i<m_update; ++i)
+            {
+               qt_rhs_glob[i + i_qr_start] = sub_qt[i];
+            }
+         }
+         else
+         {
+            // Compute Q^T b from scratch
+            qt_rhs_glob = rhs_avg_glob;
+            lwork = -1;
+            work.resize(10);
+            dormqr_(&lside, &trans, &m, &ione, &n_glob,
+                    mat_qr_data.GetData(), &m, tau.GetData(),
+                    qt_rhs_glob.GetData(), &m,
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // Q^T b work calculation failed
+            lwork = static_cast<int>(work[0]);
+            work.resize(lwork);
+            dormqr_(&lside, &trans, &m, &ione, &n_glob,
+                    mat_qr_data.GetData(), &m, tau.GetData(),
+                    qt_rhs_glob.GetData(), &m,
+                    work.data(), &lwork, &info);
+            MFEM_VERIFY(info == 0, ""); // Q^T b failed
+         }
+
+         if (verbosity_ > 2)
+         {
+            mfem::out << "Updated rhs " << iiter << endl;
+         }
+
+         // Apply R^{-1}; first n_glob entries of vec1 are overwritten
+         char upper = 'U';
+         char nounit = 'N';
+         vec1 = qt_rhs_glob;
+         dtrsm_(&lside, &upper, &notrans, &nounit,
+                &n_glob, &ione, &fone,
+                mat_qr_data.GetData(), &m,
+                vec1.GetData(), &n_glob);
+
+         if (verbosity_ > 2)
+         {
+            mfem::out << "Solved triangular system " << iiter << endl;
+         }
+
+         // Check if all entries are positive
+         int pos_ibool = 0;
+         double smin = n_glob > 0 ? vec1(0) : 0.0;
+         for (int i=0; i<n_glob; ++i)
+         {
+            soln_nz_glob_up(i) = vec1(i);
+            smin = std::min(smin, soln_nz_glob_up(i));
+         }
+
+         if (smin > zero_tol_)
+         {
+            pos_ibool = 1;
+            for (int i=0; i<n_glob; ++i)
+            {
+               soln_nz_glob(i) = soln_nz_glob_up(i);
+            }
+         }
+
+         if (pos_ibool == 1)
+         {
+            break;
+         }
+
+         if (verbosity_ > 2)
+         {
+            mfem::out << "Start pruning " << iiter << endl;
+            for (int i = 0; i < n_glob; ++i)
+            {
+               if (soln_nz_glob_up(i) <= zero_tol_)
+               {
+                  mfem::out << i << " " << n_glob << " " << soln_nz_glob_up(i) << endl;
+               }
+            }
+         }
+
+         if (soln_nz_glob_up(n_glob - 1) <= zero_tol_)
+         {
+            stalledFlag = 1;
+            if (verbosity_ > 2)
+            {
+               if (qr_residual_mode_ == QRresidualMode::hybrid)
+               {
+                  mfem::out << "Detected stall due to adding and removing same "
+                            << "column. Switching to QR residual calculation "
+                            << "method." << endl;
+               }
+               else
+               {
+                  mfem::out << "Detected stall due to adding and removing same"
+                            << " column. Exiting now." << endl;
+               }
+            }
+         }
+
+         if (stalledFlag == 1 && qr_residual_mode_ == QRresidualMode::hybrid)
+         {
+            NNLS_qrres_on_ = true;
+            break;
+         }
+
+         double alpha = 1.0e300;
+         // Find maximum permissible step
+         for (int i = 0; i < n_glob; ++i)
+         {
+            if (soln_nz_glob_up(i) <= zero_tol_)
+            {
+               alpha = std::min(alpha, soln_nz_glob(i)/(soln_nz_glob(i) - soln_nz_glob_up(i)));
+            }
+         }
+         // Update solution
+         smin = 0.0;
+         for (int i = 0; i < n_glob; ++i)
+         {
+            soln_nz_glob(i) += alpha*(soln_nz_glob_up(i) - soln_nz_glob(i));
+            if (i == 0 || soln_nz_glob(i) < smin)
+            {
+               smin = soln_nz_glob(i);
+            }
+         }
+
+         while (smin > zero_tol_)
+         {
+            // This means there was a rounding error, as we should have
+            // a zero element by definition. Recalculate alpha based on
+            // the index that corresponds to the element that should be
+            // zero.
+
+            int index_min = 0;
+            smin = soln_nz_glob(0);
+            for (int i = 1; i < n_glob; ++i)
+            {
+               if (soln_nz_glob(i) < smin)
+               {
+                  smin = soln_nz_glob(i);
+                  index_min = i;
+               }
+            }
+
+            alpha = soln_nz_glob(index_min)/(soln_nz_glob(index_min)
+                                             - soln_nz_glob_up(index_min));
+
+            // Reupdate solution
+            for (int i = 0; i < n_glob; ++i)
+            {
+               soln_nz_glob(i) += alpha*(soln_nz_glob_up(i) - soln_nz_glob(i));
+            }
+         }
+
+         // Clean up zeroed entry
+         i_qr_start = n_glob+1;
+         while (true)
+         {
+            // Check if there is a zero entry
+            int zero_ibool;
+
+            smin = n_glob > 0 ? soln_nz_glob(0) : 0.0;
+            for (int i=1; i<n_glob; ++i)
+            {
+               smin = std::min(smin, soln_nz_glob(i));
+            }
+
+            if (smin < zero_tol_)
+            {
+               zero_ibool = 1;
+            }
+            else
+            {
+               zero_ibool = 0;
+            }
+
+            if (zero_ibool == 0)   // Break if there is no more zero entry
+            {
+               break;
+            }
+
+            int ind_zero = -1; // Index where the first zero is encountered
+            nz_ind_zero = 0;
+
+            // Identify global index of the zeroed element
+            for (int i = 0; i < n_glob; ++i)
+            {
+               if (soln_nz_glob(i) < zero_tol_)
+               {
+                  ind_zero = i;
+                  break;
+               }
+            }
+            MFEM_VERIFY(ind_zero != -1, "");
+            // Identify the local index for nz_ind to which the zeroed entry
+            // belongs
+            for (int i = 0; i < ind_zero; ++i)
+            {
+               ++nz_ind_zero;
+            }
+
+            {
+               // Copy mat_0.cols[ind_zero+1,n_glob) to mat_qr.cols[ind_zero,n_glob-1)
+               for (int i=0; i<m; ++i)
+                  for (int j=ind_zero; j<n_glob-1; ++j)
+                  {
+                     mat_qr_data(i + (j*m)) = mat_0_data(i + ((j+1)*m));
+                  }
+
+               // Copy mat_qr.cols[ind_zero,n_glob-1) to
+               // mat_0.cols[ind_zero,n_glob-1)
+               for (int i=0; i<m; ++i)
+                  for (int j=ind_zero; j<n_glob-1; ++j)
+                  {
+                     mat_0_data(i + (j*m)) = mat_qr_data(i + (j*m));
+                  }
+            }
+
+            // Remove the zeroed entry from the local matrix index
+            for (int i = nz_ind_zero; i < n_nz_ind-1; ++i)
+            {
+               nz_ind[i] = nz_ind[i+1];
+            }
+            --n_nz_ind;
+
+            // Shift soln_nz_glob and proc_index
+            for (int i = ind_zero; i < n_glob-1; ++i)
+            {
+               soln_nz_glob(i) = soln_nz_glob(i+1);
+            }
+
+            i_qr_start = std::min(i_qr_start, ind_zero);
+            --n_glob;
+         } // End of pruning loop
+
+         if (verbosity_ > 2)
+         {
+            mfem::out << "Finished pruning " << iiter << endl;
+         }
+      } // End of inner loop
+
+      // Check if we have stalled
+      if (stalledFlag == 1)
+      {
+         --n_glob;
+         --n_nz_ind;
+         num_stalled = stalled_indices.size();
+         stalled_indices.resize(num_stalled + 1);
+         stalled_indices[num_stalled] = imax;
+         if (verbosity_ > 2)
+         {
+            mfem::out << "Adding index " << imax << " to stalled index list "
+                      << "of size " << num_stalled << endl;
+         }
+      }
+
+      // Compute residual
+      if (!NNLS_qrres_on_)
+      {
+         res_glob = rhs_avg_glob;
+         double fmone = -1.0;
+         dgemv_(&notrans, &m, &n_glob, &fmone,
+                mat_0_data.GetData(), &m,
+                soln_nz_glob.GetData(), &ione, &fone,
+                res_glob.GetData(), &ione);
+      }
+      else
+      {
+         // Compute residual using res = b - Q*Q^T*b, where Q is from an
+         // economical QR decomposition
+         lwork = -1;
+         work.resize(10);
+         qqt_rhs_glob = 0.0;
+         for (int i=0; i<n_glob; ++i)
+         {
+            qqt_rhs_glob(i) = qt_rhs_glob(i);
+         }
+
+         dormqr_(&lside, &notrans, &m, &ione, &n_glob, mat_qr_data.GetData(), &m,
+                 tau.GetData(), qqt_rhs_glob.GetData(), &m,
+                 work.data(), &lwork, &info);
+
+         MFEM_VERIFY(info == 0, ""); // Q Q^T b work calculation failed.
+         lwork = static_cast<int>(work[0]);
+         work.resize(lwork);
+         dormqr_(&lside, &notrans, &m, &ione, &n_glob, mat_qr_data.GetData(), &m,
+                 tau.GetData(), qqt_rhs_glob.GetData(), &m,
+                 work.data(), &lwork, &info);
+         MFEM_VERIFY(info == 0, ""); // Q Q^T b calculation failed.
+         res_glob = rhs_avg_glob;
+         res_glob -= qqt_rhs_glob;
+      }
+
+      if (verbosity_ > 2)
+      {
+         mfem::out << "Computed residual" << endl;
+      }
+
+      ++n_outer_iter;
+   } // End of outer loop
+
+   // Insert the solutions
+   MFEM_VERIFY(n_glob == n_nz_ind, "");
+   soln = 0.0;
+   for (int i = 0; i < n_glob; ++i)
+   {
+      soln(nz_ind[i]) = soln_nz_glob(i);
+   }
+
+   if (verbosity_ > 0)
+   {
+      mfem::out << "NNLS solver: m = " << m << ", n = " << n
+                << ", outer_iter = " << n_outer_iter << ", inner_iter = "
+                << n_total_inner_iter;
+
+      if (exit_flag == 0)
+      {
+         mfem::out << ": converged" << endl;
+      }
+      else
+      {
+         mfem::out << endl << "Warning, NNLS convergence stalled: "
+                   << (exit_flag == 2) << endl;
+         mfem::out << "resErr = " << rmax << " vs tol = " << const_tol_
+                   << "; mumax = " << mumax << " vs tol = " << mu_tol << endl;
+      }
+   }
+}
+#endif // MFEM_USE_LAPACK
+
 }

linalg/solvers.hpp

@@ -432,7 +432,7 @@ public:
 
    ~OperatorChebyshevSmoother() {}
 
-   void Mult(const Vector&x, Vector &y) const;
+   void Mult(const Vector &x, Vector &y) const;
 
    void MultTranspose(const Vector &x, Vector &y) const { Mult(x, y); }
 
@@ -1245,6 +1245,126 @@ public:
 };
 #endif // MFEM_USE_MPI
 
+#ifdef MFEM_USE_LAPACK
+/** Non-negative least squares (NNLS) solver class, for computing a vector
+    with non-negative entries approximately satisfying an under-determined
+    linear system. */
+class NNLSSolver : public Solver
+{
+public:
+   NNLSSolver();
+
+   ~NNLSSolver() { }
+
+   /// The operator must be a DenseMatrix.
+   void SetOperator(const Operator &op) override;
+
+   void Mult(const Vector &w, Vector &sol) const override;
+
+   /**
+     * Set verbosity. If set to 0: print nothing; if 1: just print results;
+     * if 2: print short update on every iteration; if 3: print longer update
+     * each iteration.
+     */
+   void SetVerbosity(int v) { verbosity_ = v; }
+
+   void SetTolerance(double tol) { const_tol_ = tol; }
+
+   /// Set the minimum number of nonzeros required for the solution.
+   void SetMinNNZ(int min_nnz) { min_nnz_ = min_nnz; }
+
+   /// Set the maximum number of nonzeros required for the solution, as an early
+   /// termination condition.
+   void SetMaxNNZ(int max_nnz) { max_nnz_ = max_nnz; }
+
+   /// Set threshold on relative change in residual over nStallCheck_ iterations.
+   void SetResidualChangeTolerance(double tol)
+   { res_change_termination_tol_ = tol; }
+
+   void SetZeroTolerance(double tol) { zero_tol_ = tol; }
+
+   /// Set RHS vector constant shift, defining rhs_lb and rhs_ub in Solve().
+   void SetRHSDelta(double d) { rhs_delta_ = d; }
+
+   /// Set the maximum number of outer iterations in Solve().
+   void SetOuterIterations(int n) { n_outer_ = n; }
+
+   /// Set the maximum number of inner iterations in Solve().
+   void SetInnerIterations(int n) { n_inner_ = n; }
+
+   /// Set the number of iterations to use for stall checking.
+   void SetStallCheck(int n) { nStallCheck_ = n; }
+
+   /// Set a flag to determine whether to call NormalizeConstraints().
+   void SetNormalize(bool n) { normalize_ = n; }
+
+   /**
+     * Enumerated types of QRresidual mode. Options are 'off': the residual is
+     * calculated normally, 'on': the residual is calculated using the QR
+     * method, 'hybrid': the residual is calculated normally until we experience
+     * rounding errors, then the QR method is used. The default is 'hybrid',
+     * which should see the best performance. Recommend using 'hybrid' or 'off'
+     * only, since 'on' is computationally expensive.
+     */
+   enum class QRresidualMode {off, on, hybrid};
+
+   /**
+    * Set the residual calculation mode for the NNLS solver. See QRresidualMode
+    * enum above for details.
+    */
+   void SetQRResidualMode(const QRresidualMode qr_residual_mode);
+
+   /**
+    * @brief Solve the NNLS problem. Specifically, we find a vector @a soln,
+    * such that rhs_lb < mat*soln < rhs_ub is satisfied, where mat is the
+    * DenseMatrix input to SetOperator().
+    *
+    * The method by which we find the solution is the active-set method
+    * developed by Lawson and Hanson (1974) using lapack. To decrease rounding
+    * errors in the case of very tight tolerances, we have the option to compute
+    * the residual using the QR factorization of A, by res = b - Q*Q^T*b. This
+    * residual calculation results in less rounding error, but is more
+    * computationally expensive. To select whether to use the QR residual method
+    * or not, see set_qrresidual_mode above.
+    */
+   void Solve(const Vector& rhs_lb, const Vector& rhs_ub, Vector& soln) const;
+
+   /**
+     * Normalize the constraints such that the tolerances for each constraint
+     * (i.e. (UB - LB)/2) are equal. This seems to help the performance in most
+     * cases.
+     */
+   void NormalizeConstraints(Vector& rhs_lb, Vector& rhs_ub) const;
+
+private:
+   const DenseMatrix *mat;
+
+   double const_tol_;
+   int min_nnz_; // minimum number of nonzero entries
+   mutable int max_nnz_; // maximum number of nonzero entries
+   int verbosity_;
+
+   /**
+    * @brief Threshold on relative change in residual over nStallCheck_
+    * iterations, for stall sensing.
+    */
+   double res_change_termination_tol_;
+
+   double zero_tol_;
+   double rhs_delta_;
+   int n_outer_;
+   int n_inner_;
+   int nStallCheck_;
+
+   bool normalize_;
+
+   mutable bool NNLS_qrres_on_;
+   QRresidualMode qr_residual_mode_;
+
+   mutable Vector row_scaling_;
+};
+#endif // MFEM_USE_LAPACK
+
 }
 
 #endif // MFEM_SOLVERS

mesh/mesh.cpp

@@ -1444,7 +1444,7 @@ Element::Type Mesh::GetFaceElementType(int Face) const
 
 Array<int> Mesh::GetFaceToBdrElMap() const
 {
-   Array<int> face_to_be(NumOfFaces);
+   Array<int> face_to_be(GetNumFaces());
    face_to_be = -1;
    for (int i = 0; i < NumOfBdrElements; i++)
    {
@@ -3752,8 +3752,8 @@ Mesh& Mesh::operator=(Mesh &&mesh)
    return *this;
 }
 
-Mesh Mesh::LoadFromFile(const char *filename, int generate_edges, int refine,
-                        bool fix_orientation)
+Mesh Mesh::LoadFromFile(const std::string &filename, int generate_edges,
+                        int refine, bool fix_orientation)
 {
    Mesh mesh;
    named_ifgzstream imesh(filename);
@@ -3807,7 +3807,7 @@ Mesh Mesh::MakeRefined(Mesh &orig_mesh, const Array<int> &ref_factors,
    return mesh;
 }
 
-Mesh::Mesh(const char *filename, int generate_edges, int refine,
+Mesh::Mesh(const std::string &filename, int generate_edges, int refine,
            bool fix_orientation)
 {
    // Initialization as in the default constructor
@@ -5107,6 +5107,43 @@ std::vector<int> Mesh::CreatePeriodicVertexMapping(
    return v2v;
 }
 
+void Mesh::RefineNURBSFromFile(std::string ref_file)
+{
+   MFEM_VERIFY(NURBSext,"Mesh::RefineNURBSFromFile: Not a NURBS mesh!");
+   mfem::out<<"Refining NURBS from refinement file: "<<ref_file<<endl;
+
+   int nkv;
+   ifstream input(ref_file);
+   input >> nkv;
+
+   // Check if the number of knot vectors in the refinement file and mesh match
+   if ( nkv != NURBSext->GetNKV())
+   {
+      mfem::out<<endl;
+      mfem::out<<"Knot vectors in ref_file: "<<nkv<<endl;
+      mfem::out<<"Knot vectors in NURBSExt: "<<NURBSext->GetNKV()<<endl;
+      MFEM_ABORT("Refine file does not have the correct number of knot vectors");
+   }
+
+   // Read knot vectors from file
+   Array<Vector *> knotVec(nkv);
+   for (int kv = 0; kv < nkv; kv++)
+   {
+      knotVec[kv] = new Vector();
+      knotVec[kv]-> Load(input);
+   }
+   input.close();
+
+   // Insert knots
+   KnotInsert(knotVec);
+
+   // Delete knots
+   for (int kv = 0; kv < nkv; kv++)
+   {
+      delete knotVec[kv];
+   }
+}
+
 void Mesh::KnotInsert(Array<KnotVector *> &kv)
 {
    if (NURBSext == NULL)
@@ -5279,17 +5316,24 @@ void Mesh::LoadPatchTopo(std::istream &input, Array<int> &edge_to_knot)
 
    input >> ident; // 'edges'
    input >> NumOfEdges;
-   edge_vertex = new Table(NumOfEdges, 2);
-   edge_to_knot.SetSize(NumOfEdges);
-   for (int j = 0; j < NumOfEdges; j++)
+   if (NumOfEdges > 0)
    {
-      int *v = edge_vertex->GetRow(j);
-      input >> edge_to_knot[j] >> v[0] >> v[1];
-      if (v[0] > v[1])
+      edge_vertex = new Table(NumOfEdges, 2);
+      edge_to_knot.SetSize(NumOfEdges);
+      for (int j = 0; j < NumOfEdges; j++)
       {
-         edge_to_knot[j] = -1 - edge_to_knot[j];
+         int *v = edge_vertex->GetRow(j);
+         input >> edge_to_knot[j] >> v[0] >> v[1];
+         if (v[0] > v[1])
+         {
+            edge_to_knot[j] = -1 - edge_to_knot[j];
+         }
       }
    }
+   else
+   {
+      edge_to_knot.SetSize(0);
+   }
 
    skip_comment_lines(input, '#');
 
@@ -5299,6 +5343,198 @@ void Mesh::LoadPatchTopo(std::istream &input, Array<int> &edge_to_knot)
 
    FinalizeTopology();
    CheckBdrElementOrientation(); // check and fix boundary element orientation
+
+   /* Generate knot 2 edge mapping -- if edges are not specified in the mesh file
+      See data/two-squares-nurbs-autoedge.mesh for an example */
+   if (edge_to_knot.Size() == 0)
+   {
+      edge_vertex = new Table(NumOfEdges, 2);
+      edge_to_knot.SetSize(NumOfEdges);
+      constexpr int notset = -9999999;
+      edge_to_knot = notset;
+      Array<int> edges;
+      Array<int> oedge;
+      int knot = 0;
+
+      Array<int> edge0, edge1;
+      int flip = 1;
+      if (Dimension() == 2 )
+      {
+         edge0.SetSize(2);
+         edge1.SetSize(2);
+
+         edge0[0] = 0; edge1[0] = 2;
+         edge0[1] = 1; edge1[1] = 3;
+         flip = 1;
+      }
+      else if (Dimension() == 3 )
+      {
+         edge0.SetSize(9);
+         edge1.SetSize(9);
+
+         edge0[0] = 0; edge1[0] = 2;
+         edge0[1] = 0; edge1[1] = 4;
+         edge0[2] = 0; edge1[2] = 6;
+
+         edge0[3] = 1; edge1[3] = 3;
+         edge0[4] = 1; edge1[4] = 5;
+         edge0[5] = 1; edge1[5] = 7;
+
+         edge0[6] = 8; edge1[6] = 9;
+         edge0[7] = 8; edge1[7] = 10;
+         edge0[8] = 8; edge1[8] = 11;
+         flip = -1;
+      }
+
+      /* Initial assignment of knots to edges. This is an algorithm that loops over the
+         patches and assigns knot vectors to edges. It starts with assigning knot vector 0
+         and 1 to the edges of the first patch. Then it uses: 1) patches can share edges
+         2) knot vectors on opposing edges in a patch are equal, to create edge_to_knot */
+      int e0, e1, v0, v1, df;
+      int p,j,k;
+      for (p = 0; p < GetNE(); p++)
+      {
+         GetElementEdges(p, edges, oedge);
+
+         const int *v = elements[p]->GetVertices();
+         for (j = 0; j < edges.Size(); j++)
+         {
+            int *vv = edge_vertex->GetRow(edges[j]);
+            const int *e = elements[p]->GetEdgeVertices(j);
+            if (oedge[j] == 1)
+            {
+               vv[0] = v[e[0]];
+               vv[1] = v[e[1]];
+            }
+            else
+            {
+               vv[0] = v[e[1]];
+               vv[1] = v[e[0]];
+            }
+         }
+
+         for (j = 0; j < edge1.Size(); j++)
+         {
+            e0 = edges[edge0[j]];
+            e1 = edges[edge1[j]];
+            v0 = edge_to_knot[e0];
+            v1 = edge_to_knot[e1];
+            df = flip*oedge[edge0[j]]*oedge[edge1[j]];
+
+            // Case 1: knot vector is not set
+            if ((v0 == notset) && (v1 == notset))
+            {
+               edge_to_knot[e0] = knot;
+               edge_to_knot[e1] = knot;
+               knot++;
+            }
+            // Case 2 & 3: knot vector on one of the two edges
+            // is set earlier (in another patch). We just have
+            // to copy it for the opposing edge.
+            else if ((v0 != notset) && (v1 == notset))
+            {
+               edge_to_knot[e1] = (df >= 0 ? -v0-1 : v0);
+            }
+            else if ((v0 == notset) && (v1 != notset))
+            {
+               edge_to_knot[e0] = (df >= 0 ? -v1-1 : v1);
+            }
+         }
+      }
+
+      /* Verify correct assignment, make sure that corresponding edges
+         within patch point to same knot vector. If not assign the lowest number.
+
+         We bound the while by GetNE() + 1 as this is probably the most unlucky
+         case. +1 to finish without corrections. Note that this is a check and
+         in general the initial assignment is correct. Then the while is performed
+         only once. Only on very tricky meshes it might need corrections.*/
+      int corrections;
+      int passes = 0;
+      do
+      {
+         corrections = 0;
+         for (p = 0; p < GetNE(); p++)
+         {
+            GetElementEdges(p, edges, oedge);
+            for (j = 0; j < edge1.Size(); j++)
+            {
+               e0 = edges[edge0[j]];
+               e1 = edges[edge1[j]];
+               v0 = edge_to_knot[e0];
+               v1 = edge_to_knot[e1];
+               v0 = ( v0 >= 0 ?  v0 : -v0-1);
+               v1 = ( v1 >= 0 ?  v1 : -v1-1);
+               if (v0 != v1)
+               {
+                  corrections++;
+                  if (v0 < v1)
+                  {
+                     edge_to_knot[e1] = (oedge[edge1[j]] >= 0 ? v0 : -v0-1);
+                  }
+                  else if (v1 < v0)
+                  {
+                     edge_to_knot[e0] = (oedge[edge0[j]] >= 0 ? v1 : -v1-1);
+                  }
+               }
+            }
+         }
+
+         passes++;
+      }
+      while (corrections > 0 && passes < GetNE() + 1);
+
+      // Check the validity of corrections applied
+      if (corrections > 0 )
+      {
+         mfem::err<<"Edge_to_knot mapping potentially incorrect"<<endl;
+         mfem::err<<"  passes      = "<<passes<<endl;
+         mfem::err<<"  corrections = "<<corrections<<endl;
+      }
+
+      /* Renumber knotvectors, such that:
+         -- numbering is consecutive
+         -- starts at zero */
+      Array<int> cnt(NumOfEdges);
+      cnt = 0;
+      for (j = 0; j < NumOfEdges; j++)
+      {
+         k = edge_to_knot[j];
+         cnt[(k >= 0 ? k : -k-1)]++;
+      }
+
+      k = 0;
+      for (j = 0; j < cnt.Size(); j++)
+      {
+         cnt[j] = (cnt[j] > 0 ? k++ : -1);
+      }
+
+      for (j = 0; j < NumOfEdges; j++)
+      {
+         k = edge_to_knot[j];
+         edge_to_knot[j] = (k >= 0 ? cnt[k]:-cnt[-k-1]-1);
+      }
+
+      // Print knot to edge mapping
+      mfem::out<<"Generated edge to knot mapping:"<<endl;
+      for (j = 0; j < NumOfEdges; j++)
+      {
+         int *v = edge_vertex->GetRow(j);
+         k = edge_to_knot[j];
+
+         v0 = v[0];
+         v1 = v[1];
+         if (k < 0)
+         {
+            v[0] = v1;
+            v[1] = v0;
+         }
+         mfem::out<<(k >= 0 ? k:-k-1)<<" "<< v[0] <<" "<<v[1]<<endl;
+      }
+
+      // Terminate here upon failure after printing to have an idea of edge_to_knot.
+      if (corrections > 0 ) {mfem_error("Mesh::LoadPatchTopo");}
+   }
 }
 
 void XYZ_VectorFunction(const Vector &p, Vector &v)
@@ -10466,7 +10702,7 @@ void Mesh::PrintTopo(std::ostream &os,const Array<int> &e_to_k) const
    os << "\nvertices\n" << NumOfVertices << '\n';
 }
 
-void Mesh::Save(const char *fname, int precision) const
+void Mesh::Save(const std::string &fname, int precision) const
 {
    ofstream ofs(fname);
    ofs.precision(precision);

mesh/mesh.hpp

@@ -643,8 +643,8 @@ public:
    /** Creates mesh by reading a file in MFEM, Netgen, or VTK format. If
        generate_edges = 0 (default) edges are not generated, if 1 edges are
        generated. See also @a Mesh::LoadFromFile. */
-   explicit Mesh(const char *filename, int generate_edges = 0, int refine = 1,
-                 bool fix_orientation = true);
+   explicit Mesh(const std::string &filename, int generate_edges = 0,
+                 int refine = 1, bool fix_orientation = true);
 
    /** Creates mesh by reading data stream in MFEM, Netgen, or VTK format. If
        generate_edges = 0 (default) edges are not generated, if 1 edges are
@@ -697,7 +697,7 @@ public:
        @note @a filename is not cached by the Mesh object and can be
        safely deleted following this function call.
    */
-   static Mesh LoadFromFile(const char *filename,
+   static Mesh LoadFromFile(const std::string &filename,
                             int generate_edges = 0, int refine = 1,
                             bool fix_orientation = true);
 
@@ -1984,6 +1984,15 @@ public:
    /// @}
 
    ///@{ @name NURBS mesh refinement methods
+   /** Refine a NURBS mesh with the knots specified in the file named @a ref_file.
+       The file has the number of knot vectors on the first line. It is the same
+       number of knot vectors specified in the NURBS mesh in the section edges. Then
+       for each knot vector specified in the section edges (with the same ordering),
+       a line describes (in this order): 1) an integer giving the number of knots
+       inserted, 2) the knots inserted as a double. The advantage of this method
+       is that it is possible to specifically refine a coarse NURBS mesh without
+       changing the mesh file itself. Examples in miniapps/nurbs/meshes. */
+   void RefineNURBSFromFile(std::string ref_file);
    void KnotInsert(Array<KnotVector *> &kv);
    void KnotInsert(Array<Vector *> &kv);
    /* For each knot vector:
@@ -2003,7 +2012,7 @@ public:
 
    /// Save the mesh to a file using Mesh::Print. The given @a precision will be
    /// used for ASCII output.
-   virtual void Save(const char *fname, int precision=16) const;
+   virtual void Save(const std::string &fname, int precision=16) const;
 
    /// Print the mesh to the given stream using the adios2 bp format
 #ifdef MFEM_USE_ADIOS2

mesh/ncmesh.cpp

@@ -2176,9 +2176,9 @@ void NCMesh::UpdateVertices()
    //   - ghost (non-local) vertices (code -3)
    //   - vertices beyond the ghost layer (code -4)
 
-   for (auto node = nodes.begin(); node != nodes.end(); ++node)
+   for (auto & node : nodes)
    {
-      node->vert_index = -4; // assume beyond ghost layer
+      node.vert_index = -4; // assume beyond ghost layer
    }
 
    for (int i = 0; i < leaf_elements.Size(); i++)
@@ -2208,11 +2208,11 @@ void NCMesh::UpdateVertices()
    // STEP 2: assign indices of top-level local vertices, in original order
 
    NVertices = 0;
-   for (auto node = nodes.begin(); node != nodes.end(); ++node)
+   for (auto &node : nodes)
    {
-      if (node->vert_index == -1)
+      if (node.vert_index == -1)
       {
-         node->vert_index = NVertices++;
+         node.vert_index = NVertices++;
       }
    }
 
@@ -2308,20 +2308,20 @@ void NCMesh::UpdateVertices()
       }
 
       vertex_nodeId.SetSize(NVertices);
-      for (auto node = nodes.begin(); node != nodes.end(); ++node)
+      for (auto &node : nodes)
       {
-         if (node->HasVertex() && node->vert_index >= 0)
+         if (node.HasVertex() && node.vert_index >= 0)
          {
-            vertex_nodeId[node->vert_index] = node.index();
+            vertex_nodeId[node.vert_index] = node.index();
          }
       }
 
       NGhostVertices = 0;
-      for (auto node = nodes.begin(); node != nodes.end(); ++node)
+      for (auto &node : nodes)
       {
-         if (node->HasVertex() && node->vert_index < 0)
+         if (node.HasVertex() && node.vert_index < 0)
          {
-            node->vert_index = NVertices + (NGhostVertices++);
+            node.vert_index = NVertices + (NGhostVertices++);
          }
       }
    }
@@ -2545,13 +2545,13 @@ void NCMesh::OnMeshUpdated(Mesh *mesh)
    NFaces = mesh->GetNumFaces();
    if (Dim < 2) { NFaces = 0; }
    // clear Node::edge_index and Face::index
-   for (auto node = nodes.begin(); node != nodes.end(); ++node)
+   for (auto &node : nodes)
    {
-      if (node->HasEdge()) { node->edge_index = -1; }
+      if (node.HasEdge()) { node.edge_index = -1; }
    }
-   for (auto face = faces.begin(); face != faces.end(); ++face)
+   for (auto &face : faces)
    {
-      face->index = -1;
+      face.index = -1;
    }
 
    // get edge enumeration from the Mesh
@@ -2615,19 +2615,19 @@ void NCMesh::OnMeshUpdated(Mesh *mesh)
 
    // count ghost edges and assign their indices
    NGhostEdges = 0;
-   for (auto node = nodes.begin(); node != nodes.end(); ++node)
+   for (auto &node : nodes)
    {
-      if (node->HasEdge() && node->edge_index < 0)
+      if (node.HasEdge() && node.edge_index < 0)
       {
-         node->edge_index = NEdges + (NGhostEdges++);
+         node.edge_index = NEdges + (NGhostEdges++);
       }
    }
 
    // count ghost faces
    NGhostFaces = 0;
-   for (auto face = faces.begin(); face != faces.end(); ++face)
+   for (auto &face : faces)
    {
-      if (face->index < 0) { NGhostFaces++; }
+      if (face.index < 0) { NGhostFaces++; }
    }
 
    if (Dim == 2)
@@ -2671,9 +2671,9 @@ void NCMesh::OnMeshUpdated(Mesh *mesh)
    }
 
    // assign valid indices also to faces beyond the ghost layer
-   for (auto face = faces.begin(); face != faces.end(); ++face)
+   for (auto &face : faces)
    {
-      if (face->index < 0) { face->index = NFaces + (nghosts++); }
+      if (face.index < 0) { face.index = NFaces + (nghosts++); }
    }
    MFEM_ASSERT(nghosts == NGhostFaces, "");
 }
@@ -3452,7 +3452,7 @@ const NCMesh::MeshId& NCMesh::NCList::LookUp(int index, int *type) const
 
    if (!type)
    {
-      MFEM_VERIFY(key >= 0, "entity not found.");
+      MFEM_VERIFY(key >= 0, "index " << index << " not found.");
    }
    else // return entity type if requested, don't abort when not found
    {

mesh/nurbs.cpp

@@ -1522,6 +1522,7 @@ NURBSExtension::NURBSExtension(const NURBSExtension &orig)
      own_topo(true),
      edge_to_knot(orig.edge_to_knot),
      knotVectors(orig.knotVectors.Size()), // knotVectors are copied in the body
+     knotVectorsCompr(orig.knotVectorsCompr.Size()),
      weights(orig.weights),
      d_to_d(orig.d_to_d),
      master(orig.master),
@@ -1547,6 +1548,7 @@ NURBSExtension::NURBSExtension(const NURBSExtension &orig)
    {
       knotVectors[i] = new KnotVector(*orig.knotVectors[i]);
    }
+   CreateComprehensiveKV();
 
    // Copy the patches:
    for (int p = 0; p < patches.Size(); p++)
@@ -1649,6 +1651,8 @@ NURBSExtension::NURBSExtension(std::istream &input)
       MFEM_ABORT("invalid section: " << ident);
    }
 
+   CreateComprehensiveKV();
+
    SetOrdersFromKnotVectors();
 
    GenerateOffsets();
@@ -1728,6 +1732,7 @@ NURBSExtension::NURBSExtension(NURBSExtension *parent, int newOrder)
 
    NumOfKnotVectors = parent->GetNKV();
    knotVectors.SetSize(NumOfKnotVectors);
+   knotVectorsCompr.SetSize(parent->GetNP()*parent->Dimension());
    const Array<int> &pOrders = parent->GetOrders();
    for (int i = 0; i < NumOfKnotVectors; i++)
    {
@@ -1741,6 +1746,7 @@ NURBSExtension::NURBSExtension(NURBSExtension *parent, int newOrder)
          knotVectors[i] = new KnotVector(*parent->GetKnotVector(i));
       }
    }
+   CreateComprehensiveKV();
 
    // copy some data from parent
    NumOfElements    = parent->NumOfElements;
@@ -1798,6 +1804,7 @@ NURBSExtension::NURBSExtension(NURBSExtension *parent,
          knotVectors[i] = new KnotVector(*parent->GetKnotVector(i));
       }
    }
+   CreateComprehensiveKV();
 
    // copy some data from parent
    NumOfElements    = parent->NumOfElements;
@@ -1848,6 +1855,7 @@ NURBSExtension::NURBSExtension(Mesh *mesh_array[], int num_pieces)
    {
       knotVectors[i] = new KnotVector(*parent->GetKnotVector(i));
    }
+   CreateComprehensiveKV();
 
    GenerateOffsets();
    CountElements();
@@ -1881,6 +1889,11 @@ NURBSExtension::~NURBSExtension()
       delete knotVectors[i];
    }
 
+   for (int i = 0; i < knotVectorsCompr.Size(); i++)
+   {
+      delete knotVectorsCompr[i];
+   }
+
    for (int i = 0; i < patches.Size(); i++)
    {
       delete patches[i];
@@ -2360,17 +2373,6 @@ void NURBSExtension::CheckPatches()
                    << ")\n  Inconsistent edge-to-knot mapping!\n";
          mfem_error();
       }
-
-      if ((Dimension() == 2 &&
-           (edges[0] < 0 || edges[1] < 0)) ||
-
-          (Dimension() == 3 &&
-           (edges[0] < 0 || edges[3] < 0 || edges[8] < 0)))
-      {
-         mfem::err << "NURBSExtension::CheckPatch (patch = " << p
-                   << ") : Bad orientation!\n";
-         mfem_error();
-      }
    }
 }
 
@@ -2402,6 +2404,269 @@ void NURBSExtension::CheckBdrPatches()
    }
 }
 
+void NURBSExtension::CheckKVDirection(int p, Array <int> &kvdir)
+{
+   // patchTopo->GetElementEdges is not yet implemented for 1D
+   MFEM_VERIFY(Dimension()>1, "1D not yet implemented.");
+
+   kvdir.SetSize(Dimension());
+   kvdir = 0;
+
+   Array<int> patchvert, edges, orient, edgevert;
+
+   patchTopo->GetElementVertices(p, patchvert);
+
+   patchTopo->GetElementEdges(p, edges, orient);
+
+   // Compare the vertices of the patches with the vertices of the knotvectors of knot2dge
+   // Based on the match the orientation will be a 1 or a -1
+   // -1: direction is flipped
+   //  1: direction is not flipped
+
+
+   for (int i = 0; i < edges.Size(); i++)
+   {
+      // First side
+      patchTopo->GetEdgeVertices(edges[i], edgevert);
+      if (edgevert[0] == patchvert[0]  && edgevert[1] == patchvert[1])
+      {
+         kvdir[0] = 1;
+      }
+
+      if (edgevert[0] == patchvert[1]  && edgevert[1] == patchvert[0])
+      {
+         kvdir[0] = -1;
+      }
+
+      // Second side
+      if (edgevert[0] == patchvert[1]  && edgevert[1] == patchvert[2])
+      {
+         kvdir[1] = 1;
+      }
+
+      if (edgevert[0] == patchvert[2]  && edgevert[1] == patchvert[1])
+      {
+         kvdir[1] = -1;
+      }
+   }
+
+   if (Dimension() == 3)
+   {
+      // Third side
+      for (int i = 0; i < edges.Size(); i++)
+      {
+         patchTopo->GetEdgeVertices(edges[i], edgevert);
+
+         if (edgevert[0] == patchvert[0]  && edgevert[1] == patchvert[4])
+         {
+            kvdir[2] = 1;
+         }
+
+         if (edgevert[0] == patchvert[4]  && edgevert[1] == patchvert[0])
+         {
+            kvdir[2] = -1;
+         }
+      }
+   }
+
+   MFEM_VERIFY(kvdir.Find(0) == -1, "Could not find direction of knotvector.");
+}
+
+void NURBSExtension::CreateComprehensiveKV()
+{
+   Array<int> edges, orient, kvdir;
+   Array<int> e(Dimension());
+
+   // 1D: comprehensive and unique KV are the same
+   if (Dimension() == 1)
+   {
+      knotVectorsCompr.SetSize(GetNKV());
+      for (int i = 0; i < GetNKV(); i++)
+      {
+         knotVectorsCompr[i] = new KnotVector(*(KnotVec(i)));
+      }
+      return;
+   }
+   else if (Dimension() == 2)
+   {
+      knotVectorsCompr.SetSize(GetNP()*Dimension());
+      e[0] = 0;
+      e[1] = 1;
+   }
+   else if (Dimension() == 3)
+   {
+      knotVectorsCompr.SetSize(GetNP()*Dimension());
+      e[0] = 0;
+      e[1] = 3;
+      e[2] = 8;
+   }
+
+   for (int p = 0; p < GetNP(); p++)
+   {
+      CheckKVDirection(p, kvdir);
+
+      patchTopo->GetElementEdges(p, edges, orient);
+
+      for (int d = 0; d < Dimension(); d++)
+      {
+         // Indices in unique and comprehensive sets of the KnotVector
+         int iun = edges[e[d]];
+         int icomp = Dimension()*p+d;
+
+         knotVectorsCompr[icomp] = new KnotVector(*(KnotVec(iun)));
+
+         if (kvdir[d] == -1) {knotVectorsCompr[icomp]->Flip();}
+      }
+   }
+
+   MFEM_VERIFY(ConsistentKVSets(), "Mismatch in KnotVectors");
+}
+
+void NURBSExtension::UpdateUniqueKV()
+{
+   Array<int> e(Dimension());
+
+   // 1D: comprehensive and unique KV are the same
+   if (Dimension() == 1)
+   {
+      for (int i = 0; i < GetNKV(); i++)
+      {
+         *(KnotVec(i)) = *(knotVectorsCompr[i]);
+      }
+      return;
+   }
+   else if (Dimension() == 2)
+   {
+      e[0] = 0;
+      e[1] = 1;
+   }
+   else if (Dimension() == 3)
+   {
+      e[0] = 0;
+      e[1] = 3;
+      e[2] = 8;
+   }
+
+   for (int p = 0; p < GetNP(); p++)
+   {
+      Array<int> edges, orient, kvdir;
+
+      patchTopo->GetElementEdges(p, edges, orient);
+      CheckKVDirection(p, kvdir);
+
+      for ( int d = 0; d < Dimension(); d++)
+      {
+         bool flip = false;
+         if (kvdir[d] == -1) {flip = true;}
+
+         // Indices in unique and comprehensive sets of the KnotVector
+         int iun = edges[e[d]];
+         int icomp = Dimension()*p+d;
+
+         // Check if difference in order
+         int o1 = KnotVec(iun)->GetOrder();
+         int o2 = knotVectorsCompr[icomp]->GetOrder();
+         int diffo = abs(o1 - o2);
+
+         if (diffo)
+         {
+            // Update reduced set of knotvectors
+            *(KnotVec(iun)) = *(knotVectorsCompr[icomp]);
+
+            // Give correct direction to unique knotvector.
+            if (flip) { KnotVec(iun)->Flip(); }
+         }
+
+         // Check if difference between knots
+         Vector diffknot;
+
+         if (flip) { knotVectorsCompr[icomp]->Flip(); }
+
+         KnotVec(iun)->Difference(*(knotVectorsCompr[icomp]), diffknot);
+
+         if (flip) { knotVectorsCompr[icomp]->Flip(); }
+
+         if (diffknot.Size() > 0)
+         {
+            // Update reduced set of knotvectors
+            *(KnotVec(iun)) = *(knotVectorsCompr[icomp]);
+
+            // Give correct direction to unique knotvector.
+            if (flip) {KnotVec(iun)->Flip();}
+         }
+      }
+   }
+
+   MFEM_VERIFY(ConsistentKVSets(), "Mismatch in KnotVectors");
+}
+
+bool NURBSExtension::ConsistentKVSets()
+{
+   // patchTopo->GetElementEdges is not yet implemented for 1D
+   MFEM_VERIFY(Dimension()>1, "1D not yet implemented.");
+
+   Array<int> edges, orient, kvdir;
+   Vector diff;
+
+   Array<int>e(Dimension());
+
+   e[0] = 0;
+
+   if (Dimension() == 2)
+   {
+      e[1] = 1;
+   }
+   else if (Dimension() == 3)
+   {
+      e[1] = 3;
+      e[2] = 8;
+   }
+
+   for (int p = 0; p < GetNP(); p++)
+   {
+      patchTopo->GetElementEdges(p, edges, orient);
+
+      CheckKVDirection(p, kvdir);
+
+      for (int d = 0; d < Dimension(); d++)
+      {
+         bool flip = false;
+         if (kvdir[d] == -1) {flip = true;}
+
+         // Indices in unique and comprehensive sets of the KnotVector
+         int iun = edges[e[d]];
+         int icomp = Dimension()*p+d;
+
+         // Check if KnotVectors are of equal order
+         int o1 = KnotVec(iun)->GetOrder();
+         int o2 = knotVectorsCompr[icomp]->GetOrder();
+         int diffo = abs(o1 - o2);
+
+         if (diffo)
+         {
+            mfem::out << "\norder of knotVectorsCompr " << d << " of patch " << p;
+            mfem::out << " does not agree with knotVectors " << KnotInd(iun) << "\n";
+            return false;
+         }
+
+         // Check if Knotvectors have the same knots
+         if (flip) {knotVectorsCompr[icomp]->Flip();}
+
+         KnotVec(iun)->Difference(*(knotVectorsCompr[icomp]), diff);
+
+         if (flip) {knotVectorsCompr[icomp]->Flip();}
+
+         if (diff.Size() > 0)
+         {
+            mfem::out << "\nknotVectorsCompr " << d << " of patch " << p;
+            mfem::out << " does not agree with knotVectors " << KnotInd(iun) << "\n";
+            return false;
+         }
+      }
+   }
+   return true;
+}
+
 void NURBSExtension::GetPatchKnotVectors(int p, Array<KnotVector *> &kv)
 {
    Array<int> edges, orient;
@@ -2410,20 +2675,18 @@ void NURBSExtension::GetPatchKnotVectors(int p, Array<KnotVector *> &kv)
 
    if (Dimension() == 1)
    {
-      kv[0] = KnotVec(p);
+      kv[0] = knotVectorsCompr[Dimension()*p];
    }
    else if (Dimension() == 2)
    {
-      patchTopo->GetElementEdges(p, edges, orient);
-      kv[0] = KnotVec(edges[0]);
-      kv[1] = KnotVec(edges[1]);
+      kv[0] = knotVectorsCompr[Dimension()*p];
+      kv[1] = knotVectorsCompr[Dimension()*p + 1];
    }
    else
    {
-      patchTopo->GetElementEdges(p, edges, orient);
-      kv[0] = KnotVec(edges[0]);
-      kv[1] = KnotVec(edges[3]);
-      kv[2] = KnotVec(edges[8]);
+      kv[0] = knotVectorsCompr[Dimension()*p];
+      kv[1] = knotVectorsCompr[Dimension()*p + 1];
+      kv[2] = knotVectorsCompr[Dimension()*p + 2];
    }
 }
 
@@ -2433,22 +2696,21 @@ const
    Array<int> edges, orient;
 
    kv.SetSize(Dimension());
+
    if (Dimension() == 1)
    {
-      kv[0] = KnotVec(p);
+      kv[0] = knotVectorsCompr[Dimension()*p];
    }
    else if (Dimension() == 2)
    {
-      patchTopo->GetElementEdges(p, edges, orient);
-      kv[0] = KnotVec(edges[0]);
-      kv[1] = KnotVec(edges[1]);
+      kv[0] = knotVectorsCompr[Dimension()*p];
+      kv[1] = knotVectorsCompr[Dimension()*p + 1];
    }
    else
    {
-      patchTopo->GetElementEdges(p, edges, orient);
-      kv[0] = KnotVec(edges[0]);
-      kv[1] = KnotVec(edges[3]);
-      kv[2] = KnotVec(edges[8]);
+      kv[0] = knotVectorsCompr[Dimension()*p];
+      kv[1] = knotVectorsCompr[Dimension()*p + 1];
+      kv[2] = knotVectorsCompr[Dimension()*p + 2];
    }
 }
 
@@ -3089,6 +3351,55 @@ void NURBSExtension::Generate3DElementDofTable()
    el_dof = new Table(NumOfActiveElems, el_dof_list);
 }
 
+void NURBSExtension::GetPatchDofs(const int patch, Array<int> &dofs)
+{
+   const KnotVector *kv[3];
+   NURBSPatchMap p2g(this);
+
+   p2g.SetPatchDofMap(patch, kv);
+
+   if (Dimension() == 1)
+   {
+      const int nx = kv[0]->GetNCP();
+      dofs.SetSize(nx);
+
+      for (int i=0; i<nx; ++i)
+      {
+         dofs[i] = DofMap(p2g(i));
+      }
+   }
+   else if (Dimension() == 2)
+   {
+      const int nx = kv[0]->GetNCP();
+      const int ny = kv[1]->GetNCP();
+      dofs.SetSize(nx * ny);
+
+      for (int j=0; j<ny; ++j)
+         for (int i=0; i<nx; ++i)
+         {
+            dofs[i + (nx * j)] = DofMap(p2g(i, j));
+         }
+   }
+   else if (Dimension() == 3)
+   {
+      const int nx = kv[0]->GetNCP();
+      const int ny = kv[1]->GetNCP();
+      const int nz = kv[2]->GetNCP();
+      dofs.SetSize(nx * ny * nz);
+
+      for (int k=0; k<nz; ++k)
+         for (int j=0; j<ny; ++j)
+            for (int i=0; i<nx; ++i)
+            {
+               dofs[i + (nx * (j + (k * ny)))] = DofMap(p2g(i, j, k));
+            }
+   }
+   else
+   {
+      MFEM_ABORT("Only 1D/2D/3D supported currently in NURBSExtension::GetPatchDofs");
+   }
+}
+
 void NURBSExtension::GenerateBdrElementDofTable()
 {
    if (Dimension() == 1)
@@ -3348,6 +3659,7 @@ void NURBSExtension::SetKnotsFromPatches()
       }
    }
 
+   UpdateUniqueKV();
    SetOrdersFromKnotVectors();
 
    GenerateOffsets();
@@ -3471,6 +3783,7 @@ void NURBSExtension::KnotInsert(Array<KnotVector *> &kv)
 {
    Array<int> edges;
    Array<int> orient;
+   Array<int> kvdir;
 
    Array<KnotVector *> pkv(Dimension());
 
@@ -3494,7 +3807,26 @@ void NURBSExtension::KnotInsert(Array<KnotVector *> &kv)
          pkv[2] = kv[KnotInd(edges[8])];
       }
 
-      patches[p]->KnotInsert(pkv);
+
+      // Check whether inserted knots should be flipped before inserting.
+      // Knotvectors are stored in a different array pkvc such that the original
+      // knots which are inserted are not changed.
+      // We need those knots for multiple patches so they have to remain original
+      CheckKVDirection(p, kvdir);
+
+      Array<KnotVector *> pkvc(Dimension());
+      for (int d = 0; d < Dimension(); d++)
+      {
+         pkvc[d] = new KnotVector(*(pkv[d]));
+
+         if (kvdir[d] == -1)
+         {
+            pkvc[d]->Flip();
+         }
+      }
+
+      patches[p]->KnotInsert(pkvc);
+      for (int d = 0; d < Dimension(); d++) { delete pkvc[d]; }
    }
 }
 
@@ -3502,6 +3834,7 @@ void NURBSExtension::KnotInsert(Array<Vector *> &kv)
 {
    Array<int> edges;
    Array<int> orient;
+   Array<int> kvdir;
 
    Array<Vector *> pkv(Dimension());
 
@@ -3525,11 +3858,41 @@ void NURBSExtension::KnotInsert(Array<Vector *> &kv)
          pkv[2] = kv[KnotInd(edges[8])];
       }
 
-      patches[p]->KnotInsert(pkv);
+
+      // Check whether inserted knots should be flipped before inserting.
+      // Knotvectors are stored in a different array pkvc such that the original
+      // knots which are inserted are not changed.
+      CheckKVDirection(p, kvdir);
+
+      Array<Vector *> pkvc(Dimension());
+      for (int d = 0; d < Dimension(); d++)
+      {
+         pkvc[d] = new Vector(*(pkv[d]));
+
+         if (kvdir[d] == -1)
+         {
+            // Find flip point, for knotvectors that do not have the domain [0:1]
+            KnotVector *kva = knotVectorsCompr[Dimension()*p+d];
+            double apb = (*kva)[0] + (*kva)[kva->Size()-1];
+
+            // Flip vector
+            int size = pkvc[d]->Size();
+            int ns = ceil(size/2.0);
+            for (int j = 0; j < ns; j++)
+            {
+               double tmp = apb - pkvc[d]->Elem(j);
+               pkvc[d]->Elem(j) = apb - pkvc[d]->Elem(size-1-j);
+               pkvc[d]->Elem(size-1-j) = tmp;
+            }
+         }
+      }
+
+      patches[p]->KnotInsert(pkvc);
+
+      for (int i = 0; i < Dimension(); i++) { delete pkvc[i]; }
    }
 }
 
-
 void NURBSExtension::GetPatchNets(const Vector &coords, int vdim)
 {
    if (Dimension() == 1)
@@ -3730,6 +4093,12 @@ void NURBSExtension::Set3DSolutionVector(Vector &coords, int vdim)
    }
 }
 
+void NURBSExtension::GetElementIJK(int elem, Array<int> & ijk)
+{
+   MFEM_VERIFY(ijk.Size() == el_to_IJK.NumCols(), "");
+   el_to_IJK.GetRow(elem, ijk);
+}
+
 void NURBSExtension::SetPatchToElements()
 {
    const int np = GetNP();
@@ -3813,6 +4182,7 @@ ParNURBSExtension::ParNURBSExtension(MPI_Comm comm, NURBSExtension *parent,
    {
       knotVectors[i] = new KnotVector(*parent->GetKnotVector(i));
    }
+   CreateComprehensiveKV();
 
    GenerateOffsets();
    CountElements();
@@ -3868,6 +4238,7 @@ ParNURBSExtension::ParNURBSExtension(NURBSExtension *parent,
 
    NumOfKnotVectors = parent->NumOfKnotVectors;
    Swap(knotVectors, parent->knotVectors);
+   Swap(knotVectorsCompr, parent->knotVectorsCompr);
 
    NumOfVertices    = parent->NumOfVertices;
    NumOfElements    = parent->NumOfElements;
@@ -4171,22 +4542,23 @@ void NURBSPatchMap::GetPatchKnotVectors(int p, const KnotVector *kv[])
 
    if (Ext->Dimension() == 1)
    {
-      kv[0] = Ext->KnotVec(p);
+      kv[0] = Ext->knotVectorsCompr[Ext->Dimension()*p];
    }
    else if (Ext->Dimension() == 2)
    {
       Ext->patchTopo->GetElementEdges(p, edges, oedge);
-      kv[0] = Ext->KnotVec(edges[0]);
-      kv[1] = Ext->KnotVec(edges[1]);
+
+      kv[0] = Ext->knotVectorsCompr[Ext->Dimension()*p];
+      kv[1] = Ext->knotVectorsCompr[Ext->Dimension()*p + 1];
    }
    else if (Ext->Dimension() == 3)
    {
       Ext->patchTopo->GetElementEdges(p, edges, oedge);
       Ext->patchTopo->GetElementFaces(p, faces, oface);
 
-      kv[0] = Ext->KnotVec(edges[0]);
-      kv[1] = Ext->KnotVec(edges[3]);
-      kv[2] = Ext->KnotVec(edges[8]);
+      kv[0] = Ext->knotVectorsCompr[Ext->Dimension()*p];
+      kv[1] = Ext->knotVectorsCompr[Ext->Dimension()*p + 1];
+      kv[2] = Ext->knotVectorsCompr[Ext->Dimension()*p + 2];
    }
    opatch = 0;
 }

mesh/nurbs.hpp

@@ -21,6 +21,7 @@
 #include "../general/communication.hpp"
 #endif
 #include <iostream>
+#include <set>
 
 namespace mfem
 {
@@ -218,7 +219,11 @@ protected:
    Mesh *patchTopo;
    int own_topo;
    Array<int> edge_to_knot;
+   /** Set of knotvectors containing unique KnotVectors only */
    Array<KnotVector *> knotVectors;
+   /** Comprehensive set of knotvectors. This set contains a KnotVector for
+       every edge.*/
+   Array<KnotVector *> knotVectorsCompr;
    Vector weights;
 
    // periodic BC info:
@@ -261,10 +266,22 @@ protected:
    void CheckPatches();
    void CheckBdrPatches();
 
+   /** Checks the direction of the knotvectors in the patch based on
+       the patch orientation for patch @a p returns the direction of
+       the Knotvectors in @a kvdir.*/
+   void CheckKVDirection(int p, Array <int> &kvdir);
+   /**  Creates the comprehensive set of KnotVectors. They are the same for 1D. */
+   void CreateComprehensiveKV();
+   /**  Updates the unique set of KnotVectors */
+   void UpdateUniqueKV();
+
+   /** Checks if the comprehensive array of KnotVectors agrees with
+       the reduced set of KnotVectors. Returns false if it finds
+       a difference. */
+   bool ConsistentKVSets();
+
    void GetPatchKnotVectors   (int p, Array<KnotVector *> &kv);
-   void GetPatchKnotVectors   (int p, Array<const KnotVector *> &kv) const;
    void GetBdrPatchKnotVectors(int p, Array<KnotVector *> &kv);
-   void GetBdrPatchKnotVectors(int p, Array<const KnotVector *> &kv) const;
 
    void SetOrderFromOrders();
    void SetOrdersFromKnotVectors();
@@ -407,6 +424,11 @@ public:
    int GetNTotalDof() const { return NumOfDofs; }
    int GetNDof()      const { return NumOfActiveDofs; }
 
+   /// Returns knotvectors in each dimension for patch @a p.
+   void GetPatchKnotVectors(int p, Array<const KnotVector *> &kv) const;
+
+   void GetBdrPatchKnotVectors(int p, Array<const KnotVector *> &kv) const;
+
    // Knotvector read-only access function
    const KnotVector *GetKnotVector(int i) const { return knotVectors[i]; }
 
@@ -466,6 +488,16 @@ public:
    void KnotInsert(Array<KnotVector *> &kv);
    void KnotInsert(Array<Vector *> &kv);
 
+   /// Returns the index of the patch containing element @a elem.
+   int GetElementPatch(int elem) const { return el_to_patch[elem]; }
+
+   /** Returns the Cartesian indices (i,j) in 2D or (i,j,k) in 3D of element
+       @a elem, in the knot-span tensor product ordering for its patch. */
+   void GetElementIJK(int elem, Array<int> & ijk);
+
+   // Returns the degrees of freedom on the patch, in Cartesian order.
+   void GetPatchDofs(const int patch, Array<int> &dofs);
+
    const Array<int>& GetPatchElements(int patch);
    const Array<int>& GetPatchBdrElements(int patch);
 };

mesh/pmesh.cpp

@@ -4952,7 +4952,7 @@ void ParMesh::Print(std::ostream &os) const
    }
 }
 
-void ParMesh::Save(const char *fname, int precision) const
+void ParMesh::Save(const std::string &fname, int precision) const
 {
    ostringstream fname_with_suffix;
    fname_with_suffix << fname << "." << setfill('0') << setw(6) << MyRank;
@@ -5615,7 +5615,7 @@ Mesh ParMesh::GetSerialMesh(int save_rank) const
    return serialmesh;
 }
 
-void ParMesh::SaveAsOne(const char *fname, int precision) const
+void ParMesh::SaveAsOne(const std::string &fname, int precision) const
 {
    ofstream ofs;
    if (MyRank == 0)
@@ -6505,7 +6505,7 @@ static void PrintVertex(const Vertex &v, int space_dim, ostream &os)
    }
 }
 
-void ParMesh::PrintSharedEntities(const char *fname_prefix) const
+void ParMesh::PrintSharedEntities(const std::string &fname_prefix) const
 {
    stringstream out_name;
    out_name << fname_prefix << '_' << setw(5) << setfill('0') << MyRank

mesh/pmesh.hpp

@@ -596,7 +596,7 @@ public:
    /// given suffixes according to the MPI rank. The mesh will be written to the
    /// files using ParMesh::Print. The given @a precision will be used for ASCII
    /// output.
-   void Save(const char *fname, int precision=16) const override;
+   void Save(const std::string &fname, int precision=16) const override;
 
 #ifdef MFEM_USE_ADIOS2
    /** Print the part of the mesh in the calling processor using adios2 bp
@@ -625,7 +625,7 @@ public:
 
    /// Save the mesh as a single file (using ParMesh::PrintAsOne). The given
    /// @a precision is used for ASCII output.
-   void SaveAsOne(const char *fname, int precision=16) const;
+   void SaveAsOne(const std::string &fname, int precision=16) const;
 
    /// Old mesh format (Netgen/Truegrid) version of 'PrintAsOne'
    void PrintAsOneXG(std::ostream &out = mfem::out);
@@ -662,7 +662,7 @@ public:
                   InverseElementTransformation *inv_trans = NULL) override;
 
    /// Debugging method
-   void PrintSharedEntities(const char *fname_prefix) const;
+   void PrintSharedEntities(const std::string &fname_prefix) const;
 
    virtual ~ParMesh();
 

mesh/pncmesh.cpp

@@ -435,25 +435,21 @@ void ParNCMesh::CreateGroups(int nentities, Array<Connection> &index_rank,
    entity_group = 0;
 
    CommGroup group;
-   group.reserve(128);
 
-   int begin = 0, end = 0;
-   while (begin < index_rank.Size())
+   for (auto begin = index_rank.begin(); begin != index_rank.end(); /* nothing */)
    {
-      int index = index_rank[begin].from;
-      if (index >= nentities)
-      {
-         break; // probably creating entity_conf_group (no ghosts)
-      }
-      while (end < index_rank.Size() && index_rank[end].from == index)
-      {
-         end++;
-      }
-      group.resize(end - begin);
-      for (int i = begin; i < end; i++)
-      {
-         group[i - begin] = index_rank[i].to;
-      }
+      const auto &index = begin->from;
+      if (index >= nentities) { break; }
+
+      // Locate the next connection that is not from this index
+      const auto end = std::find_if(begin, index_rank.end(),
+      [&index](const mfem::Connection &c) { return c.from != index;});
+
+      // For each connection from this index, collect the ranks connected.
+      group.resize(std::distance(begin, end));
+      std::transform(begin, end, group.begin(), [](const mfem::Connection &c) { return c.to; });
+
+      // assign this entity's group and advance the search start
       entity_group[index] = GetGroupId(group);
       begin = end;
    }
@@ -461,9 +457,9 @@ void ParNCMesh::CreateGroups(int nentities, Array<Connection> &index_rank,
 
 void ParNCMesh::AddConnections(int entity, int index, const Array<int> &ranks)
 {
-   for (int i = 0; i < ranks.Size(); i++)
+   for (auto rank : ranks)
    {
-      entity_index_rank[entity].Append(Connection(index, ranks[i]));
+      entity_index_rank[entity].Append(Connection(index, rank));
    }
 }
 
@@ -480,9 +476,8 @@ void ParNCMesh::CalculatePMatrixGroups()
    ranks.Reserve(256);
 
    // connect slave edges to master edges and their vertices
-   for (int i = 0; i < shared_edges.masters.Size(); i++)
+   for (const auto &master_edge : shared_edges.masters)
    {
-      const Master &master_edge = shared_edges.masters[i];
       ranks.SetSize(0);
       for (int j = master_edge.slaves_begin; j < master_edge.slaves_end; j++)
       {
@@ -502,9 +497,8 @@ void ParNCMesh::CalculatePMatrixGroups()
    }
 
    // connect slave faces to master faces and their edges and vertices
-   for (int i = 0; i < shared_faces.masters.Size(); i++)
+   for (const auto &master_face : shared_faces.masters)
    {
-      const Master &master_face = shared_faces.masters[i];
       ranks.SetSize(0);
       for (int j = master_face.slaves_begin; j < master_face.slaves_end; j++)
       {

mesh/pncmesh.hpp

@@ -82,23 +82,23 @@ public:
    /** An override of NCMesh::Refine, which is called eventually, after making
        sure that refinements that occur on the processor boundary are sent to
        the neighbor processors so they can keep their ghost layers up to date.*/
-   virtual void Refine(const Array<Refinement> &refinements);
+   void Refine(const Array<Refinement> &refinements) override;
 
    /// Parallel version of NCMesh::LimitNCLevel.
-   virtual void LimitNCLevel(int max_nc_level);
+   void LimitNCLevel(int max_nc_level) override;
 
    /** Parallel version of NCMesh::CheckDerefinementNCLevel. */
-   virtual void CheckDerefinementNCLevel(const Table &deref_table,
-                                         Array<int> &level_ok, int max_nc_level);
+   void CheckDerefinementNCLevel(const Table &deref_table,
+                                 Array<int> &level_ok, int max_nc_level) override;
 
    /** Parallel reimplementation of NCMesh::Derefine, keeps ghost layers
        in sync. The interface is identical. */
-   virtual void Derefine(const Array<int> &derefs);
+   void Derefine(const Array<int> &derefs) override;
 
    /** Gets partitioning for the coarse mesh if the current fine mesh were to
        be derefined. */
-   virtual void GetFineToCoarsePartitioning(const Array<int> &derefs,
-                                            Array<int> &new_ranks) const;
+   void GetFineToCoarsePartitioning(const Array<int> &derefs,
+                                    Array<int> &new_ranks) const;
 
    /** Migrate leaf elements of the global refinement hierarchy (including ghost
        elements) so that each processor owns the same number of leaves (+-1).
@@ -116,7 +116,7 @@ public:
    int GetNGhostVertices() const { return NGhostVertices; }
    int GetNGhostEdges() const { return NGhostEdges; }
    int GetNGhostFaces() const { return NGhostFaces; }
-   int GetNGhostElements() const { return NGhostElements; }
+   int GetNGhostElements() const override { return NGhostElements; }
 
    // Return a list of vertices/edges/faces shared by this processor and at
    // least one other processor. These are subsets of NCMesh::<entity>_list. */
@@ -232,12 +232,12 @@ public:
 
    /** Extension of NCMesh::GetBoundaryClosure. Filters out ghost vertices and
        ghost edges from 'bdr_vertices' and 'bdr_edges'. */
-   virtual void GetBoundaryClosure(const Array<int> &bdr_attr_is_ess,
-                                   Array<int> &bdr_vertices,
-                                   Array<int> &bdr_edges);
+   void GetBoundaryClosure(const Array<int> &bdr_attr_is_ess,
+                           Array<int> &bdr_vertices,
+                           Array<int> &bdr_edges) override;
 
    /// Save memory by releasing all non-essential and cached data.
-   virtual void Trim();
+   void Trim() override;
 
    /// Return total number of bytes allocated.
    std::size_t MemoryUsage(bool with_base = true) const;
@@ -267,8 +267,8 @@ protected: // implementation
    MPI_Comm MyComm;
    int NRanks;
 
-   typedef std::vector<CommGroup> GroupList;
-   typedef std::map<CommGroup, GroupId> GroupMap;
+   using GroupList = std::vector<CommGroup>;
+   using GroupMap = std::map<CommGroup, GroupId>;
 
    GroupList groups;  // comm group list; NOTE: groups[0] = { MyRank }
    GroupMap group_id; // search index over groups
@@ -299,7 +299,7 @@ protected: // implementation
    Array<int> ghost_layer;    ///< list of elements whose 'element_type' == 2.
    Array<int> boundary_layer; ///< list of type 3 elements
 
-   virtual void Update();
+   void Update() override;
 
    /// Return the processor number for a global element number.
    int Partition(long index, long total_elements) const
@@ -313,13 +313,13 @@ protected: // implementation
    long PartitionFirstIndex(int rank, long total_elements) const
    { return (rank * total_elements + NRanks-1) / NRanks; }
 
-   virtual void BuildFaceList();
-   virtual void BuildEdgeList();
-   virtual void BuildVertexList();
+   void BuildFaceList() override;
+   void BuildEdgeList() override;
+   void BuildVertexList() override;
 
-   virtual void ElementSharesFace(int elem, int local, int face);
-   virtual void ElementSharesEdge(int elem, int local, int enode);
-   virtual void ElementSharesVertex(int elem, int local, int vnode);
+   void ElementSharesFace(int elem, int local, int face) override;
+   void ElementSharesEdge(int elem, int local, int enode) override;
+   void ElementSharesVertex(int elem, int local, int vnode) override;
 
    GroupId GetGroupId(const CommGroup &group);
    GroupId GetSingletonGroup(int rank);
@@ -451,8 +451,8 @@ protected: // implementation
    protected:
       ParNCMesh* pncmesh;
 
-      virtual void Encode(int);
-      virtual void Decode(int);
+      void Encode(int) override;
+      void Decode(int) override;
    };
 
    /** Used by ParNCMesh::Refine() to inform neighbors about refinements at
@@ -513,8 +513,8 @@ protected: // implementation
    protected:
       ElementSet eset;
 
-      virtual void Encode(int);
-      virtual void Decode(int);
+      void Encode(int) override;
+      void Decode(int) override;
    };
 
    /** Assign new Element::rank to leaf elements and send them to their new

mesh/submesh/ptransfermap.cpp

@@ -162,6 +162,8 @@ void ParTransferMap::Transfer(const ParGridFunction &src,
    if (category_ == TransferCategory::ParentToSubMesh)
    {
       // dst = S1^T src
+      src.HostRead();
+      dst.HostWrite(); // dst is fully overwritten
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
          double s = 1.0;
@@ -178,6 +180,8 @@ void ParTransferMap::Transfer(const ParGridFunction &src,
       //
       // G is identity if the partitioning matches
 
+      src.HostRead();
+      dst.HostReadWrite(); // dst is only partially overwritten
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
          double s = 1.0;
@@ -196,6 +200,9 @@ void ParTransferMap::Transfer(const ParGridFunction &src,
       //
       // G is identity if the partitioning matches
 
+      src.HostRead();
+      dst.HostReadWrite();
+
       z_ = 0.0;
 
       for (int i = 0; i < sub2_to_parent_map_.Size(); i++)

mesh/submesh/transfermap.cpp

@@ -152,6 +152,8 @@ void TransferMap::Transfer(const GridFunction &src,
    if (category_ == TransferCategory::ParentToSubMesh)
    {
       // dst = S1^T src
+      src.HostRead();
+      dst.HostWrite(); // dst is fully overwritten
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
          double s = 1.0;
@@ -168,6 +170,8 @@ void TransferMap::Transfer(const GridFunction &src,
       //
       // G is identity if the partitioning matches
 
+      src.HostRead();
+      dst.HostReadWrite(); // dst is only partially overwritten
       for (int i = 0; i < sub1_to_parent_map_.Size(); i++)
       {
          double s = 1.0;
@@ -184,6 +188,9 @@ void TransferMap::Transfer(const GridFunction &src,
       //
       // G is identity if the partitioning matches
 
+      src.HostRead();
+      dst.HostReadWrite();
+
       z_ = 0.0;
 
       for (int i = 0; i < sub2_to_parent_map_.Size(); i++)

miniapps/nurbs/CMakeLists.txt

@@ -12,11 +12,15 @@
 add_mfem_miniapp(nurbs_ex1
   MAIN nurbs_ex1.cpp
   LIBRARIES mfem)
-  
+
 add_mfem_miniapp(nurbs_curveint
   MAIN nurbs_curveint.cpp
   LIBRARIES mfem)
 
+add_mfem_miniapp(nurbs_patch_ex1
+  MAIN nurbs_patch_ex1.cpp
+  LIBRARIES mfem)
+
 if (MFEM_ENABLE_TESTING)
   add_test(NAME nurbs_ex1_1d_r1_o2_ser
     COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
@@ -63,6 +67,49 @@ if (MFEM_ENABLE_TESTING)
   add_test(NAME nurbs_ex1_weak_patch_format_r1_ser
     COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
     -m ${PROJECT_SOURCE_DIR}/data/square-disc-nurbs-patch.mesh -o 2 --weak-bc -r 1)
+
+  if (MFEM_USE_LAPACK)
+    add_test(NAME nurbs_patch_ex1_o4_r2_iro8_patcha_ser
+      COMMAND $<TARGET_FILE:nurbs_patch_ex1> -incdeg 3 -ref 2 -iro 8 -patcha)
+  endif()
+
+  add_test(NAME nurbs_patch_ex1_o4_r2_iro8_patcha_pa_ser
+    COMMAND $<TARGET_FILE:nurbs_patch_ex1> -incdeg 3 -ref 2 -iro 8 -patcha -pa)
+
+  add_test(NAME nurbs_patch_ex1_o4_r2_iro8_patcha_fint_ser
+    COMMAND $<TARGET_FILE:nurbs_patch_ex1> -incdeg 3 -ref 2 -iro 8 -patcha -fint)
+
+  add_test(NAME nurbs_curveint_unit_weight_ser
+    COMMAND $<TARGET_FILE:nurbs_curveint> -no-vis -no-visit
+    -uw -n 9)
+
+  add_test(NAME nurbs_curveint_non_unit_weight_ser
+    COMMAND $<TARGET_FILE:nurbs_curveint> -no-vis -no-visit
+    -nw -n 9)
+
+  add_test(NAME nurbs_ex1_two_squares_knot_insert
+    COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+    -m ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-squares-nurbs.mesh -o 1 -rf ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-squares.ref)
+
+  add_test(NAME nurbs_ex1_two_squares_rot_knot_insert
+    COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+    -m ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-squares-nurbs-rot.mesh -o 1 -rf ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-squares.ref)
+
+  add_test(NAME nurbs_ex1_two_squares_autoedge_knot_insert
+    COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+    -m ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-squares-nurbs-autoedge.mesh -o 1 -rf ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-squares.ref)
+
+  add_test(NAME nurbs_ex1_two_cubes_knot_insert
+    COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+    -m ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-cubes-nurbs.mesh -o 1 -r 3 -rf ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-cubes.ref)
+
+  add_test(NAME nurbs_ex1_two_cubes_rot_knot_insert
+    COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+    -m ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-cubes-nurbs-rot.mesh -o 1 -r 3 -rf ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-cubes.ref)
+
+  add_test(NAME nurbs_ex1_two_cubes_autoedge_knot_insert
+    COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+    -m ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-cubes-nurbs-autoedge.mesh -o 1 -r 3 -rf ${PROJECT_SOURCE_DIR}/miniapps/nurbs/meshes/two-cubes.ref)
 endif()
 
 if (MFEM_USE_MPI)

miniapps/nurbs/makefile

@@ -21,7 +21,7 @@ CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_MINIAPPS = nurbs_ex1
+SEQ_MINIAPPS = nurbs_ex1 nurbs_patch_ex1 nurbs_curveint
 PAR_MINIAPPS = nurbs_ex1p nurbs_ex11p
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
@@ -58,12 +58,18 @@ EX1_ARGS_2 := -r 0 -o 4
 EX1_ARGS_3 := -r 2
 EX1_ARGS_4 := -m ../../data/beam-hex-nurbs.mesh -pm 1 -ps 2
 EX1_ARGS_5 := -m ../../data/pipe-nurbs-2d.mesh -o 2 -no-ibp -r 0
-EX1_ARGS_6 := -m ../..//data/pipe-nurbs-2d.mesh -o 2 -no-ibp -r 2
+EX1_ARGS_6 := -m ../../data/pipe-nurbs-2d.mesh -o 2 -no-ibp -r 2
 EX1_ARGS_7 := -m ../../data/pipe-nurbs-2d.mesh -o 2 --weak-bc -r 0
 EX1_ARGS_8 := -m ../../data/pipe-nurbs-2d.mesh -o 2 --weak-bc -r 2
 EX1_ARGS_9 := -m ../../data/ball-nurbs.mesh -o 2 --weak-bc -r 0
 EX1_ARGS_10 := -m ../../data/square-disc-nurbs-patch.mesh -o 2 --weak-bc -r 0
 EX1_ARGS_11 := -m ../../data/square-disc-nurbs-patch.mesh -o 2 --weak-bc -r 1
+EX1_ARGS_12 := -m $(MFEM_DIR)/miniapps/nurbs/meshes/two-squares-nurbs.mesh -o 1 -rf $(MFEM_DIR)/miniapps/nurbs/meshes/two-squares.ref
+EX1_ARGS_13 := -m $(MFEM_DIR)/miniapps/nurbs/meshes/two-squares-nurbs-rot.mesh -o 1 -rf $(MFEM_DIR)/miniapps/nurbs/meshes/two-squares.ref
+EX1_ARGS_14 := -m $(MFEM_DIR)/miniapps/nurbs/meshes/two-squares-nurbs-autoedge.mesh -o 1 -rf $(MFEM_DIR)/miniapps/nurbs/meshes/two-squares.ref
+EX1_ARGS_15 := -m $(MFEM_DIR)/miniapps/nurbs/meshes/two-cubes-nurbs.mesh -o 1 -r 3 -rf $(MFEM_DIR)/miniapps/nurbs/meshes/two-cubes.ref
+EX1_ARGS_16 := -m $(MFEM_DIR)/miniapps/nurbs/meshes/two-cubes-nurbs-rot.mesh -o 1 -r 3 -rf $(MFEM_DIR)/miniapps/nurbs/meshes/two-cubes.ref
+EX1_ARGS_17 := -m $(MFEM_DIR)/miniapps/nurbs/meshes/two-cubes-nurbs-autoedge.mesh -o 1 -r 3 -rf $(MFEM_DIR)/miniapps/nurbs/meshes/two-cubes.ref
 
 nurbs_ex1-test-seq: nurbs_ex1
 	@$(call mfem-test,$<,, NURBS miniapp)
@@ -77,6 +83,29 @@ nurbs_ex1-test-seq: nurbs_ex1
 	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_9))
 	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_10))
 	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_11))
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_12))
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_13))
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_14))
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_15))
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_16))
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1_ARGS_17))
+
+EX1PATCH_ARGS_1 := -incdeg 3 -ref 2 -iro 8 -patcha
+EX1PATCH_ARGS_2 := -incdeg 3 -ref 2 -iro 8 -patcha -pa
+EX1PATCH_ARGS_3 := -incdeg 3 -ref 2 -iro 8 -patcha -fint
+nurbs_patch_ex1-test-seq: nurbs_patch_ex1
+ifeq ($(MFEM_USE_LAPACK),YES)
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1PATCH_ARGS_1))
+endif
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1PATCH_ARGS_2))
+	@$(call mfem-test,$<,, NURBS miniapp,$(EX1PATCH_ARGS_3))
+
+CI_ARGS_1 := -uw -n 9 -no-visit
+CI_ARGS_2 := -nw -n 9 -no-visit
+
+nurbs_curveint-test-seq: nurbs_curveint
+	@$(call mfem-test,$<,, NURBS miniapp,$(CI_ARGS_1))
+	@$(call mfem-test,$<,, NURBS miniapp,$(CI_ARGS_2))
 
 EX1P_ARGS_1 :=
 EX1P_ARGS_2 := -m ../../data/pipe-nurbs-2d.mesh -o 2 -no-ibp
@@ -109,5 +138,6 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
-	@rm -f refined.mesh mesh.* sol.* mode_*
+	@rm -f refined.mesh sin-fit.mesh mesh.* sol.* mode_*
 	@rm -rf Example1*
+	@rm -rf CurveInt

miniapps/nurbs/meshes/two-cubes-nurbs-autoedge.mesh

@@ -0,0 +1,64 @@
+MFEM NURBS mesh v1.0
+
+# 3D two cubes mesh consisting of two patches 
+# with the origin in the bottom left corner at z = 0.
+# The edges are not defined to test the usage of 
+# automated edge generation.
+
+dimension
+3
+
+elements
+2
+1 5 0 1 4 3 6 7 10 9
+1 5 1 2 5 4 7 8 11 10
+
+boundary
+0
+
+edges
+0
+
+vertices
+12
+
+patches
+
+knotvectors
+3
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+3
+
+controlpoints
+0 0 0 1
+1 0 0 1
+0 1 0 1
+1 1 0 1
+0 0 1 1
+1 0 1 1
+0 1 1 1
+1 1 1 1
+
+knotvectors
+3
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+3
+
+controlpoints
+1 0 0 1
+2 0 0 1
+1 1 0 1
+2 1 0 1
+1 0 1 1
+2 0 1 1
+1 1 1 1
+2 1 1 1
+

miniapps/nurbs/meshes/two-cubes-nurbs-rot.mesh

@@ -0,0 +1,84 @@
+MFEM NURBS mesh v1.0
+
+# 2D two cubes mesh consisting of two patches 
+# with the origin of the left square in the bottom
+# left corner and the origin of the right square 
+# in the top right corner. Both at z = 0;
+
+dimension
+3
+
+elements
+2
+1 5 0 1 4 3 6 7 10 9
+1 5 5 4 1 2 11 10 7 8
+
+boundary
+0
+
+edges
+20
+0 0 1
+1 1 4
+0 3 4
+1 0 3
+2 1 2
+1 2 5
+2 4 5
+0 6 7
+1 7 10
+0 9 10
+1 6 9
+2 7 8
+1 8 11
+2 10 11
+3 0 6
+3 1 7
+3 2 8
+3 3 9
+3 4 10
+3 5 11
+
+vertices
+12
+
+patches
+
+knotvectors
+3
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+3
+
+controlpoints
+0 0 0 1
+1 0 0 1
+0 1 0 1
+1 1 0 1
+0 0 1 1
+1 0 1 1
+0 1 1 1
+1 1 1 1
+
+knotvectors
+3
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+3
+
+controlpoints
+2 1 0 1
+1 1 0 1
+2 0 0 1
+1 0 0 1
+2 1 1 1
+1 1 1 1
+2 0 1 1
+1 0 1 1
+

miniapps/nurbs/meshes/two-cubes-nurbs.mesh

@@ -0,0 +1,86 @@
+MFEM NURBS mesh v1.0
+
+# 3D two cubes mesh consisting of two patches 
+# with the origin in the bottom left corner at z = 0.
+# Also see two-cubes-nurbs-rot.mesh which
+# has the origin of the left square in the bottom
+# left corner and the origin of the right square 
+# in the top right corner.
+
+dimension
+3
+
+elements
+2
+1 5 0 1 4 3 6 7 10 9
+1 5 1 2 5 4 7 8 11 10
+
+boundary
+0
+
+edges
+20
+0 0 1
+1 1 4
+0 3 4
+1 0 3
+2 1 2
+1 2 5
+2 4 5
+0 6 7
+1 7 10
+0 9 10
+1 6 9
+2 7 8
+1 8 11
+2 10 11
+3 0 6
+3 1 7
+3 2 8
+3 3 9
+3 4 10
+3 5 11
+
+vertices
+12
+
+patches
+
+knotvectors
+3
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+3
+
+controlpoints
+0 0 0 1
+1 0 0 1
+0 1 0 1
+1 1 0 1
+0 0 1 1
+1 0 1 1
+0 1 1 1
+1 1 1 1
+
+knotvectors
+3
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+3
+
+controlpoints
+1 0 0 1
+2 0 0 1
+1 1 0 1
+2 1 0 1
+1 0 1 1
+2 0 1 1
+1 1 1 1
+2 1 1 1
+

miniapps/nurbs/meshes/two-cubes.ref

@@ -0,0 +1,5 @@
+4
+1 0.2
+1 0.2
+1 0.2
+1 0.2

miniapps/nurbs/meshes/two-squares-nurbs-autoedge.mesh

@@ -0,0 +1,54 @@
+MFEM NURBS mesh v1.0
+
+# 2D two squares mesh consisting of two patches 
+# with the origin in the bottom left corner.
+# The edges are not defined to test the usage of 
+# automated edge generation.
+
+dimension
+2
+
+elements
+2
+1 3 0 1 4 3
+1 3 1 2 5 4
+
+boundary
+0
+
+edges
+0
+
+vertices
+6
+
+patches
+
+knotvectors
+2
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+2
+
+controlpoints
+0 0 1
+1 0 1
+0 1 1
+1 1 1
+
+knotvectors
+2
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+2
+
+controlpoints
+1 0 1
+2 0 1
+1 1 1
+2 1 1
+

miniapps/nurbs/meshes/two-squares-nurbs-rot.mesh

@@ -0,0 +1,61 @@
+MFEM NURBS mesh v1.0
+
+# 2D two squares mesh consisting of two patches 
+# with the origin of the left square in the bottom
+# left corner and the origin of the right square 
+# in the top right corner.
+
+dimension
+2
+
+elements
+2
+1 3 0 1 4 3
+1 3 5 4 1 2
+
+boundary
+0
+
+edges
+7
+0 0 1
+1 1 4
+0 3 4
+1 0 3
+2 1 2
+1 2 5
+2 4 5
+
+vertices
+6
+
+patches
+
+knotvectors
+2
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+2
+
+controlpoints
+0 0 1
+1 0 1
+0 1 1
+1 1 1
+
+knotvectors
+2
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+2
+
+controlpoints
+2 1 1
+1 1 1
+2 0 1
+1 0 1
+

miniapps/nurbs/meshes/two-squares-nurbs.mesh

@@ -0,0 +1,63 @@
+MFEM NURBS mesh v1.0
+
+# 2D two squares mesh consisting of two patches 
+# with the origin in the bottom left corner.
+# Also see two-squares-nurbs-rot.mesh which
+# has the origin of the left square in the bottom
+# left corner and the origin of the right square 
+# in the top right corner.
+
+dimension
+2
+
+elements
+2
+1 3 0 1 4 3
+1 3 1 2 5 4
+
+boundary
+0
+
+edges
+7
+0 0 1
+1 1 4
+0 3 4
+1 0 3
+2 1 2
+1 2 5
+2 4 5
+
+vertices
+6
+
+patches
+
+knotvectors
+2
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+2
+
+controlpoints
+0 0 1
+1 0 1
+0 1 1
+1 1 1
+
+knotvectors
+2
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+dimension
+2
+
+controlpoints
+1 0 1
+2 0 1
+1 1 1
+2 1 1
+

miniapps/nurbs/meshes/two-squares.ref

@@ -0,0 +1,4 @@
+3
+1 0.2
+1 0.2
+1 0.2

miniapps/nurbs/nurbs_curveint.cpp

@@ -1,12 +1,26 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//        ------------------------------------------------------------
+//        NURBS CurveInt Miniapp: Interpolate a Curve in a NURBS Patch
+//        ------------------------------------------------------------
+//
 // Compile with: make nurbs_curveint
 //
 // Sample runs:  ./nurbs_curveint -uw -n 9
 //               ./nurbs_curveint -nw -n 9
-
 //
-// Description:  This example code demonstrates the use of MFEM to interpolate
-//               a curve in a NURBS patch. We first define a square shaped
-//               NURBS patch. We then interpolate a sine function on the bottom
+// Description:  This example code demonstrates the use of MFEM to interpolate a
+//               curve in a NURBS patch. We first define a square shaped NURBS
+//               patch. We then interpolate a sine function on the bottom
 //               edge. The results can be viewed in VisIt.
 //
 //               We use curve interpolation for curves with all weights being 1,
@@ -53,6 +67,8 @@ int main(int argc, char *argv[])
    int ncp        = 9;
    int order      = 2;
    bool ifbspline = true;
+   bool visualization = true;
+   bool visit = true;
 
    args.AddOption(&l, "-l", "--box-side-length",
                   "Height and width of the box");
@@ -65,8 +81,13 @@ int main(int argc, char *argv[])
    args.AddOption(&ifbspline, "-uw", "--unit-weight", "-nw",
                   "--non-unit-weight",
                   "Use a unit-weight for B-splines (default) or not: for general NURBS");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization. This is a dummy option to enable testing.");
+   args.AddOption(&visit, "-visit", "--visit", "-no-visit", "--no-visit",
+                  "Enable or disable VisIt visualization.");
 
-   // Parse and print commandline options
+   // Parse and print command line options
    args.Parse();
    if (!args.Good())
    {
@@ -129,9 +150,8 @@ int main(int argc, char *argv[])
    }
    patch.KnotInsert(0, *kv);
 
-   // We locate the controlpoints at the location of the
-   // maxima of the knotvectors. This works very well
-   // for patches with unit weights.
+   // We locate the control points at the location of the maxima of the
+   // knot vectors. This works very well for patches with unit weights.
    kv->FindMaxima(i_args,xi_args, u_args);
 
    for (int i = 0; i < ncp; i++)
@@ -153,7 +173,7 @@ int main(int argc, char *argv[])
    if (!ifbspline)
    {
       // Convert to homogeneous coordinates. FindInterpolant returns
-      // cartesian coordinates.
+      // Cartesian coordinates.
       for (int i = 0; i < ncp; i++)
       {
          patch(i,0,0) *= patch(i,0,2);
@@ -210,16 +230,21 @@ int main(int argc, char *argv[])
    Mesh *mesh = new Mesh(mesh_file.c_str(), 1, 1);
    mesh->PrintInfo();
 
-   // Print mesh to file for visualisation
-   VisItDataCollection dc = VisItDataCollection("mesh", mesh);
-   dc.SetPrefixPath("solution");
-   dc.SetCycle(0);
-   dc.SetTime(0.0);
-   dc.Save();
+   if (visit)
+   {
+      // Print mesh to file for visualization
+      VisItDataCollection dc = VisItDataCollection("mesh", mesh);
+      dc.SetPrefixPath("CurveInt");
+      dc.SetCycle(0);
+      dc.SetTime(0.0);
+      dc.Save();
+   }
 
+   delete mesh;
    delete kv_o1;
    delete kv;
-   delete mesh;
+   delete xy[0];
+   delete xy[1];
 
    return 0;
-}
+}

miniapps/nurbs/nurbs_ex1.cpp

@@ -10,6 +10,12 @@
 //               nurbs_ex1 -m ../../data/disc-nurbs.mesh -o -1
 //               nurbs_ex1 -m ../../data/pipe-nurbs.mesh -o -1
 //               nurbs_ex1 -m ../../data/beam-hex-nurbs.mesh -pm 1 -ps 2
+//               nurbs_ex1 -m ../../data/two-squares-nurbs.mesh -o 1 -rf ../../data/two-squares.ref
+//               nurbs_ex1 -m ../../data/two-squares-nurbs-rot.mesh -o 1 -rf ../../data/two-squares.ref
+//               nurbs_ex1 -m ../../data/two-squares-nurbs-autoedge.mesh -o 1 -rf ../../data/two-squares.ref
+//               nurbs_ex1 -m ../../data/two-cubes-nurbs.mesh -o 1 -r 3 -rf ../../data/two-cubes.ref
+//               nurbs_ex1 -m ../../data/two-cubes-nurbs-rot.mesh -o 1 -r 3 -rf ../../data/two-cubes.ref
+//               nurbs_ex1 -m ../../data/two-cubes-nurbs-autoedge.mesh -o 1 -r 3 -rf ../../data/two-cubes.ref
 //               nurbs_ex1 -m ../../data/segment-nurbs.mesh -r 2 -o 2 -lod 3
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -137,6 +143,7 @@ int main(int argc, char *argv[])
    // 1. Parse command-line options.
    const char *mesh_file = "../../data/star.mesh";
    const char *per_file  = "none";
+   const char *ref_file  = "";
    int ref_levels = -1;
    Array<int> master(0);
    Array<int> slave(0);
@@ -156,6 +163,8 @@ int main(int argc, char *argv[])
                   "Number of times to refine the mesh uniformly, -1 for auto.");
    args.AddOption(&per_file, "-p", "--per",
                   "Periodic BCS file.");
+   args.AddOption(&ref_file, "-rf", "--ref-file",
+                  "File with refinement data");
    args.AddOption(&master, "-pm", "--master",
                   "Master boundaries for periodic BCs");
    args.AddOption(&slave, "-ps", "--slave",
@@ -198,10 +207,16 @@ int main(int argc, char *argv[])
    int dim = mesh->Dimension();
 
    // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
+   //    'ref_levels' of uniform refinement and knot insertion of knots defined
+   //    in a refinement file. We choose 'ref_levels' to be the largest number
+   //    that gives a final mesh with no more than 50,000 elements.
    {
+      // Mesh refinement as defined in refinement file
+      if (mesh->NURBSext && (strlen(ref_file) != 0))
+      {
+         mesh->RefineNURBSFromFile(ref_file);
+      }
+
       if (ref_levels < 0)
       {
          ref_levels =

miniapps/nurbs/nurbs_patch_ex1.cpp

@@ -0,0 +1,333 @@
+//               MFEM Example 1 - NURBS with patch-wise assembly
+//
+// Compile with: make nurbs_patch_ex1
+//
+// Sample runs:  nurbs_patch_ex1 -incdeg 3 -ref 2 -iro 8 -patcha
+//               nurbs_patch_ex1 -incdeg 3 -ref 2 -iro 8 -patcha -pa
+//               nurbs_patch_ex1 -incdeg 3 -ref 2 -iro 8 -patcha -fint
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               This example is a specialization of ex1 which demonstrates
+//               patch-wise matrix assembly and partial assembly on NURBS
+//               meshes. There is the option to compare run times of patch
+//               and element assembly, as well as relative error computation.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void AssembleAndSolve(LinearForm & b, BilinearFormIntegrator * bfi,
+                      Array<int> const& ess_tdof_list, const bool pa,
+                      const bool algebraic_ceed, GridFunction & x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/beam-hex-nurbs.mesh";
+   int order = -1;
+   bool pa = false;
+   const char *device_config = "cpu";
+   bool visualization = true;
+   bool algebraic_ceed = false;
+   bool patchAssembly = false;
+   bool reducedIntegration = true;
+   bool compareToElementWise = true;
+   int nurbs_degree_increase = 0;  // Elevate the NURBS mesh degree by this
+   int ref_levels = 0;
+   int ir_order = -1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+#ifdef MFEM_USE_CEED
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
+                  "Use algebraic Ceed solver");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&patchAssembly, "-patcha", "--patch-assembly", "-no-patcha",
+                  "--no-patch-assembly", "Enable patch-wise assembly.");
+   args.AddOption(&reducedIntegration, "-rint", "--reduced-integration", "-fint",
+                  "--full-integration", "Enable reduced integration rules.");
+   args.AddOption(&ref_levels, "-ref", "--refine",
+                  "Number of uniform mesh refinements.");
+   args.AddOption(&ir_order, "-iro", "--integration-order",
+                  "Order of integration rule.");
+   args.AddOption(&nurbs_degree_increase, "-incdeg", "--nurbs-degree-increase",
+                  "Elevate NURBS mesh degree by this amount.");
+   args.AddOption(&compareToElementWise, "-cew", "--compare-element",
+                  "-no-compare", "-no-compare-element",
+                  "Compute element-wise solution for comparison");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   MFEM_VERIFY(!(pa && !patchAssembly), "Patch assembly must be used with -pa");
+
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. For this NURBS patch example,
+   //    only 3D hexahedral meshes are currently supported. The NURBS degree is
+   //    optionally increased.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (nurbs_degree_increase > 0) { mesh.DegreeElevate(nurbs_degree_increase); }
+
+   // 4. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 5. Define an isoparametric/isogeometric finite element space on the mesh.
+   FiniteElementCollection *fec = nullptr;
+   bool delete_fec;
+   if (mesh.GetNodes())
+   {
+      fec = mesh.GetNodes()->OwnFEC();
+      delete_fec = false;
+      cout << "Using isoparametric FEs: " << fec->Name() << endl;
+   }
+   else
+   {
+      MFEM_ABORT("Mesh must have nodes");
+   }
+   FiniteElementSpace fespace(&mesh, fec);
+   cout << "Number of finite element unknowns: "
+        << fespace.GetTrueVSize() << endl;
+
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm b(&fespace);
+   ConstantCoefficient one(1.0);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
+
+   // 8. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(&fespace);
+   x = 0.0;
+
+   // 9. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator.
+   DiffusionIntegrator *di = new DiffusionIntegrator(one);
+
+   if (patchAssembly && reducedIntegration && !pa)
+   {
+      di->SetIntegrationMode(NonlinearFormIntegrator::Mode::PATCHWISE_REDUCED);
+   }
+   else if (patchAssembly)
+   {
+      di->SetIntegrationMode(NonlinearFormIntegrator::Mode::PATCHWISE);
+   }
+
+   NURBSMeshRules *patchRule = nullptr;
+   if (order < 0)
+   {
+      if (ir_order == -1) { ir_order = 2*fec->GetOrder(); }
+      cout << "Using ir_order " << ir_order << endl;
+
+      patchRule = new NURBSMeshRules(mesh.NURBSext->GetNP(), dim);
+      // Loop over patches and set a different rule for each patch.
+      for (int p=0; p<mesh.NURBSext->GetNP(); ++p)
+      {
+         Array<const KnotVector*> kv(dim);
+         mesh.NURBSext->GetPatchKnotVectors(p, kv);
+
+         std::vector<const IntegrationRule*> ir1D(dim);
+         const IntegrationRule *ir = &IntRules.Get(Geometry::SEGMENT, ir_order);
+
+         // Construct 1D integration rules by applying the rule ir to each
+         // knot span.
+         for (int i=0; i<dim; ++i)
+         {
+            ir1D[i] = ir->ApplyToKnotIntervals(*kv[i]);
+         }
+
+         patchRule->SetPatchRules1D(p, ir1D);
+      }  // loop (p) over patches
+
+      patchRule->Finalize(mesh);
+      di->SetNURBSPatchIntRule(patchRule);
+   }
+
+   // 10. Assemble and solve the linear system
+   cout << "Assembling system patch-wise and solving" << endl;
+   AssembleAndSolve(b, di, ess_tdof_list, pa, algebraic_ceed, x);
+
+   delete patchRule;
+
+   // 11. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh.Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 12. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << x << flush;
+   }
+
+   // 13. Optionally assemble element-wise and solve the linear system, to
+   //     compare timings and compute relative error.
+   if (compareToElementWise)
+   {
+      Vector x_pw, x_ew;
+      x.GetTrueDofs(x_pw);
+
+      cout << "Assembling system element-wise and solving" << endl;
+      DiffusionIntegrator *d = new DiffusionIntegrator(one);
+      // Element-wise partial assembly is not supported on NURBS meshes, so we
+      // pass pa = false here.
+      AssembleAndSolve(b, d, ess_tdof_list, false, algebraic_ceed, x);
+
+      x.GetTrueDofs(x_ew);
+
+      const double solNorm = x_ew.Norml2();
+      x_ew -= x_pw;
+
+      cout << "Element-wise solution norm " << solNorm << endl;
+      cout << "Relative error of patch-wise solution "
+           << x_ew.Norml2() / solNorm << endl;
+   }
+
+   // 14. Free the used memory.
+   if (delete_fec)
+   {
+      delete fec;
+   }
+
+   return 0;
+}
+
+// This function deletes bfi when the BilinearForm goes out of scope.
+void AssembleAndSolve(LinearForm & b, BilinearFormIntegrator * bfi,
+                      Array<int> const& ess_tdof_list, const bool pa,
+                      const bool algebraic_ceed, GridFunction & x)
+{
+   FiniteElementSpace *fespace = b.FESpace();
+   BilinearForm a(fespace);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+
+   a.AddDomainIntegrator(bfi);  // Takes ownership of bfi
+
+   StopWatch sw;
+   sw.Start();
+
+   // Assemble the bilinear form and the corresponding linear system, applying
+   // any necessary transformations such as: eliminating boundary conditions,
+   // applying conforming constraints for non-conforming AMR, etc.
+   a.Assemble();
+
+   sw.Stop();
+
+   const double timeAssemble = sw.RealTime();
+
+   sw.Clear();
+   sw.Start();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   sw.Stop();
+
+   const double timeFormLinearSystem = sw.RealTime();
+
+   cout << "Timing for Assemble: " << timeAssemble << " seconds" << endl;
+   cout << "Timing for FormLinearSystem: " << timeFormLinearSystem << " seconds"
+        << endl;
+   cout << "Timing for entire setup: " << timeAssemble + timeFormLinearSystem
+        << " seconds" << endl;
+
+   sw.Clear();
+   sw.Start();
+
+   // Solve the linear system A X = B.
+   if (!pa)
+   {
+#ifndef MFEM_USE_SUITESPARSE
+      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+      GSSmoother M((SparseMatrix&)(*A));
+      PCG(*A, M, B, X, 1, 200, 1e-20, 0.0);
+#else
+      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(*A);
+      umf_solver.Mult(B, X);
+#endif
+   }
+   else
+   {
+      if (UsesTensorBasis(*fespace))
+      {
+         if (algebraic_ceed)
+         {
+            ceed::AlgebraicSolver M(a, ess_tdof_list);
+            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+         }
+         else
+         {
+            OperatorJacobiSmoother M(a, ess_tdof_list);
+            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+         }
+      }
+      else
+      {
+         CG(*A, B, X, 1, 400, 1e-20, 0.0);
+      }
+   }
+
+   sw.Stop();
+   cout << "Timing for solve " << sw.RealTime() << endl;
+
+   // Recover the solution as a finite element grid function.
+   a.RecoverFEMSolution(X, b, x);
+}

miniapps/tools/CMakeLists.txt

@@ -28,5 +28,10 @@ add_mfem_miniapp(convert-dc
 add_mfem_miniapp(lor-transfer
   MAIN lor-transfer.cpp LIBRARIES mfem)
 
+if (MFEM_USE_MPI)
+  add_mfem_miniapp(plor-transfer
+    MAIN plor-transfer.cpp LIBRARIES mfem)
+endif()
+
 add_mfem_miniapp(check-tmop-metric
   MAIN check-tmop-metric.cpp LIBRARIES mfem)

miniapps/tools/lor-transfer.cpp

@@ -95,13 +95,7 @@ int main(int argc, char *argv[])
    args.AddOption(&use_pointwise_transfer, "-t", "--use-pointwise-transfer",
                   "-no-t", "--dont-use-pointwise-transfer",
                   "Use pointwise transfer operators instead of L2 projection.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
+   args.ParseCheck();
 
    // Read the mesh from the given mesh file.
    Mesh mesh(mesh_file, 1, 1);
@@ -157,6 +151,10 @@ int main(int argc, char *argv[])
    direction = "HO -> LOR @ HO";
    FunctionCoefficient RHO(RHO_exact);
    rho.ProjectCoefficient(RHO);
+   // Make sure AMR constraints are satisfied
+   rho.SetTrueVector();
+   rho.SetFromTrueVector();
+
    double ho_mass = compute_mass(&fespace, -1.0, HO_dc, "HO       ");
    if (vis) { visualize(HO_dc, "HO", Wx, Wy); Wx += offx; }
 
@@ -193,17 +191,13 @@ int main(int argc, char *argv[])
    }
 
    // HO* to LOR* dual fields
-   GridFunction ones(&fespace), ones_lor(&fespace_lor);
-   ones = 1.0;
-   ones_lor = 1.0;
    LinearForm M_rho(&fespace), M_rho_lor(&fespace_lor);
    if (!use_pointwise_transfer && gt->SupportsBackwardsOperator())
    {
       const Operator &P = gt->BackwardOperator();
       M_ho.Mult(rho, M_rho);
       P.MultTranspose(M_rho, M_rho_lor);
-      cout << "HO -> LOR dual field: " << fabs(M_rho(ones)-M_rho_lor(ones_lor))
-           << endl << endl;
+      cout << "HO -> LOR dual field: " << abs(M_rho.Sum()-M_rho_lor.Sum()) << "\n\n";
    }
 
    // LOR projections
@@ -239,8 +233,7 @@ int main(int argc, char *argv[])
    {
       M_lor.Mult(rho_lor, M_rho_lor);
       R.MultTranspose(M_rho_lor, M_rho);
-      cout << "LOR -> HO dual field: " << fabs(M_rho(ones)-M_rho_lor(ones_lor))
-           << '\n';
+      cout << "LOR -> HO dual field: " << abs(M_rho.Sum() - M_rho_lor.Sum()) << '\n';
    }
 
    delete fec;
@@ -288,14 +281,11 @@ double compute_mass(FiniteElementSpace *L2, double massL2,
                     VisItDataCollection &dc, string prefix)
 {
    ConstantCoefficient one(1.0);
-   BilinearForm ML2(L2);
-   ML2.AddDomainIntegrator(new MassIntegrator(one));
-   ML2.Assemble();
+   LinearForm lf(L2);
+   lf.AddDomainIntegrator(new DomainLFIntegrator(one));
+   lf.Assemble();
 
-   GridFunction rhoone(L2);
-   rhoone = 1.0;
-
-   double newmass = ML2.InnerProduct(*dc.GetField("density"),rhoone);
+   double newmass = lf(*dc.GetField("density"));
    cout.precision(18);
    cout << space << " " << prefix << " mass   = " << newmass;
    if (massL2 >= 0)

miniapps/tools/makefile

@@ -28,7 +28,7 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_MINIAPPS = display-basis load-dc convert-dc get-values lor-transfer \
 	check-tmop-metric tmop-metric-magnitude
-PAR_MINIAPPS =
+PAR_MINIAPPS = plor-transfer
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
 else
@@ -81,9 +81,9 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<,, Tools miniapp)
 
 # Testing: Specific execution options
-# Do not test: display-basis, load-dc, convert-dc, get-values, lor-transfer
+# Do not test: display-basis, load-dc, convert-dc, get-values, lor-transfer, plor-transfer
 NO_TEST_APPS = display-basis load-dc convert-dc get-values lor-transfer \
-	check-tmop-metric tmop-metric-magnitude
+	plor-transfer check-tmop-metric tmop-metric-magnitude
 $(foreach app,$(NO_TEST_APPS),$(app)-test-seq $(app)-test-par):
 	@true
 

miniapps/tools/plor-transfer.cpp

@@ -0,0 +1,360 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//   -----------------------------------------------------------------------
+//   Parallel LOR Transfer Miniapp:  Map functions between HO and LOR spaces
+//   -----------------------------------------------------------------------
+//
+// This miniapp visualizes the maps between a high-order (HO) finite element
+// space, typically using high-order functions on a high-order mesh, and a
+// low-order refined (LOR) finite element space, typically defined by 0th or 1st
+// order functions on a low-order refinement of the HO mesh.
+//
+// The grid transfer operators are represented using either
+// InterpolationGridTransfer or L2ProjectionGridTransfer (depending on the
+// options requested by the user). The two transfer operators are then:
+//
+//  1. R: HO -> LOR, defined by GridTransfer::ForwardOperator
+//  2. P: LOR -> HO, defined by GridTransfer::BackwardOperator
+//
+// While defined generally, these operators have some nice properties for
+// particular finite element spaces. For example they satisfy PR=I, plus mass
+// conservation in both directions for L2 fields.
+//
+// Compile with: make plor-transfer
+//
+// Sample runs:  plor-transfer
+//               plor-transfer -h1
+//               plor-transfer -t
+//               plor-transfer -m ../../data/star-q2.mesh -lref 5 -p 4
+//               plor-transfer -m ../../data/star-mixed.mesh -lref 3 -p 2
+//               plor-transfer -lref 4 -o 4 -lo 0 -p 1
+//               plor-transfer -lref 5 -o 4 -lo 0 -p 1
+//               plor-transfer -lref 5 -o 4 -lo 3 -p 2
+//               plor-transfer -lref 5 -o 4 -lo 0 -p 3
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int problem = 1; // problem type
+
+int Wx = 0, Wy = 0; // window position
+int Ww = 350, Wh = 350; // window size
+int offx = Ww+5, offy = Wh+25; // window offsets
+
+string space;
+string direction;
+
+// Exact functions to project
+double RHO_exact(const Vector &x);
+
+// Helper functions
+void visualize(VisItDataCollection &, string, int, int);
+double compute_mass(ParFiniteElementSpace *, double, VisItDataCollection &,
+                    string);
+
+int main(int argc, char *argv[])
+{
+   // Initialize MPI and HYPRE.
+   Mpi::Init(argc, argv);
+   Hypre::Init();
+
+   // Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int order = 3;
+   int lref = order+1;
+   int lorder = 0;
+   bool vis = true;
+   bool useH1 = false;
+   bool use_pointwise_transfer = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&problem, "-p", "--problem",
+                  "Problem type (see the RHO_exact function).");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&lref, "-lref", "--lor-ref-level", "LOR refinement level.");
+   args.AddOption(&lorder, "-lo", "--lor-order",
+                  "LOR space order (polynomial degree, zero by default).");
+   args.AddOption(&vis, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&useH1, "-h1", "--use-h1", "-l2", "--use-l2",
+                  "Use H1 spaces instead of L2.");
+   args.AddOption(&use_pointwise_transfer, "-t", "--use-pointwise-transfer",
+                  "-no-t", "--dont-use-pointwise-transfer",
+                  "Use pointwise transfer operators instead of L2 projection.");
+   args.ParseCheck();
+
+   // Read the mesh from the given mesh file.
+   Mesh serial_mesh(mesh_file, 1, 1);
+   ParMesh mesh(MPI_COMM_WORLD, serial_mesh);
+   serial_mesh.Clear();
+   int dim = mesh.Dimension();
+
+   // Create the low-order refined mesh
+   int basis_lor = BasisType::GaussLobatto; // BasisType::ClosedUniform;
+   ParMesh mesh_lor = ParMesh::MakeRefined(mesh, lref, basis_lor);
+
+   // Create spaces
+   FiniteElementCollection *fec, *fec_lor;
+   if (useH1)
+   {
+      space = "H1";
+      if (lorder == 0)
+      {
+         lorder = 1;
+         if (Mpi::Root())
+         {
+            cerr << "Switching the H1 LOR space order from 0 to 1\n";
+         }
+      }
+      fec = new H1_FECollection(order, dim);
+      fec_lor = new H1_FECollection(lorder, dim);
+   }
+   else
+   {
+      space = "L2";
+      fec = new L2_FECollection(order, dim);
+      fec_lor = new L2_FECollection(lorder, dim);
+   }
+
+   ParFiniteElementSpace fespace(&mesh, fec);
+   ParFiniteElementSpace fespace_lor(&mesh_lor, fec_lor);
+
+   ParGridFunction rho(&fespace);
+   ParGridFunction rho_lor(&fespace_lor);
+
+   // Data collections for vis/analysis
+   VisItDataCollection HO_dc(MPI_COMM_WORLD, "HO", &mesh);
+   HO_dc.RegisterField("density", &rho);
+   VisItDataCollection LOR_dc(MPI_COMM_WORLD, "LOR", &mesh_lor);
+   LOR_dc.RegisterField("density", &rho_lor);
+
+   ParBilinearForm M_ho(&fespace);
+   M_ho.AddDomainIntegrator(new MassIntegrator);
+   M_ho.Assemble();
+   M_ho.Finalize();
+   HypreParMatrix* M_ho_tdof = M_ho.ParallelAssemble();
+
+   ParBilinearForm M_lor(&fespace_lor);
+   M_lor.AddDomainIntegrator(new MassIntegrator);
+   M_lor.Assemble();
+   M_lor.Finalize();
+   HypreParMatrix* M_lor_tdof = M_lor.ParallelAssemble();
+
+   // HO projections
+   direction = "HO -> LOR @ HO";
+   FunctionCoefficient RHO(RHO_exact);
+   rho.ProjectCoefficient(RHO);
+   // Make sure AMR constraints are satisfied
+   rho.SetTrueVector();
+   rho.SetFromTrueVector();
+
+   double ho_mass = compute_mass(&fespace, -1.0, HO_dc, "HO       ");
+   if (vis) { visualize(HO_dc, "HO", Wx, Wy); Wx += offx; }
+
+   GridTransfer *gt;
+   if (use_pointwise_transfer)
+   {
+      gt = new InterpolationGridTransfer(fespace, fespace_lor);
+   }
+   else
+   {
+      gt = new L2ProjectionGridTransfer(fespace, fespace_lor);
+   }
+   const Operator &R = gt->ForwardOperator();
+
+   // HO->LOR restriction
+   direction = "HO -> LOR @ LOR";
+   R.Mult(rho, rho_lor);
+   compute_mass(&fespace_lor, ho_mass, LOR_dc, "R(HO)    ");
+   if (vis) { visualize(LOR_dc, "R(HO)", Wx, Wy); Wx += offx; }
+   auto global_max = [](const Vector& v)
+   {
+      double max = v.Normlinf();
+      MPI_Allreduce(MPI_IN_PLACE, &max, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
+      return max;
+   };
+
+   if (gt->SupportsBackwardsOperator())
+   {
+      const Operator &P = gt->BackwardOperator();
+      // LOR->HO prolongation
+      direction = "HO -> LOR @ HO";
+      ParGridFunction rho_prev = rho;
+      P.Mult(rho_lor, rho);
+      compute_mass(&fespace, ho_mass, HO_dc, "P(R(HO)) ");
+      if (vis) { visualize(HO_dc, "P(R(HO))", Wx, Wy); Wx = 0; Wy += offy; }
+
+      rho_prev -= rho;
+      Vector rho_prev_true(fespace.GetTrueVSize());
+      rho_prev.GetTrueDofs(rho_prev_true);
+      double l_inf = global_max(rho_prev_true);
+      if (Mpi::Root())
+      {
+         cout.precision(12);
+         cout << "|HO - P(R(HO))|_∞   = " << l_inf << endl;
+      }
+   }
+
+   // HO* to LOR* dual fields
+   ParLinearForm M_rho(&fespace), M_rho_lor(&fespace_lor);
+   auto global_sum = [](const Vector& v)
+   {
+      double sum = v.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+      return sum;
+   };
+   if (!use_pointwise_transfer && gt->SupportsBackwardsOperator())
+   {
+      Vector M_rho_true(fespace.GetTrueVSize());
+      M_ho_tdof->Mult(rho.GetTrueVector(), M_rho_true);
+      fespace.GetRestrictionOperator()->MultTranspose(M_rho_true, M_rho);
+      const Operator &P = gt->BackwardOperator();
+      P.MultTranspose(M_rho, M_rho_lor);
+      double ho_dual_mass = global_sum(M_rho);
+      double lor_dual_mass = global_sum(M_rho_lor);
+      if (Mpi::Root())
+      {
+         cout << "HO -> LOR dual field: " << abs(ho_dual_mass - lor_dual_mass) << "\n\n";
+      }
+   }
+
+   // LOR projections
+   direction = "LOR -> HO @ LOR";
+   rho_lor.ProjectCoefficient(RHO);
+   ParGridFunction rho_lor_prev = rho_lor;
+   double lor_mass = compute_mass(&fespace_lor, -1.0, LOR_dc, "LOR      ");
+   if (vis) { visualize(LOR_dc, "LOR", Wx, Wy); Wx += offx; }
+
+   if (gt->SupportsBackwardsOperator())
+   {
+      const Operator &P = gt->BackwardOperator();
+      // Prolongate to HO space
+      direction = "LOR -> HO @ HO";
+      P.Mult(rho_lor, rho);
+      compute_mass(&fespace, lor_mass, HO_dc, "P(LOR)   ");
+      if (vis) { visualize(HO_dc, "P(LOR)", Wx, Wy); Wx += offx; }
+
+      // Restrict back to LOR space. This won't give the original function because
+      // the rho_lor doesn't necessarily live in the range of R.
+      direction = "LOR -> HO @ LOR";
+      R.Mult(rho, rho_lor);
+      compute_mass(&fespace_lor, lor_mass, LOR_dc, "R(P(LOR))");
+      if (vis) { visualize(LOR_dc, "R(P(LOR))", Wx, Wy); }
+
+      rho_lor_prev -= rho_lor;
+      Vector rho_lor_prev_true(fespace_lor.GetTrueVSize());
+      rho_lor_prev.GetTrueDofs(rho_lor_prev_true);
+      double l_inf = global_max(rho_lor_prev_true);
+      if (Mpi::Root())
+      {
+         cout.precision(12);
+         cout << "|LOR - R(P(LOR))|_∞ = " << l_inf << endl;
+      }
+   }
+
+   // LOR* to HO* dual fields
+   if (!use_pointwise_transfer)
+   {
+      Vector M_rho_lor_true(fespace_lor.GetTrueVSize());
+      M_lor_tdof->Mult(rho_lor.GetTrueVector(), M_rho_lor_true);
+      fespace_lor.GetRestrictionOperator()->MultTranspose(M_rho_lor_true,
+                                                          M_rho_lor);
+      R.MultTranspose(M_rho_lor, M_rho);
+      double ho_dual_mass = global_sum(M_rho);
+      double lor_dual_mass = global_sum(M_rho_lor);
+
+      cout << lor_dual_mass << '\n';
+      cout << ho_dual_mass << '\n';
+
+      if (Mpi::Root())
+      {
+         cout << "LOR -> HO dual field: " << abs(ho_dual_mass - lor_dual_mass) << '\n';
+      }
+   }
+
+   delete fec;
+   delete fec_lor;
+   delete M_ho_tdof;
+   delete M_lor_tdof;
+   delete gt;
+
+   return 0;
+}
+
+
+double RHO_exact(const Vector &x)
+{
+   switch (problem)
+   {
+      case 1: // smooth field
+         return x(1)+0.25*cos(2*M_PI*x.Norml2());
+      case 2: // cubic function
+         return x(1)*x(1)*x(1) + 2*x(0)*x(1) + x(0);
+      case 3: // sharp gradient
+         return M_PI/2-atan(5*(2*x.Norml2()-1));
+      case 4: // basis function
+         return (x.Norml2() < 0.1) ? 1 : 0;
+      default:
+         return 1.0;
+   }
+}
+
+
+void visualize(VisItDataCollection &dc, string prefix, int x, int y)
+{
+   int w = Ww, h = Wh;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   socketstream sol_sockL2(vishost, visport);
+   sol_sockL2 << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() <<
+              "\n";
+   sol_sockL2.precision(8);
+   sol_sockL2 << "solution\n" << *dc.GetMesh() << *dc.GetField("density")
+              << "window_geometry " << x << " " << y << " " << w << " " << h
+              << "plot_caption '" << space << " " << prefix << " Density'"
+              << "window_title '" << direction << "'" << flush;
+}
+
+
+double compute_mass(ParFiniteElementSpace *L2, double massL2,
+                    VisItDataCollection &dc, string prefix)
+{
+   ConstantCoefficient one(1.0);
+   ParLinearForm lf(L2);
+   lf.AddDomainIntegrator(new DomainLFIntegrator(one));
+   lf.Assemble();
+
+   double newmass = lf(*dc.GetParField("density"));
+   if (Mpi::Root())
+   {
+      cout.precision(18);
+      cout << space << " " << prefix << " mass   = " << newmass;
+      if (massL2 >= 0)
+      {
+         cout.precision(4);
+         cout << " ("  << fabs(newmass-massL2)*100/massL2 << "%)";
+      }
+      cout << endl;
+   }
+   return newmass;
+}

tests/scripts/branch-history

@@ -155,10 +155,10 @@ foreach my $sha (@commits) {
         my $sz1 = int(`git diff -U0 --binary $src1 $dst | gzip -c | wc -c`);
         my $sz2 = int(`git diff -U0 --binary $src2 $dst | gzip -c | wc -c`);
         $blob_size += $sz1 < $sz2 ? $sz1 : $sz2; }
-      elsif ($mode eq "AM") {
+      elsif ($mode =~ m/A[MR]/) {
         my $sz2 = int(`git diff -U0 --binary $src2 $dst | gzip -c | wc -c`);
         $blob_size += $sz2; }
-      elsif ($mode eq "MA") {
+      elsif ($mode =~ m/[MR]A/) {
         my $sz1 = int(`git diff -U0 --binary $src1 $dst | gzip -c | wc -c`);
         $blob_size += $sz1; }
       else { die "Unknown git status letter: $mode, commit: $sha, file: $fname.\n\t" }

tests/unit/linalg/test_matrix_dense.cpp

@@ -691,4 +691,32 @@ TEST_CASE("Eigensystem Problems",
    }
 }
 
+TEST_CASE("NNLS", "[DenseMatrix]")
+{
+   const int m = 3;
+   const int n = 5;
+   DenseMatrix G(m,n);
+   G = 0.0;
+
+   for (int i=0; i<m; ++i)
+      for (int j=0; j<n; ++j)
+      {
+         G(i,j) = j;
+      }
+
+   Vector w(n);
+   w = 1.0;
+
+   Vector sol(n);
+
+   NNLSSolver nnls;
+   nnls.SetVerbosity(2);
+   nnls.SetOperator(G);
+
+   nnls.Mult(w, sol);
+
+   REQUIRE(sol.Norml2() == MFEM_Approx(2.5));
+   REQUIRE(sol[4] == MFEM_Approx(2.5));
+}
+
 #endif // if MFEM_USE_LAPACK

tests/unit/mesh/test_ncmesh.cpp

@@ -294,9 +294,110 @@ TEST_CASE("pNCMesh PA diagonal",  "[Parallel], [NCMesh]")
          MPI_Barrier(MPI_COMM_WORLD);
       }
    }
-
 } // test case
 
+
+// Given a parallel and a serial mesh, perform an L2 projection and check the
+// solutions match exactly.
+void CheckL2Projection(ParMesh& pmesh, Mesh& smesh, int order,
+                       std::function<double(Vector const&)> exact_soln)
+{
+   REQUIRE(pmesh.GetGlobalNE() == smesh.GetNE());
+   REQUIRE(pmesh.Dimension() == smesh.Dimension());
+   REQUIRE(pmesh.SpaceDimension() == smesh.SpaceDimension());
+
+   // Make an H1 space, then a mass matrix operator and invert it.
+   // If all non-conformal constraints have been conveyed correctly, the
+   // resulting DOF should match exactly on the serial and the parallel
+   // solution.
+
+   H1_FECollection fec(order, smesh.Dimension());
+   ConstantCoefficient one(1.0);
+   FunctionCoefficient rhs_coef(exact_soln);
+
+   constexpr double linear_tol = 1e-16;
+
+   // serial solve
+   auto serror = [&]
+   {
+      FiniteElementSpace fes(&smesh, &fec);
+      // solution vectors
+      GridFunction x(&fes);
+      x = 0.0;
+
+      double snorm = x.ComputeL2Error(rhs_coef);
+
+      LinearForm b(&fes);
+      b.AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
+      b.Assemble();
+
+      BilinearForm a(&fes);
+      a.AddDomainIntegrator(new MassIntegrator(one));
+      a.Assemble();
+
+      SparseMatrix A;
+      Vector B, X;
+
+      Array<int> empty_tdof_list;
+      a.FormLinearSystem(empty_tdof_list, x, b, A, X, B);
+
+#ifndef MFEM_USE_SUITESPARSE
+      // 9. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+      //    solve the system AX=B with PCG.
+      GSSmoother M(A);
+      PCG(A, M, B, X, -1, 500, linear_tol, 0.0);
+#else
+      // 9. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(A);
+      umf_solver.Mult(B, X);
+#endif
+
+      a.RecoverFEMSolution(X, b, x);
+      return x.ComputeL2Error(rhs_coef) / snorm;
+   }();
+
+   auto perror = [&]
+   {
+      // parallel solve
+      ParFiniteElementSpace fes(&pmesh, &fec);
+      ParLinearForm b(&fes);
+
+      ParGridFunction x(&fes);
+      x = 0.0;
+
+      double pnorm = x.ComputeL2Error(rhs_coef);
+
+      b.AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
+      b.Assemble();
+
+      ParBilinearForm a(&fes);
+      a.AddDomainIntegrator(new MassIntegrator(one));
+      a.Assemble();
+
+      HypreParMatrix A;
+      Vector B, X;
+      Array<int> empty_tdof_list;
+      a.FormLinearSystem(empty_tdof_list, x, b, A, X, B);
+
+      HypreBoomerAMG amg(A);
+      HyprePCG pcg(A);
+      amg.SetPrintLevel(-1);
+      pcg.SetTol(linear_tol);
+      pcg.SetMaxIter(500);
+      pcg.SetPrintLevel(-1);
+      pcg.SetPreconditioner(amg);
+      pcg.Mult(B, X);
+      a.RecoverFEMSolution(X, b, x);
+      return x.ComputeL2Error(rhs_coef) / pnorm;
+   }();
+
+   constexpr double test_tol = 1e-9;
+   CHECK(std::abs(serror - perror) < test_tol);
+};
+
+
 TEST_CASE("FaceEdgeConstraint",  "[Parallel], [NCMesh]")
 {
    constexpr int refining_rank = 0;
@@ -321,103 +422,6 @@ TEST_CASE("FaceEdgeConstraint",  "[Parallel], [NCMesh]")
       return std::sin(d * d);
    };
 
-   // Given a parallel and a serial mesh, perform an L2 projection and check the
-   // solutions match exactly.
-   auto check_l2_projection = [&exact_soln](ParMesh& pmesh, Mesh& smesh, int order)
-   {
-
-      REQUIRE(pmesh.GetGlobalNE() == smesh.GetNE());
-      REQUIRE(pmesh.Dimension() == smesh.Dimension());
-      REQUIRE(pmesh.SpaceDimension() == smesh.SpaceDimension());
-
-      // Make an H1 space, then a mass matrix operator and invert it.
-      // If all non-conformal constraints have been conveyed correctly, the
-      // resulting DOF should match exactly on the serial and the parallel
-      // solution.
-
-      H1_FECollection fec(order, smesh.Dimension());
-      ConstantCoefficient one(1.0);
-      FunctionCoefficient rhs_coef(exact_soln);
-
-      constexpr double linear_tol = 1e-16;
-
-      // serial solve
-      auto serror = [&]
-      {
-         FiniteElementSpace fes(&smesh, &fec);
-         // solution vectors
-         GridFunction x(&fes);
-         x = 0.0;
-
-         LinearForm b(&fes);
-         b.AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
-         b.Assemble();
-
-         BilinearForm a(&fes);
-         a.AddDomainIntegrator(new MassIntegrator(one));
-         a.Assemble();
-
-         SparseMatrix A;
-         Vector B, X;
-
-         Array<int> empty_tdof_list;
-         a.FormLinearSystem(empty_tdof_list, x, b, A, X, B);
-
-#ifndef MFEM_USE_SUITESPARSE
-         // 9. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-         //    solve the system AX=B with PCG.
-         GSSmoother M(A);
-         PCG(A, M, B, X, -1, 500, linear_tol, 0.0);
-#else
-         // 9. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-         UMFPackSolver umf_solver;
-         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-         umf_solver.SetOperator(A);
-         umf_solver.Mult(B, X);
-#endif
-
-         a.RecoverFEMSolution(X, b, x);
-         return x.ComputeL2Error(rhs_coef);
-      }();
-
-      auto perror = [&]
-      {
-         // parallel solve
-         ParFiniteElementSpace fes(&pmesh, &fec);
-         ParLinearForm b(&fes);
-
-         ParGridFunction x(&fes);
-         x = 0.0;
-
-         b.AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
-         b.Assemble();
-
-         ParBilinearForm a(&fes);
-         a.AddDomainIntegrator(new MassIntegrator(one));
-         a.Assemble();
-
-         HypreParMatrix A;
-         Vector B, X;
-         Array<int> empty_tdof_list;
-         a.FormLinearSystem(empty_tdof_list, x, b, A, X, B);
-
-         HypreBoomerAMG amg(A);
-         HyprePCG pcg(A);
-         amg.SetPrintLevel(-1);
-         pcg.SetTol(linear_tol);
-         pcg.SetMaxIter(500);
-         pcg.SetPrintLevel(-1);
-         pcg.SetPreconditioner(amg);
-         pcg.Mult(B, X);
-         a.RecoverFEMSolution(X, b, x);
-         return x.ComputeL2Error(rhs_coef);
-      }();
-
-      constexpr double test_tol = 1e-9;
-      CHECK(std::abs(serror - perror) < test_tol);
-
-   };
-
    REQUIRE(smesh.GetNE() == 2);
    smesh.EnsureNCMesh(true);
    smesh.Finalize();
@@ -498,27 +502,267 @@ TEST_CASE("FaceEdgeConstraint",  "[Parallel], [NCMesh]")
       for (int iface = 0; iface < sttmp.GetNumFaces(); ++iface)
       {
          const auto face_transform = sttmp.GetFaceElementTransformations(iface);
-
          CHECK(face_transform->CheckConsistency(0) < 1e-12);
       }
 
       for (int iface = 0; iface < ttmp.GetNumFacesWithGhost(); ++iface)
       {
          const auto face_transform = ttmp.GetFaceElementTransformations(iface);
-
          CHECK(face_transform->CheckConsistency(0) < 1e-12);
       }
 
       // Use P4 to ensure there's a few fully interior DOF.
-      check_l2_projection(ttmp, sttmp, 4);
+      CheckL2Projection(ttmp, sttmp, 4, exact_soln);
 
       ttmp.ExchangeFaceNbrData();
       ttmp.Rebalance();
 
-      check_l2_projection(ttmp, sttmp, 4);
+      CheckL2Projection(ttmp, sttmp, 4, exact_soln);
    }
 } // test case
 
+Mesh CylinderMesh(Geometry::Type el_type, bool quadratic, int variant = 0)
+{
+   double c[3];
+
+   int nnodes = (el_type == Geometry::CUBE) ? 24 : 15;
+   int nelems = 8; // Geometry::PRISM
+   if (el_type == Geometry::CUBE)        { nelems = 10; }
+   if (el_type == Geometry::TETRAHEDRON) { nelems = 24; }
+
+   Mesh mesh(3, nnodes, nelems);
+
+   for (int i=0; i<3; i++)
+   {
+      if (el_type != Geometry::CUBE)
+      {
+         c[0] = 0.0;  c[1] = 0.0;  c[2] = 2.74 * i;
+         mesh.AddVertex(c);
+      }
+
+      for (int j=0; j<4; j++)
+      {
+         if (el_type == Geometry::CUBE)
+         {
+            c[0] = 1.14 * ((j + 1) % 2) * (1 - j);
+            c[1] = 1.14 * (j % 2) * (2 - j);
+            c[2] = 2.74 * i;
+            mesh.AddVertex(c);
+         }
+
+         c[0] = 2.74 * ((j + 1) % 2) * (1 - j);
+         c[1] = 2.74 * (j % 2) * (2 - j);
+         c[2] = 2.74 * i;
+         mesh.AddVertex(c);
+      }
+   }
+
+   for (int i=0; i<2; i++)
+   {
+      if (el_type == Geometry::CUBE)
+      {
+         mesh.AddHex(8*i, 8*i+2, 8*i+4, 8*i+6,
+                     8*(i+1), 8*(i+1)+2, 8*(i+1)+4, 8*(i+1)+6);
+      }
+
+      for (int j=0; j<4; j++)
+      {
+         if (el_type == Geometry::PRISM)
+         {
+            switch (variant)
+            {
+               case 0:
+                  mesh.AddWedge(5*i, 5*i+j+1, 5*i+(j+1)%4+1,
+                                5*(i+1), 5*(i+1)+j+1, 5*(i+1)+(j+1)%4+1);
+                  break;
+               case 1:
+                  mesh.AddWedge(5*i, 5*i+j+1, 5*i+(j+1)%4+1,
+                                5*(i+1), 5*(i+1)+j+1, 5*(i+1)+(j+1)%4+1);
+                  break;
+               case 2:
+                  mesh.AddWedge(5*i+(j+1)%4+1, 5*i, 5*i+j+1,
+                                5*(i+1)+(j+1)%4+1, 5*(i+1), 5*(i+1)+j+1);
+                  break;
+            }
+         }
+         else if (el_type == Geometry::CUBE)
+         {
+            mesh.AddHex(8*i+2*j, 8*i+2*j+1, 8*i+(2*j+3)%8, 8*i+(2*j+2)%8,
+                        8*(i+1)+2*j, 8*(i+1)+2*j+1, 8*(i+1)+(2*j+3)%8,
+                        8*(i+1)+(2*j+2)%8);
+         }
+         else if (el_type == Geometry::TETRAHEDRON)
+         {
+            mesh.AddTet(5*i, 5*i+j+1, 5*i+(j+1)%4+1, 5*(i+1));
+            mesh.AddTet(5*i+j+1, 5*i+(j+1)%4+1, 5*(i+1), 5*(i+1)+j+1);
+            mesh.AddTet(5*i+(j+1)%4+1, 5*(i+1), 5*(i+1)+j+1, 5*(i+1)+(j+1)%4+1);
+         }
+      }
+   }
+
+   mesh.FinalizeTopology();
+
+   if (quadratic)
+   {
+      mesh.SetCurvature(2);
+
+      if (el_type == Geometry::CUBE)
+      {
+         auto quad_cyl_hex = [](const Vector& x, Vector& d)
+         {
+            d.SetSize(3);
+            d = x;
+            const double Rmax = 2.74;
+            const double Rmin = 1.14;
+            double ax = std::abs(x[0]);
+            if (ax <= 1e-6) { return; }
+            double ay = std::abs(x[1]);
+            if (ay <= 1e-6) { return; }
+            double r = ax + ay;
+            if (r <= Rmin + 1e-6) { return; }
+
+            double sx = std::copysign(1.0, x[0]);
+            double sy = std::copysign(1.0, x[1]);
+
+            double R = (Rmax - Rmin) * Rmax / (r - Rmin);
+            double r2 = r * r;
+            double R2 = R * R;
+
+            double acosarg = 0.5 * (r + std::sqrt(2.0 * R2 - r2)) / R;
+            double tR = std::acos(std::min(acosarg, 1.0));
+            double tQ = (1.0 + sx * sy * (ay - ax) / r);
+            double tP = 0.25 * M_PI * (3.0 - (2.0 + sx) * sy);
+
+            double t = tR + (0.25 * M_PI - tR) * tQ + tP;
+
+            double s0 = std::sqrt(2.0 * R2 - r2);
+            double s1 = 0.25 * std::pow(r + s0, 2);
+            double s = std::sqrt(R2 - s1);
+
+            d[0] = R * std::cos(t) - sx * s;
+            d[1] = R * std::sin(t) - sy * s;
+
+            return;
+         };
+
+         mesh.Transform(quad_cyl_hex);
+      }
+      else
+      {
+         auto quad_cyl = [](const Vector& x, Vector& d)
+         {
+            d.SetSize(3);
+            d = x;
+            double ax = std::abs(x[0]);
+            double ay = std::abs(x[1]);
+            double r = ax + ay;
+            if (r < 1e-6) { return; }
+
+            double sx = std::copysign(1.0, x[0]);
+            double sy = std::copysign(1.0, x[1]);
+
+            double t = ((2.0 - (1.0 + sx) * sy) * ax +
+                        (2.0 - sy) * ay) * 0.5 * M_PI / r;
+            d[0] = r * std::cos(t);
+            d[1] = r * std::sin(t);
+
+            return;
+         };
+
+         mesh.Transform(quad_cyl);
+      }
+   }
+
+   mesh.Finalize(true);
+
+   return mesh;
+}
+
+TEST_CASE("P2Q1PureTetHexPri",  "[Parallel], [NCMesh]")
+{
+   auto exact_soln = [](const Vector& x)
+   {
+      // sin(|| x - d ||^2) -> non polynomial but very smooth.
+      Vector d(3);
+      d[0] = -0.5; d[1] = -1; d[2] = -2; // arbitrary
+      d -= x;
+      return std::sin(d * d);
+   };
+
+   auto el_type = GENERATE(Geometry::TETRAHEDRON,
+                           Geometry::CUBE,
+                           Geometry::PRISM);
+   int variant = GENERATE(0,1,2);
+
+   if (variant > 0 && el_type != Geometry::PRISM)
+   {
+      return;
+   }
+
+   CAPTURE(el_type, variant);
+
+   auto smesh = CylinderMesh(el_type, false, variant);
+
+   for (auto ref : {0,1,2})
+   {
+      if (ref == 1) { smesh.UniformRefinement(); }
+
+      smesh.EnsureNCMesh(true);
+
+      if (ref == 2) { smesh.UniformRefinement(); }
+
+      smesh.Finalize();
+
+      auto pmesh = ParMesh(MPI_COMM_WORLD, smesh);
+
+      // P2 ensures there are triangles without dofs
+      CheckL2Projection(pmesh, smesh, 2, exact_soln);
+   }
+} // test case
+
+TEST_CASE("PNQ2PureTetHexPri",  "[Parallel], [NCMesh]")
+{
+   auto exact_soln = [](const Vector& x)
+   {
+      // sin(|| x - d ||^2) -> non polynomial but very smooth.
+      Vector d(3);
+      d[0] = -0.5; d[1] = -1; d[2] = -2; // arbitrary
+      d -= x;
+      return std::sin(d * d);
+   };
+
+   auto el_type = GENERATE(Geometry::TETRAHEDRON,
+                           Geometry::CUBE,
+                           Geometry::PRISM);
+   int variant = GENERATE(0,1,2);
+
+   if (variant > 0 && el_type != Geometry::PRISM)
+   {
+      return;
+   }
+
+   CAPTURE(el_type, variant);
+
+   auto smesh = CylinderMesh(el_type, true);
+
+   for (auto ref : {0,1,2})
+   {
+      if (ref == 1) { smesh.UniformRefinement(); }
+
+      smesh.EnsureNCMesh(true);
+
+      if (ref == 2) { smesh.UniformRefinement(); }
+
+      smesh.Finalize();
+
+      auto pmesh = ParMesh(MPI_COMM_WORLD, smesh);
+
+      for (int p = 1; p < 3; ++p)
+      {
+         CheckL2Projection(pmesh, smesh, p, exact_soln);
+      }
+   }
+} // test case
 
 #endif // MFEM_USE_MPI