Cleaned up code

Was changing the size of the work vector `z` when should have been
using `w` instead. This would cause EliminateRHS to fail if it
was called after Mult

CHANGELOG

@@ -68,7 +68,7 @@ Discretization improvements
   DofToQuad and GeometricFactors (associated with the classes BilinearForm,
   FiniteElementSpace, FiniteElement and Mesh, respectively). The kernels for
   partial assembled Setup/Assembly and Action/Mult are implemented in the
-  BilinearFormIntegrator methods AssemblePA and AddMultPA.
+  BilinearFormIntegrator methods Setup and AddMultPA.
 
 - Added support for a general "low-order refined"-to-"high-order" transfer of
   GridFunction data from a "low-order refined" (LOR) space defined on a refined

CMakeLists.txt

@@ -450,7 +450,9 @@ add_subdirectory(examples EXCLUDE_FROM_ALL)
 # Create a target for all miniapps and, optionally, enable it.
 set(MFEM_ALL_MINIAPPS_TARGET_NAME miniapps)
 add_mfem_target(${MFEM_ALL_MINIAPPS_TARGET_NAME} ${MFEM_ENABLE_MINIAPPS})
-add_subdirectory(miniapps EXCLUDE_FROM_ALL)
+if(${MFEM_ENABLE_MINIAPPS})
+  add_subdirectory(miniapps)
+endif()
 
 # Target to build all executables, i.e. everything.
 add_custom_target(exec)

config/defaults.mk

@@ -43,7 +43,7 @@ CUDA_CXX = nvcc
 CUDA_ARCH = sm_60
 CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
 # Prefixes for passing flags to the host compiler and linker when using CUDA_CXX
-CUDA_XCOMPILER = -Xcompiler=
+CUDA_XCOMPILER = -Xcompiler 
 CUDA_XLINKER   = -Xlinker=
 
 # HIP configuration options
@@ -202,7 +202,7 @@ MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
 SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include
-SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
+SUITESPARSE_LIB = $(XCOMPILER)\\"-Wl,-rpath,$(SUITESPARSE_DIR)/lib\\" -L$(SUITESPARSE_DIR)/lib\
  -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd -lccolamd\
  -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)
 

examples/ex2.cpp

@@ -47,8 +47,8 @@ using namespace mfem;
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options.
-   const char *mesh_file = "../data/beam-tri.mesh";
-   int order = 1;
+   const char *mesh_file = "../data/beam-quad.mesh";
+   int order = 2;
    bool static_cond = false;
    bool visualization = 1;
 
@@ -96,7 +96,7 @@ int main(int argc, char *argv[])
    //    elements.
    {
       int ref_levels =
-         (int)floor(log(5000./mesh->GetNE())/log(2.)/dim);
+         (int)floor(log(500./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
@@ -168,11 +168,11 @@ int main(int argc, char *argv[])
    //    constants coefficient lambda and mu.
    Vector lambda(mesh->attributes.Max());
    lambda = 1.0;
-   lambda(0) = lambda(1)*50;
+   // lambda(0) = lambda(1)*5;
    PWConstCoefficient lambda_func(lambda);
    Vector mu(mesh->attributes.Max());
    mu = 1.0;
-   mu(0) = mu(1)*50;
+   // mu(0) = mu(1)*5;
    PWConstCoefficient mu_func(mu);
 
    BilinearForm *a = new BilinearForm(fespace);
@@ -194,10 +194,37 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A.Height() << endl;
 
 #ifndef MFEM_USE_SUITESPARSE
-   // 11. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   // 11. Define a simple p-multigrid preconditioner and use it to
    //     solve the system Ax=b with PCG.
-   GSSmoother M(A);
+   Array<int> orders;
+   orders.Append(order);
+   int coarseOrder = order / 2;
+   while (coarseOrder > 0)
+   {
+      orders.Append(coarseOrder);
+      coarseOrder /= 2;
+   }
+   orders.Sort();
+   SpaceHierarchy spaceHierarchy;
+   Array<H1_FECollection*> collections;
+   for (int level = 0; level < orders.Size() - 1; ++level)
+   {
+      collections.Append(new H1_FECollection(orders[level], dim));
+      FiniteElementSpace *fesp = new FiniteElementSpace(mesh, collections.Last(), dim);
+      spaceHierarchy.AddLevel(mesh, fesp, false, true);
+   }
+   spaceHierarchy.AddLevel(mesh, fespace, false, false);
+   MultigridBilinearForm mgOperator(spaceHierarchy, A,  ess_bdr);
+   MultigridSolver M(&mgOperator, MultigridSolver::CycleType::VCYCLE, 1, 1);
    PCG(A, M, B, X, 1, 500, 1e-8, 0.0);
+   for (int level = 0; level < orders.Size() - 1; ++level)
+   {
+      delete collections[level];
+   }
+
+   // GSSmoother M(A);
+   // PCG(A, M, B, X, 1, 500, 1e-8, 0.0);
+
 #else
    // 11. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
    UMFPackSolver umf_solver;

fem/CMakeLists.txt

@@ -14,7 +14,11 @@ set(SRCS
   bilinearform_ext.cpp
   bilininteg.cpp
   bilininteg_diffusion.cpp
+  bilininteg_gradient.cpp
   bilininteg_mass.cpp
+  bilininteg_vecdiffusion.cpp
+  bilininteg_vecdivergence.cpp
+  bilininteg_vecmass.cpp
   coefficient.cpp
   datacollection.cpp
   eltrans.cpp
@@ -30,6 +34,7 @@ set(SRCS
   lininteg.cpp
   mgbilinearform.cpp
   nonlinearform.cpp
+  nonlinearform_ext.cpp
   nonlininteg.cpp
   spacehierarchy.cpp
   staticcond.cpp
@@ -57,6 +62,7 @@ set(HDRS
   lininteg.hpp
   mgbilinearform.hpp
   nonlinearform.hpp
+  nonlinearform_ext.hpp
   nonlininteg.hpp
   spacehierarchy.hpp
   staticcond.hpp

fem/bilinearform.cpp

@@ -587,6 +587,7 @@ void BilinearForm::ConformingAssemble()
    Finalize(0);
    MFEM_ASSERT(mat, "the BilinearForm is not assembled");
 
+   // TODO3: use new functions
    const SparseMatrix *P = fes->GetConformingProlongation();
    if (!P) { return; } // conforming mesh
 
@@ -630,14 +631,13 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
                                     Vector &b, OperatorHandle &A, Vector &X,
                                     Vector &B, int copy_interior)
 {
-   const SparseMatrix *P = fes->GetConformingProlongation();
-
    if (ext)
    {
       ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
       return;
    }
-
+   // TODO3: use new functions
+   const SparseMatrix *P = fes->GetConformingProlongation();
    FormSystemMatrix(ess_tdof_list, A);
 
    // Transform the system and perform the elimination in B, based on the
@@ -673,6 +673,7 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       if (hybridization)
       {
          // Reduction to the Lagrange multipliers system
+         // TODO3: use new functions
          const SparseMatrix *R = fes->GetConformingRestriction();
          Vector conf_b(P->Width()), conf_x(P->Width());
          P->MultTranspose(b, conf_b);
@@ -686,6 +687,7 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       else
       {
          // Variational restriction with P
+         // TODO3: use new functions
          const SparseMatrix *R = fes->GetConformingRestriction();
          B.SetSize(P->Width());
          P->MultTranspose(b, B);
@@ -723,6 +725,7 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
    {
       if (!mat_e)
       {
+         // TODO3: use new functions
          const SparseMatrix *P = fes->GetConformingProlongation();
          if (P) { ConformingAssemble(); }
          EliminateVDofs(ess_tdof_list, diag_policy);
@@ -749,6 +752,7 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,
       return;
    }
 
+   // TODO3: use new functions
    const SparseMatrix *P = fes->GetConformingProlongation();
    if (!P) // conforming space
    {
@@ -783,6 +787,7 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,
          // Primal unknowns recovery
          Vector conf_b(P->Width()), conf_x(P->Width());
          P->MultTranspose(b, conf_b);
+         // TODO3: use new functions
          const SparseMatrix *R = fes->GetConformingRestriction();
          R->Mult(x, conf_x); // get essential b.c. from x
          hybridization->ComputeSolution(conf_b, X, conf_x);
@@ -978,6 +983,18 @@ void BilinearForm::EliminateVDofsInRHS(
    mat->PartMult(vdofs, x, b);
 }
 
+void BilinearForm::Mult(const Vector &x, Vector &y) const
+{
+   if (ext)
+   {
+      ext->Mult(x, y);
+   }
+   else
+   {
+      mat->Mult(x, y);
+   }
+}
+
 void BilinearForm::Update(FiniteElementSpace *nfes)
 {
    bool full_update;
@@ -1053,7 +1070,11 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
    trial_fes = tr_fes;
    test_fes = te_fes;
    mat = NULL;
+   mat_e = NULL;
    extern_bfs = 0;
+
+   assembly = AssemblyLevel::FULL;
+   ext = NULL;
 }
 
 MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
@@ -1064,6 +1085,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
    trial_fes = tr_fes;
    test_fes = te_fes;
    mat = NULL;
+   mat_e = NULL;
    extern_bfs = 1;
 
    // Copy the pointers to the integrators
@@ -1074,6 +1096,38 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
 
    bbfi_marker = mbf->bbfi_marker;
    btfbfi_marker = mbf->btfbfi_marker;
+
+   assembly = AssemblyLevel::FULL;
+   ext = NULL;
+}
+
+void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
+{
+   if (ext)
+   {
+      MFEM_ABORT("the assembly level has already been set!");
+   }
+   assembly = assembly_level;
+   switch (assembly)
+   {
+      case AssemblyLevel::FULL:
+         // ext = new FAMixedBilinearFormExtension(this);
+         // Use the original BilinearForm implementation for now
+         break;
+      case AssemblyLevel::ELEMENT:
+         mfem_error("Element assembly not supported yet... stay tuned!");
+         // ext = new EAMixedBilinearFormExtension(this);
+         break;
+      case AssemblyLevel::PARTIAL:
+         ext = new PAMixedBilinearFormExtension(this);
+         break;
+      case AssemblyLevel::NONE:
+         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         // ext = new MFMixedBilinearFormExtension(this);
+         break;
+      default:
+         mfem_error("Unknown assembly level");
+   }
 }
 
 double & MixedBilinearForm::Elem (int i, int j)
@@ -1086,31 +1140,63 @@ const double & MixedBilinearForm::Elem (int i, int j) const
    return (*mat)(i, j);
 }
 
-void MixedBilinearForm::Mult (const Vector & x, Vector & y) const
+void MixedBilinearForm::Mult(const Vector & x, Vector & y) const
 {
-   mat -> Mult (x, y);
+   y = 0.0;
+   AddMult(x, y);
 }
 
-void MixedBilinearForm::AddMult (const Vector & x, Vector & y,
-                                 const double a) const
+void MixedBilinearForm::AddMult(const Vector & x, Vector & y,
+                                const double a) const
 {
-   mat -> AddMult (x, y, a);
+   if (ext)
+   {
+      ext->AddMult(x, y, a);
+   }
+   else
+   {
+      mat->AddMult(x, y, a);
+   }
 }
 
-void MixedBilinearForm::AddMultTranspose (const Vector & x, Vector & y,
-                                          const double a) const
+void MixedBilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
-   mat -> AddMultTranspose (x, y, a);
+   y = 0.0;
+   AddMultTranspose(x, y);
+}
+
+void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,
+                                         const double a) const
+{
+   if (ext)
+   {
+      ext->AddMultTranspose(x, y, a);
+   }
+   else
+   {
+      mat->AddMultTranspose(x, y, a);
+   }
 }
 
 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   return mat -> Inverse ();
+   if (assembly != AssemblyLevel::FULL)
+   {
+      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
+      return NULL;
+   }
+   else
+   {
+      return mat -> Inverse ();
+   }
 }
 
 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   mat -> Finalize (skip_zeros);
+   if (assembly == AssemblyLevel::FULL)
+   {
+      mat -> Finalize (skip_zeros);
+   }
 }
 
 void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
@@ -1163,6 +1249,12 @@ void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
 
 void MixedBilinearForm::Assemble (int skip_zeros)
 {
+   if (ext)
+   {
+      ext->Assemble();
+      return;
+   }
+
    Array<int> tr_vdofs, te_vdofs;
    ElementTransformation *eltrans;
    DenseMatrix elemmat;
@@ -1330,8 +1422,15 @@ void MixedBilinearForm::Assemble (int skip_zeros)
 
 void MixedBilinearForm::ConformingAssemble()
 {
+   if (assembly != AssemblyLevel::FULL)
+   {
+      MFEM_WARNING("Conforming assemble not supported for this assembly level!");
+      return;
+   }
+
    Finalize();
 
+   // TODO3: use new functions
    const SparseMatrix *P2 = test_fes->GetConformingProlongation();
    if (P2)
    {
@@ -1342,6 +1441,7 @@ void MixedBilinearForm::ConformingAssemble()
       mat = RA;
    }
 
+   // TODO3: use new functions
    const SparseMatrix *P1 = trial_fes->GetConformingProlongation();
    if (P1)
    {
@@ -1488,17 +1588,88 @@ void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
       }
 }
 
+void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
+                                                    const Array<int> &test_tdof_list,
+                                                    OperatorHandle &A)
+
+{
+   if (ext)
+   {
+      ext->FormRectangularSystemOperator(trial_tdof_list, test_tdof_list, A);
+      return;
+   }
+
+   // TODO3: use new functions
+   const SparseMatrix *test_P = test_fes->GetConformingProlongation();
+   const SparseMatrix *trial_P = trial_fes->GetConformingProlongation();
+
+   mat->Finalize();
+
+   if (test_P) // TODO: Must actually check for trial_P too
+   {
+      SparseMatrix *m = RAP(*test_P, *mat, *trial_P);
+      delete mat;
+      mat = m;
+   }
+
+   Array<int> ess_trial_tdof_marker, ess_test_tdof_marker;
+   FiniteElementSpace::ListToMarker(trial_tdof_list, trial_fes->GetTrueVSize(),
+                                    ess_trial_tdof_marker);
+   FiniteElementSpace::ListToMarker(test_tdof_list, test_fes->GetTrueVSize(),
+                                    ess_test_tdof_marker);
+
+   mat_e = new SparseMatrix(mat->Height(), mat->Width());
+   mat->EliminateCols(ess_trial_tdof_marker, *mat_e);
+   // TODO: WP: are we doing the right thing here?
+   for (int i=0; i<test_tdof_list.Size(); ++i)
+   {
+      mat->EliminateRow(test_tdof_list[i]);
+   }
+   mat_e->Finalize();
+   A.Reset(mat, false);
+}
+
+void MixedBilinearForm::FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                              const Array<int> &test_tdof_list,
+                                               Vector &x, Vector &b,
+                                               OperatorHandle &A,
+                                               Vector &X, Vector &B)
+{
+   if (ext)
+   {
+      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, A, X, B);
+      return;
+   }
+
+   const Operator *Po = this->GetOutputProlongation();
+   const Operator *Ri = this->GetRestriction();
+   InitTVectors(Po, Ri, x, b, X, B);
+
+   if (!mat_e)
+   {
+      FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list, A); // Set A = mat_e
+   }
+   // Eliminate essential BCs with B -= Ab xb
+   mat_e->AddMult(X, B, -1.0);
+
+   B.SetSubVector(test_tdof_list, 0.0);
+}
+
 void MixedBilinearForm::Update()
 {
    delete mat;
    mat = NULL;
+   delete mat_e;
+   mat_e = NULL;
    height = test_fes->GetVSize();
    width = trial_fes->GetVSize();
+   if (ext) { ext->Update(); }
 }
 
 MixedBilinearForm::~MixedBilinearForm()
 {
    if (mat) { delete mat; }
+   if (mat_e) { delete mat_e; }
    if (!extern_bfs)
    {
       int i;
@@ -1507,6 +1678,7 @@ MixedBilinearForm::~MixedBilinearForm()
       for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
       for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
    }
+   delete ext;
 }
 
 

fem/bilinearform.hpp

@@ -230,7 +230,7 @@ public:
    virtual const double &Elem(int i, int j) const;
 
    /// Matrix vector multiplication.
-   virtual void Mult(const Vector &x, Vector &y) const { mat->Mult(x, y); }
+   virtual void Mult(const Vector &x, Vector &y) const;
 
    void FullMult(const Vector &x, Vector &y) const
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
@@ -331,6 +331,12 @@ public:
    /// Get the finite element space restriction matrix
    virtual const Operator *GetRestriction() const
    { return fes->GetConformingRestriction(); }
+   /// Get the output finite element space prolongation matrix
+   virtual const Operator *GetOutputProlongation() const
+   { return GetProlongation(); }
+   /// Get the output finite element space restriction matrix
+   virtual const Operator *GetOutputRestriction() const
+   { return GetRestriction(); }
 
    /** @brief Form the linear system A X = B, corresponding to this bilinear
        form and the linear form @a b(.). */
@@ -363,8 +369,9 @@ public:
                                  Vector &B, int copy_interior = 0);
 
    /** @brief Form the linear system A X = B, corresponding to this bilinear
-       form and the linear form @a b(.). */
-   /** Version of the method FormLinearSystem() where the system matrix is
+       form and the linear form @a b(.).
+
+       Version of the method FormLinearSystem() where the system matrix is
        returned in the variable @a A, of type OpType, holding a *reference* to
        the system matrix (created with the method OpType::MakeRef()). The
        reference will be invalidated when SetOperatorType(), Update(), or the
@@ -387,7 +394,7 @@ public:
    virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
                                  OperatorHandle &A);
 
-   /// Form the linear system matrix A, see FormLinearSystem() for details.
+   /// @brief Form the linear system matrix A, see FormLinearSystem() for details.
    /** Version of the method FormSystemMatrix() where the system matrix is
        returned in the variable @a A, of type OpType, holding a *reference* to
        the system matrix (created with the method OpType::MakeRef()). The
@@ -531,6 +538,9 @@ public:
    /// Sets diagonal policy used upon construction of the linear system
    void SetDiagonalPolicy(DiagonalPolicy policy);
 
+   /// Indicate that integrators are not owned by the BilinearForm
+   void SetExternBFS(int e = 1) { extern_bfs = e; };
+
    /// Destroys bilinear form.
    virtual ~BilinearForm();
 };
@@ -555,10 +565,17 @@ class MixedBilinearForm : public Matrix
 {
 protected:
    SparseMatrix *mat; ///< Owned.
+   SparseMatrix *mat_e; ///< Owned.
 
    FiniteElementSpace *trial_fes, ///< Not owned
                       *test_fes;  ///< Not owned
 
+   /// The form assembly level (full, partial, etc.)
+   AssemblyLevel assembly;
+   /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
+       Partial Assembly (PA), or Matrix Free assembly (MF). */
+   MixedBilinearFormExtension *ext;
+
    /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
        #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
    int extern_bfs;
@@ -613,16 +630,13 @@ public:
    virtual const double &Elem(int i, int j) const;
 
    virtual void Mult(const Vector & x, Vector & y) const;
-
    virtual void AddMult(const Vector & x, Vector & y,
                         const double a = 1.0) const;
 
+   virtual void MultTranspose(const Vector & x, Vector & y) const;
    virtual void AddMultTranspose(const Vector & x, Vector & y,
                                  const double a = 1.0) const;
 
-   virtual void MultTranspose(const Vector & x, Vector & y) const
-   { y = 0.0; AddMultTranspose (x, y); }
-
    virtual MatrixInverse *Inverse() const;
 
    virtual void Finalize(int skip_zeros = 1);
@@ -682,8 +696,25 @@ public:
 
    void operator=(const double a) { *mat = a; }
 
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
+   /** This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level);
+
    void Assemble(int skip_zeros = 1);
 
+   /// Get the input finite element space prolongation matrix
+   virtual const Operator *GetProlongation() const
+   { return trial_fes->GetProlongationMatrix(); }
+   /// Get the input finite element space restriction matrix
+   virtual const Operator *GetRestriction() const
+   { return trial_fes->GetRestrictionMatrix(); }
+   /// Get the output finite element space prolongation matrix
+   virtual const Operator *GetOutputProlongation() const
+   { return test_fes->GetProlongationMatrix(); }
+   /// Get the output finite element space restriction matrix
+   virtual const Operator *GetOutputRestriction() const
+   { return test_fes->GetRestrictionMatrix(); }
+
    /** For partially conforming trial and/or test FE spaces, complete the
        assembly process by performing A := P2^t A P1 where A is the internal
        sparse matrix; P1 and P2 are the conforming prolongation matrices of the
@@ -745,8 +776,81 @@ public:
 
    virtual void EliminateTestDofs(const Array<int> &bdr_attr_is_ess);
 
+   /** @brief Return in @a A a parallel (on truedofs) version of this operator.
+
+      This returns the same operator as FormRectangularLinearSystem(), but does
+      without the transformations of the right-hand side. */
+   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list,
+                                    OperatorHandle &A);
+
+   /** @brief Form the column-constrained linear system matrix A.
+       See FormRectangularSystemMatrix() for details.
+
+       Version of the method FormRectangularSystemMatrix() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
+   template <typename OpType>
+   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list, OpType &A)
+   {
+      OperatorHandle Ah;
+      FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list, Ah);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   /** @brief Form the linear system A X = B, corresponding to this mixed bilinear
+       form and the linear form @a b(.).
+
+       Return in @a A a *reference* to the system matrix that is column-constrained.
+       The reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list,
+                                    Vector &x, Vector &b,
+                                    OperatorHandle &A, Vector &X, Vector &B);
+
+   /** @brief Form the linear system A X = B, corresponding to this bilinear
+       form and the linear form @a b(.).
+
+       Version of the method FormRectangularLinearSystem() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
+   template <typename OpType>
+   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list,
+                                    Vector &x, Vector &b,
+                                    OpType &A, Vector &X, Vector &B)
+   {
+      OperatorHandle Ah;
+      FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, Ah, X, B);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
    void Update();
 
+   /// Return the trial FE space associated with the BilinearForm.
+   FiniteElementSpace *TrialFESpace() { return trial_fes; }
+   /// Read-only access to the associated trial FiniteElementSpace.
+   const FiniteElementSpace *TrialFESpace() const { return trial_fes; }
+
+   /// Return the test FE space associated with the BilinearForm.
+   FiniteElementSpace *TestFESpace() { return test_fes; }
+   /// Read-only access to the associated test FiniteElementSpace.
+   const FiniteElementSpace *TestFESpace() const { return test_fes; }
+
    virtual ~MixedBilinearForm();
 };
 

fem/bilinearform_ext.cpp

@@ -38,7 +38,8 @@ const Operator *BilinearFormExtension::GetRestriction() const
 // Data and methods for partially-assembled bilinear forms
 PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
    : BilinearFormExtension(form),
-     trialFes(a->FESpace()), testFes(a->FESpace())
+     trialFes(a->FESpace()),
+     testFes(a->FESpace())
 {
    elem_restrict_lex = trialFes->GetElementRestriction(
                           ElementDofOrdering::LEXICOGRAPHIC);
@@ -56,7 +57,7 @@ void PABilinearFormExtension::Assemble()
    const int integratorCount = integrators.Size();
    for (int i = 0; i < integratorCount; ++i)
    {
-      integrators[i]->AssemblePA(*a->FESpace());
+      integrators[i]->Setup(*a->FESpace());
    }
 }
 
@@ -103,12 +104,9 @@ void PABilinearFormExtension::Update()
 void PABilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
                                                OperatorHandle &A)
 {
-   const Operator* trialP = trialFes->GetProlongationMatrix();
-   const Operator* testP  = testFes->GetProlongationMatrix();
-   Operator *rap = this;
-   if (trialP) { rap = new RAPOperator(*testP, *this, *trialP); }
-   const bool own_A = (rap!=this);
-   A.Reset(new ConstrainedOperator(rap, ess_tdof_list, own_A));
+   Operator *oper;
+   Operator::FormSystemOperator(ess_tdof_list, oper);
+   A.Reset(oper); // A will own oper
 }
 
 void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
@@ -173,4 +171,203 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
    }
 }
 
+
+MixedBilinearFormExtension::MixedBilinearFormExtension(MixedBilinearForm *form)
+   : Operator(form->Height(), form->Width()), a(form)
+{
+   // empty
+}
+
+const Operator *MixedBilinearFormExtension::GetProlongation() const
+{
+   return a->GetProlongation();
+}
+
+const Operator *MixedBilinearFormExtension::GetRestriction() const
+{
+   return a->GetRestriction();
+}
+
+const Operator *MixedBilinearFormExtension::GetOutputProlongation() const
+{
+   return a->GetOutputProlongation();
+}
+
+const Operator *MixedBilinearFormExtension::GetOutputRestriction() const
+{
+   return a->GetOutputRestriction();
+}
+
+// Data and methods for partially-assembled bilinear forms
+
+PAMixedBilinearFormExtension::PAMixedBilinearFormExtension(
+   MixedBilinearForm *form)
+   : MixedBilinearFormExtension(form),
+     trialFes(form->TrialFESpace()),
+     testFes(form->TestFESpace()),
+     elem_restrict_trial(NULL),
+     elem_restrict_test(NULL)
+{
+   Update();
+}
+
+void PAMixedBilinearFormExtension::Assemble()
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integratorCount = integrators.Size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->Setup(*trialFes, *testFes);
+   }
+}
+
+void PAMixedBilinearFormExtension::Update()
+{
+   trialFes = a->TrialFESpace();
+   testFes  = a->TestFESpace();
+   height = testFes->GetVSize();
+   width = trialFes->GetVSize();
+   elem_restrict_trial = trialFes->GetElementRestriction(
+                            ElementDofOrdering::LEXICOGRAPHIC);
+   elem_restrict_test  =  testFes->GetElementRestriction(
+                             ElementDofOrdering::LEXICOGRAPHIC);
+   if (elem_restrict_trial)
+   {
+      localTrial.UseDevice(true);
+      localTrial.SetSize(elem_restrict_trial->Height(), Device::GetMemoryType());
+
+   }
+   if (elem_restrict_test)
+   {
+      localTest.UseDevice(true); // ensure 'localY = 0.0' is done on device
+      localTest.SetSize(elem_restrict_test->Height(), Device::GetMemoryType());
+   }
+}
+
+void PAMixedBilinearFormExtension::FormRectangularSystemOperator(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list,
+   OperatorHandle &A)
+{
+   Operator * oper;
+   Operator::FormRectangularSystemOperator(trial_tdof_list, test_tdof_list, oper);
+   A.Reset(oper); // A will own oper
+}
+
+void PAMixedBilinearFormExtension::FormRectangularLinearSystem(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list,
+   Vector &x, Vector &b,
+   OperatorHandle &A,
+   Vector &X, Vector &B)
+{
+   Operator *oper;
+   Operator::FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b,
+                                         oper, X, B);
+   A.Reset(oper); // A will own oper
+}
+
+void PAMixedBilinearFormExtension::SetupMultInputs(const Operator
+                                                   *elem_restrict_x,
+                                                   const Vector &x,
+                                                   Vector &localX,
+                                                   const Operator *elem_restrict_y,
+                                                   Vector &y,
+                                                   Vector &localY,
+                                                   const double c) const
+{
+   // * G operation: localX = c*local(x)
+   if (elem_restrict_x)
+   {
+      elem_restrict_x->Mult(x, localX);
+      if (c != 1.0)
+      {
+         localX *= c;
+      }
+   }
+   else
+   {
+      if (c == 1.0)
+      {
+         localX.SyncAliasMemory(x);
+      }
+      else
+      {
+         localX.Set(c, x);
+      }
+   }
+   if (elem_restrict_y)
+   {
+      localY = 0.0;
+   }
+   else
+   {
+      y.UseDevice(true);
+      localY.SyncAliasMemory(y);
+   }
+}
+
+void PAMixedBilinearFormExtension::Mult(const Vector &x, Vector &y) const
+{
+   y = 0.0;
+   AddMult(x, y);
+}
+
+void PAMixedBilinearFormExtension::AddMult(const Vector &x, Vector &y,
+                                           const double c) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+
+   // * G operation
+   SetupMultInputs(elem_restrict_trial, x, localTrial,
+                   elem_restrict_test, y, localTest, c);
+
+   // * B^TDB operation
+   for (int i = 0; i < iSz; ++i)
+   {
+      integrators[i]->AddMultPA(localTrial, localTest);
+   }
+
+   // * G^T operation
+   if (elem_restrict_test)
+   {
+      tempY.SetSize(y.Size());
+      elem_restrict_test->MultTranspose(localTest, tempY);
+      y += tempY;
+   }
+}
+
+void PAMixedBilinearFormExtension::MultTranspose(const Vector &x,
+                                                 Vector &y) const
+{
+   y = 0.0;
+   AddMultTranspose(x, y);
+}
+
+void PAMixedBilinearFormExtension::AddMultTranspose(const Vector &x, Vector &y,
+                                                    const double c) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+
+   // * G operation
+   SetupMultInputs(elem_restrict_test, x, localTest,
+                   elem_restrict_trial, y, localTrial, c);
+
+   // * B^TD^TB operation
+   for (int i = 0; i < iSz; ++i)
+   {
+      integrators[i]->AddMultTransposePA(localTest, localTrial);
+   }
+
+   // * G^T operation
+   if (elem_restrict_trial)
+   {
+      tempY.SetSize(y.Size());
+      elem_restrict_trial->MultTranspose(localTrial, tempY);
+      y += tempY;
+   }
+}
+
 } // namespace mfem

fem/bilinearform_ext.hpp

@@ -20,6 +20,8 @@ namespace mfem
 {
 
 class BilinearForm;
+class MixedBilinearForm;
+
 
 
 /** @brief Class extending the BilinearForm class to support the different
@@ -139,6 +141,120 @@ public:
    ~MFBilinearFormExtension() {}
 };
 
+
+/** @brief Class extending the MixedBilinearForm class to support the different
+    AssemblyLevel%s. */
+class MixedBilinearFormExtension : public Operator
+{
+protected:
+   MixedBilinearForm *a; ///< Not owned
+
+public:
+   MixedBilinearFormExtension(MixedBilinearForm *form);
+
+   virtual MemoryClass GetMemoryClass() const
+   { return Device::GetMemoryClass(); }
+
+   /// Get the finite element space prolongation matrix
+   virtual const Operator *GetProlongation() const;
+
+   /// Get the finite element space restriction matrix
+   virtual const Operator *GetRestriction() const;
+
+   /// Get the output finite element space restriction matrix
+   virtual const Operator *GetOutputProlongation() const;
+
+   /// Get the output finite element space restriction matrix
+   virtual const Operator *GetOutputRestriction() const;
+
+   virtual void Assemble() = 0;
+   virtual void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
+                                              const Array<int> &test_tdof_list,
+                                              OperatorHandle &A) = 0;
+   virtual void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                            const Array<int> &test_tdof_list,
+                                            Vector &x, Vector &b,
+                                            OperatorHandle &A, Vector &X, Vector &B) = 0;
+
+   virtual void AddMult(const Vector &x, Vector &y, const double c=1.0) const = 0;
+   virtual void AddMultTranspose(const Vector &x, Vector &y,
+                                 const double c=1.0) const = 0;
+   virtual void Update() = 0;
+};
+
+/// Data and methods for fully-assembled mixed bilinear forms
+class FAMixedBilinearFormExtension : public MixedBilinearFormExtension
+{
+public:
+   FAMixedBilinearFormExtension(MixedBilinearForm *form)
+      : MixedBilinearFormExtension(form) { }
+
+   /// TODO
+   void Assemble() {}
+   void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
+                                      const Array<int> &test_tdof_list,
+                                      OperatorHandle &A)
+   {}
+   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list,
+                                    Vector &x, Vector &b,
+                                    OperatorHandle &A, Vector &X, Vector &B) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~FAMixedBilinearFormExtension() {}
+};
+
+/// Data and methods for partially-assembled mixed bilinear forms
+class PAMixedBilinearFormExtension : public MixedBilinearFormExtension
+{
+protected:
+   const FiniteElementSpace *trialFes, *testFes; // Not owned
+   mutable Vector localTrial, localTest, tempY;
+   const Operator *elem_restrict_trial; // Not owned
+   const Operator *elem_restrict_test;  // Not owned
+private:
+   /// Helper function to set up inputs/outputs for Mult or MultTranspose
+   void SetupMultInputs(const Operator *elem_restrict_x,
+                        const Vector &x, Vector &localX,
+                        const Operator *elem_restrict_y,
+                        Vector &y, Vector &localY, const double c) const;
+
+public:
+   PAMixedBilinearFormExtension(MixedBilinearForm *form);
+
+   /// Partial assembly of all internal integrators
+   void Assemble();
+   /**
+      @brief Setup OperatorHandle A to contain constrained linear operator
+
+      OperatorHandle A contains matrix-free constrained operator formed for RAP system
+      where ess_tdof_list are in trial space and eliminated from "columns" of A.
+   */
+   void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
+                                      const Array<int> &test_tdof_list,
+                                      OperatorHandle &A);
+   /**
+      Setup OperatorHandle A to contain constrained linear operator and
+      eliminate columns corresponding to essential dofs from system,
+      updating RHS B vector with the results.
+   */
+   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list,
+                                    Vector &x, Vector &b,
+                                    OperatorHandle &A, Vector &X, Vector &B);
+   /// y = A*x
+   void Mult(const Vector &x, Vector &y) const;
+   /// y += c*A*x
+   void AddMult(const Vector &x, Vector &y, const double c=1.0) const;
+   /// y = A^T*x
+   void MultTranspose(const Vector &x, Vector &y) const;
+   /// y += c*A^T*x
+   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
+   /// Update internals for when a new MixedBilinearForm is given to this class
+   void Update();
+};
+
 }
 
 #endif

fem/bilininteg.cpp

@@ -386,6 +386,72 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
 }
 
 
+void GradientIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans,  DenseMatrix &elmat)
+{
+   int dim = test_fe.GetDim();
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   double c;
+   Vector d_col;
+
+   dshape.SetSize(trial_dof, dim);
+   gshape.SetSize(trial_dof, dim);
+   Jadj.SetSize(dim);
+   shape.SetSize(test_dof);
+   elmat.SetSize(dim * test_dof, trial_dof);
+
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
+                                                            Trans);
+
+   elmat = 0.0;
+   elmat_comp.SetSize(test_dof, trial_dof);
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      trial_fe.CalcDShape(ip, dshape);
+      test_fe.CalcShape(ip, shape);
+
+      Trans.SetIntPoint(&ip);
+      CalcAdjugate(Trans.Jacobian(), Jadj);
+
+      Mult(dshape, Jadj, gshape);
+
+      c = ip.weight;
+      if (Q)
+      {
+         c *= Q->Eval(Trans, ip);
+      }
+      shape *= c;
+
+      for (int d = 0; d < dim; ++d)
+      {
+         gshape.GetColumnReference(d, d_col);
+         MultVWt(shape, d_col, elmat_comp);
+         for (int jj = 0; jj < trial_dof; ++jj)
+         {
+            for (int ii = 0; ii < test_dof; ++ii)
+            {
+               elmat(d * test_dof + ii, jj) += elmat_comp(ii, jj);
+            }
+         }
+      }
+   }
+}
+
+const IntegrationRule &GradientIntegrator::GetRule(const FiniteElement
+                                                   &trial_fe,
+                                                   const FiniteElement &test_fe,
+                                                   ElementTransformation &Trans)
+{
+   int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder() + Trans.OrderJ();
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
+
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -423,7 +489,7 @@ void DiffusionIntegrator::AssembleElementMatrix
       {
          if (Q)
          {
-            w *= Q->Eval(Trans, ip);
+            w *= Q->Eval(Trans, ip); // w = c wq / det(J)
          }
          AddMult_a_AAt(w, dshapedxt, elmat);
       }
@@ -875,7 +941,6 @@ void ConvectionIntegrator::AssembleElementMatrix(
    }
 }
 
-
 void GroupConvectionIntegrator::AssembleElementMatrix(
    const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -1950,12 +2015,8 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(
 
    elmat.SetSize (test_dof, dim*trial_dof);
 
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder();
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-   }
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
+                                                            Trans);
 
    elmat = 0.0;
 
@@ -1963,15 +2024,15 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
 
-      trial_fe.CalcDShape (ip, dshape);
-      test_fe.CalcShape (ip, shape);
+      trial_fe.CalcDShape (ip, dshape); // dshape = grad phi_j at ip
+      test_fe.CalcShape (ip, shape);    // shape  = p_i at ip
 
       Trans.SetIntPoint (&ip);
       CalcAdjugate(Trans.Jacobian(), Jadj);
 
-      Mult (dshape, Jadj, gshape);
+      Mult (dshape, Jadj, gshape); // gshape = dshape * Jadj
 
-      gshape.GradToDiv (divshape);
+      gshape.GradToDiv (divshape); // Reshape into long "divergence" vector
 
       c = ip.weight;
       if (Q)
@@ -1985,6 +2046,15 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(
    }
 }
 
+const IntegrationRule &VectorDivergenceIntegrator::GetRule(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans)
+{
+   int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder() + Trans.OrderJ();
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
 
 void DivDivIntegrator::AssembleElementMatrix(
    const FiniteElement &el,

fem/bilininteg.hpp

@@ -1668,6 +1668,55 @@ protected:
    }
 };
 
+/** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where
+    Q is a scalar coefficient, v is a vector where each v_i is in the same space as u.
+*/
+class GradientIntegrator : public BilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+
+private:
+   Vector shape;
+   DenseMatrix dshape;
+   DenseMatrix gshape;
+   DenseMatrix Jadj;
+   DenseMatrix elmat_comp;
+   // PA extension
+   Vector pa_data;
+   const DofToQuad *trial_maps, *test_maps; ///< Not owned
+   const GeometricFactors *geom;            ///< Not owned
+   int dim, ne, nq;
+   int trial_dofs1D, test_dofs1D, quad1D;
+
+public:
+   GradientIntegrator() :
+      Q{NULL}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
+   { }
+   GradientIntegrator(Coefficient *_q) :
+      Q{_q}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
+   { }
+   GradientIntegrator(Coefficient &q) :
+      Q{&q}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
+   { }
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+   using BilinearFormIntegrator::Setup;
+   virtual void Setup(const FiniteElementSpace &trial_fes,
+                      const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans);
+};
+
 /** Class for integrating the bilinear form a(u,v) := (Q grad u, grad v) where Q
     can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
@@ -1727,7 +1776,8 @@ public:
                                     ElementTransformation &Trans,
                                     Vector &flux, Vector *d_energy = NULL);
 
-   virtual void AssemblePA(const FiniteElementSpace&);
+   using BilinearFormIntegrator::Setup;
+   virtual void Setup(const FiniteElementSpace &fes);
 
    virtual void AssembleDiagonalPA(Vector& diag) const;
 
@@ -1771,7 +1821,8 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 
-   virtual void AssemblePA(const FiniteElementSpace&);
+   using BilinearFormIntegrator::Setup;
+   virtual void Setup(const FiniteElementSpace &fes);
 
    virtual void AssembleDiagonalPA(Vector& diag) const;
 
@@ -1853,6 +1904,11 @@ protected:
    Coefficient *Q;
    VectorCoefficient *VQ;
    MatrixCoefficient *MQ;
+   // PA extension
+   Vector pa_data;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq, dofs1D, quad1D;
 
 public:
    /// Construct an integrator with coefficient 1.0
@@ -1884,6 +1940,10 @@ public:
                                        const FiniteElement &test_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
+   using BilinearFormIntegrator::Setup;
+   virtual void Setup(const FiniteElementSpace &fes);
+   virtual void AssembleDiagonalPA(Vector &diag) const;
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
 };
 
 
@@ -2107,16 +2167,39 @@ private:
    DenseMatrix dshape;
    DenseMatrix gshape;
    DenseMatrix Jadj;
+   // PA extension
+   Vector pa_data;
+   const DofToQuad *trial_maps, *test_maps; ///< Not owned
+   const GeometricFactors *geom;            ///< Not owned
+   int dim, ne, nq;
+   int trial_dofs1D, test_dofs1D, quad1D;
 
 public:
-   VectorDivergenceIntegrator() { Q = NULL; }
-   VectorDivergenceIntegrator(Coefficient *_q) { Q = _q; }
-   VectorDivergenceIntegrator(Coefficient &q) { Q = &q; }
+   VectorDivergenceIntegrator() :
+      Q(NULL), trial_maps(NULL), test_maps(NULL), geom(NULL)
+   {  }
+   VectorDivergenceIntegrator(Coefficient *_q) :
+      Q(_q), trial_maps(NULL), test_maps(NULL), geom(NULL)
+   { }
+   VectorDivergenceIntegrator(Coefficient &q) :
+      Q(&q), trial_maps(NULL), test_maps(NULL), geom(NULL)
+   { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                        const FiniteElement &test_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
+
+   using BilinearFormIntegrator::Setup;
+   virtual void Setup(const FiniteElementSpace &trial_fes,
+                      const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans);
 };
 
 /// (Q div u, div v) for RT elements
@@ -2150,6 +2233,12 @@ class VectorDiffusionIntegrator : public BilinearFormIntegrator
 protected:
    Coefficient *Q;
 
+   // PA extension
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, dofs1D, quad1D;
+   Vector pa_data;
+
 private:
    DenseMatrix Jinv;
    DenseMatrix dshape;
@@ -2166,6 +2255,10 @@ public:
    virtual void AssembleElementVector(const FiniteElement &el,
                                       ElementTransformation &Tr,
                                       const Vector &elfun, Vector &elvect);
+   using BilinearFormIntegrator::Setup;
+   virtual void Setup(const FiniteElementSpace &fes);
+   virtual void AssembleDiagonalPA(Vector &diag) const;
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
 };
 
 /** Integrator for the linear elasticity form:

fem/bilininteg_diffusion.cpp

@@ -193,7 +193,7 @@ static void PADiffusionSetup(const int dim,
    }
 }
 
-void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
+void DiffusionIntegrator::Setup(const FiniteElementSpace &fes)
 {
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();

fem/bilininteg_gradient.cpp

@@ -0,0 +1,764 @@
+// Copyright (c) 2019, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// PA Gradient Integrator
+
+// PA Gradient Assemble 2D kernel
+static void PAGradientSetup2D(const int Q1D,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              const double COEFF,
+                              Vector &op)
+{
+   const int NQ = Q1D*Q1D;
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
+   auto y = Reshape(op.Write(), NQ, 2, 2, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J21 = J(q,1,0,e);
+         const double J22 = J(q,1,1,e);
+         // Store wq * Q * adj(J)
+         y(q,0,0,e) = W[q] * COEFF *  J22; // 1,1
+         y(q,0,1,e) = W[q] * COEFF * -J12; // 1,2
+         y(q,1,0,e) = W[q] * COEFF * -J21; // 2,1
+         y(q,1,1,e) = W[q] * COEFF *  J11; // 2,2
+      }
+   });
+}
+
+// PA Gradient Assemble 3D kernel
+static void PAGradientSetup3D(const int Q1D,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              const double COEFF,
+                              Vector &op)
+{
+   const int NQ = Q1D*Q1D*Q1D;
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   auto y = Reshape(op.Write(), NQ, 3, 3, NE);
+   MFEM_FORALL(e, NE,
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double cw  = W[q] * COEFF;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+         // Store wq * Q * adj(J)
+         y(q,0,0,e) = cw * A11; // 1,1
+         y(q,0,1,e) = cw * A12; // 1,2
+         y(q,0,2,e) = cw * A13; // 1,3
+         y(q,1,0,e) = cw * A21; // 2,1
+         y(q,1,1,e) = cw * A22; // 2,2
+         y(q,1,2,e) = cw * A23; // 2,3
+         y(q,2,0,e) = cw * A31; // 3,1
+         y(q,2,1,e) = cw * A32; // 3,2
+         y(q,2,2,e) = cw * A33; // 3,3
+      }
+   });
+}
+
+static void PAGradientSetup(const int dim,
+                            const int TR_D1D,
+                            const int TE_D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &W,
+                            const Vector &J,
+                            const double COEFF,
+                            Vector &op)
+{
+   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PAGradientSetup"); }
+   if (dim == 2)
+   {
+      PAGradientSetup2D(Q1D, NE, W, J, COEFF, op);
+   }
+   if (dim == 3)
+   {
+      PAGradientSetup3D(Q1D, NE, W, J, COEFF, op);
+   }
+}
+
+void GradientIntegrator::Setup(const FiniteElementSpace &trial_fes,
+                               const FiniteElementSpace &test_fes)
+{
+   // Assumes tensor-product elements ordered by nodes
+   MFEM_ASSERT(trial_fes.GetOrdering() == Ordering::byNODES,
+               "PA Only supports Ordering::byNODES!");
+   Mesh *mesh = trial_fes.GetMesh();
+   const FiniteElement &trial_fe = *trial_fes.GetFE(0);
+   const FiniteElement &test_fe = *test_fes.GetFE(0);
+   ElementTransformation *trans = mesh->GetElementTransformation(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
+                                                            *trans);
+   const int dims = trial_fe.GetDim();
+   const int dimsToStore = dims * dims;
+   const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   ne = trial_fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   trial_maps = &trial_fe.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   trial_dofs1D = trial_maps->ndof;
+   quad1D = trial_maps->nqpt;
+   test_maps  = &test_fe.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   test_dofs1D = test_maps->ndof;
+   MFEM_ASSERT(quad1D == test_maps->nqpt,
+               "PA requires test and trial space to have same number of quadrature points!");
+   pa_data.SetSize(nq * dimsToStore * ne, Device::GetMemoryType());
+   double coeff = 1.0;
+   if (Q)
+   {
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      coeff = cQ->constant;
+   }
+   PAGradientSetup(dim, trial_dofs1D, test_dofs1D, quad1D,
+                   ne, ir->GetWeights(), geom->J, coeff, pa_data);
+}
+
+// PA Gradient Apply 2D kernel
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+static void PAGradientApply2D(const int NE,
+                              const Array<double> &b,
+                              const Array<double> &g,
+                              const Array<double> &bt,
+                              const Vector &_op,
+                              const Vector &_x,
+                              Vector &_y,
+                              const int tr_d1d = 0,
+                              const int te_d1d = 0,
+                              const int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, TR_D1D);
+   auto G = Reshape(g.Read(), Q1D, TR_D1D);
+   auto Bt = Reshape(bt.Read(), TE_D1D, Q1D);
+   auto op = Reshape(_op.Read(), Q1D*Q1D, 2,2, NE);
+   auto x = Reshape(_x.Read(), TR_D1D, TR_D1D, NE);
+   auto y = Reshape(_y.ReadWrite(), TE_D1D, TE_D1D, 2, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+      const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = 2;
+      // the following variables are evaluated at compile time
+      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      double grad[max_Q1D][max_Q1D][VDIM];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            grad[qy][qx][0] = 0.0;
+            grad[qy][qx][1] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < TR_D1D; ++dy)
+      {
+         double gradX[max_Q1D][VDIM];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            gradX[qx][0] = 0.0;
+            gradX[qx][1] = 0.0;
+         }
+         for (int dx = 0; dx < TR_D1D; ++dx)
+         {
+            const double s = x(dx,dy,e);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradX[qx][0] += s * G(qx,dx);
+               gradX[qx][1] += s * B(qx,dx);
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double wy  = B(qy,dy);
+            const double wDy = G(qy,dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               grad[qy][qx][0] += gradX[qx][0] * wy;
+               grad[qy][qx][1] += gradX[qx][1] * wDy;
+            }
+         }
+      }
+      // We've now calculated grad(p) = [Dxy, xDy] in plane
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const int q = qx + qy * Q1D;
+            const double gradX = grad[qy][qx][0];
+            const double gradY = grad[qy][qx][1];
+
+            grad[qy][qx][0] = gradX*op(q,0,0,e) + gradY*op(q,1,0,e);
+            grad[qy][qx][1] = gradX*op(q,0,1,e) + gradY*op(q,1,1,e);
+         }
+      }
+      // We've now calculated grad = grad p * op
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         double opX[max_TE_D1D][VDIM];
+         for (int dx = 0; dx < TE_D1D; ++dx)
+         {
+            opX[dx][0] = 0.0;
+            opX[dx][1] = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               opX[dx][0] += Bt(dx,qx)*grad[qy][qx][0];
+               opX[dx][1] += Bt(dx,qx)*grad[qy][qx][1];
+            }
+         }
+         for (int dy = 0; dy < TE_D1D; ++dy)
+         {
+            for (int dx = 0; dx < TE_D1D; ++dx)
+            {
+               y(dx,dy,0,e) += Bt(dy,qy)*opX[dx][0];
+               y(dx,dy,1,e) += Bt(dy,qy)*opX[dx][1];
+            }
+         }
+      }
+      // We've now calculated y = u * grad
+   });
+
+}
+
+// Shared memory PA Gradient Apply 2D kernel
+template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0,
+         const int T_NBZ = 0>
+static void SmemPAGradientApply2D(const int NE,
+                                  const Array<double> &_b,
+                                  const Array<double> &_g,
+                                  const Array<double> &_bt,
+                                  const Vector &_op,
+                                  const Vector &_x,
+                                  Vector &_y,
+                                  const int tr_d1d = 0,
+                                  const int te_d1d = 0,
+                                  const int q1d = 0)
+{
+   // TODO
+   MFEM_ASSERT(false, "SHARED MEM NOT PROGRAMMED YET");
+}
+
+// PA Gradient Apply 2D kernel transpose
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+static void PAGradientApplyTranspose2D(const int NE,
+                                       const Array<double> &bt,
+                                       const Array<double> &gt,
+                                       const Array<double> &b,
+                                       const Vector &_op,
+                                       const Vector &_x,
+                                       Vector &_y,
+                                       const int tr_d1d = 0,
+                                       const int te_d1d = 0,
+                                       const int q1d = 0)
+{
+   // TODO
+   MFEM_ASSERT(false, "Gradient PA Apply Transpose 2D NOT PROGRAMMED YET");
+}
+
+// PA Gradient Apply 3D kernel
+template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
+static void PAGradientApply3D(const int NE,
+                              const Array<double> &b,
+                              const Array<double> &g,
+                              const Array<double> &bt,
+                              const Vector &_op,
+                              const Vector &_x,
+                              Vector &_y,
+                              int tr_d1d = 0,
+                              int te_d1d = 0,
+                              int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, TR_D1D);
+   auto G = Reshape(g.Read(), Q1D, TR_D1D);
+   auto Bt = Reshape(bt.Read(), TE_D1D, Q1D);
+   auto op = Reshape(_op.Read(), Q1D*Q1D*Q1D, 3,3, NE);
+   auto x = Reshape(_x.Read(), TR_D1D, TR_D1D, TR_D1D, NE);
+   auto y = Reshape(_y.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 3, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+      const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = 3;
+      // the following variables are evaluated at compile time
+      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      double grad[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               grad[qz][qy][qx][0] = 0.0;
+               grad[qz][qy][qx][1] = 0.0;
+               grad[qz][qy][qx][2] = 0.0;
+            }
+         }
+      }
+      for (int dz = 0; dz < TR_D1D; ++dz)
+      {
+         double gradXY[max_Q1D][max_Q1D][3];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradXY[qy][qx][0] = 0.0;
+               gradXY[qy][qx][1] = 0.0;
+               gradXY[qy][qx][2] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < TR_D1D; ++dy)
+         {
+            double gradX[max_Q1D][2];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradX[qx][0] = 0.0;
+               gradX[qx][1] = 0.0;
+            }
+            for (int dx = 0; dx < TR_D1D; ++dx)
+            {
+               const double s = x(dx,dy,dz,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradX[qx][0] += s * B(qx,dx);
+                  gradX[qx][1] += s * G(qx,dx);
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy  = B(qy,dy);
+               const double wDy = G(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double wx  = gradX[qx][0];
+                  const double wDx = gradX[qx][1];
+                  gradXY[qy][qx][0] += wDx * wy;
+                  gradXY[qy][qx][1] += wx  * wDy;
+                  gradXY[qy][qx][2] += wx  * wy;
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const double wz  = B(qz,dz);
+            const double wDz = G(qz,dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+                  grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+                  grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+               }
+            }
+         }
+      }
+      // We've now calculated grad(p) = [Dxyz, xDyz, xyDz] in plane
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + (qy + qz * Q1D) * Q1D;
+               const double gradX = grad[qz][qy][qx][0];
+               const double gradY = grad[qz][qy][qx][1];
+               const double gradZ = grad[qz][qy][qx][2];
+
+               grad[qz][qy][qx][0] = gradX*op(q,0,0,e) + gradY*op(q,1,0,e) + gradZ*op(q,2,0,e);
+               grad[qz][qy][qx][1] = gradX*op(q,0,1,e) + gradY*op(q,1,1,e) + gradZ*op(q,2,1,e);
+               grad[qz][qy][qx][2] = gradX*op(q,0,2,e) + gradY*op(q,1,2,e) + gradZ*op(q,2,2,e);
+            }
+         }
+      }
+      // We've now calculated grad = grad p * op
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double opXY[max_TE_D1D][max_TE_D1D][VDIM];
+         for (int dy = 0; dy < TE_D1D; ++dy)
+         {
+            for (int dx = 0; dx < TE_D1D; ++dx)
+            {
+               opXY[dy][dx][0] = 0.0;
+               opXY[dy][dx][1] = 0.0;
+               opXY[dy][dx][2] = 0.0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double opX[max_TE_D1D][VDIM];
+            for (int dx = 0; dx < TE_D1D; ++dx)
+            {
+               opX[dx][0] = 0.0;
+               opX[dx][1] = 0.0;
+               opX[dx][2] = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  opX[dx][0] += Bt(dx,qx)*grad[qz][qy][qx][0];
+                  opX[dx][1] += Bt(dx,qx)*grad[qz][qy][qx][1];
+                  opX[dx][2] += Bt(dx,qx)*grad[qz][qy][qx][2];
+               }
+            }
+            for (int dy = 0; dy < TE_D1D; ++dy)
+            {
+               for (int dx = 0; dx < TE_D1D; ++dx)
+               {
+                  opXY[dy][dx][0] += Bt(dy,qy)*opX[dx][0];
+                  opXY[dy][dx][1] += Bt(dy,qy)*opX[dx][1];
+                  opXY[dy][dx][2] += Bt(dy,qy)*opX[dx][2];
+               }
+            }
+         }
+         for (int dz = 0; dz < TE_D1D; ++dz)
+         {
+            for (int dy = 0; dy < TE_D1D; ++dy)
+            {
+               for (int dx = 0; dx < TE_D1D; ++dx)
+               {
+                  y(dx,dy,dz,0,e) += Bt(dz,qz)*opXY[dy][dx][0];
+                  y(dx,dy,dz,1,e) += Bt(dz,qz)*opXY[dy][dx][1];
+                  y(dx,dy,dz,2,e) += Bt(dz,qz)*opXY[dy][dx][2];
+               }
+            }
+         }
+      }
+      // We've now calculated y = u * grad
+   });
+}
+
+// PA Gradient Apply 3D kernel
+template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
+static void PAGradientApplyTranspose3D(const int NE,
+                                       const Array<double> &bt,
+                                       const Array<double> &gt,
+                                       const Array<double> &b,
+                                       const Vector &_op,
+                                       const Vector &_x,
+                                       Vector &_y,
+                                       int tr_d1d = 0,
+                                       int te_d1d = 0,
+                                       int q1d = 0)
+{
+   // TODO
+   MFEM_ASSERT(false, "Gradient PA Apply Transpose 3D NOT PROGRAMMED YET");
+}
+
+// Shared memory PA Gradient Apply 3D kernel
+template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
+static void SmemPAGradientApply3D(const int NE,
+                                  const Array<double> &_b,
+                                  const Array<double> &_g,
+                                  const Array<double> &_bt,
+                                  const Vector &_op,
+                                  const Vector &_x,
+                                  Vector &_y,
+                                  const int tr_d1d = 0,
+                                  const int te_d1d = 0,
+                                  const int q1d = 0)
+{
+   // TODO
+   MFEM_ASSERT(false, "SHARED MEM NOT PROGRAMMED YET");
+}
+
+static void PAGradientApply(const int dim,
+                            const int TR_D1D,
+                            const int TE_D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &B,
+                            const Array<double> &G,
+                            const Array<double> &Bt,
+                            const Vector &op,
+                            const Vector &x,
+                            Vector &y,
+                            bool transpose=false)
+{
+
+   //if (Device::Allows(Backend::RAJA_CUDA))
+   //{
+   if (dim == 2)
+   {
+      switch ((TR_D1D << 4) | TE_D1D)
+      {
+         case 0x23: // Specialized for Taylor-Hood elements
+            if (Q1D == 3)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose2D<2,3,3>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply2D<2,3,3>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x34:
+            if (Q1D == 4)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose2D<3,4,4>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply2D<3,4,4>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x45:
+            if (Q1D == 6)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose2D<4,5,6>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply2D<4,5,6>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x56:
+            if (Q1D == 7)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose2D<5,6,7>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply2D<5,6,7>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x67:
+            if (Q1D == 9)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose2D<6,7,9>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply2D<6,7,9>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x78:
+            if (Q1D == 10)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose2D<7,8,10>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply2D<7,8,10>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x89:
+            if (Q1D == 12)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose2D<8,9,12>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply2D<8,9,12>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         default:
+            break;
+      }
+      return PAGradientApply2D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+   }
+   if (dim == 3)
+   {
+      switch ((TR_D1D << 4) | TE_D1D)
+      {
+         case 0x23: // Specialized for Taylor-Hood elements
+            if (Q1D == 4)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose3D<2,3,4>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply3D<2,3,4>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x34:
+            if (Q1D == 6)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose3D<3,4,6>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply3D<3,4,6>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x45:
+            if (Q1D == 8)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose3D<4,5,8>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply3D<4,5,8>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x56:
+            if (Q1D == 10)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose3D<5,6,10>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply3D<5,6,10>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x67:
+            if (Q1D == 12)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose3D<6,7,12>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply3D<6,7,12>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x78:
+            if (Q1D == 14)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose3D<7,8,14>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply3D<7,8,14>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         case 0x89:
+            if (Q1D == 16)
+            {
+               if (transpose)
+               {
+                  return PAGradientApplyTranspose3D<8,9,16>(NE,B,G,Bt,op,x,y);
+               }
+               else
+               {
+                  return PAGradientApply3D<8,9,16>(NE,B,G,Bt,op,x,y);
+               }
+            }
+            break;
+         default:
+            break;
+      }
+      if (transpose)
+      {
+         return PAGradientApplyTranspose3D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+      }
+      else
+      {
+         return PAGradientApply3D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+      }
+   }
+   //}
+   MFEM_ABORT("Unknown kernel.");
+}
+
+// PA Gradient Apply kernel
+void GradientIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   PAGradientApply(dim, trial_dofs1D, test_dofs1D, quad1D, ne,
+                   trial_maps->B, trial_maps->G, test_maps->Bt, pa_data, x, y,
+                   false);
+}
+
+// PA Gradient Apply kernel
+void GradientIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   MFEM_ABORT("PA Gradient AddMultTransposePA not yet programmed!");
+}
+
+} // namespace mfem
+

fem/bilininteg_mass.cpp

@@ -21,7 +21,7 @@ namespace mfem
 // PA Mass Integrator
 
 // PA Mass Assemble kernel
-void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
+void MassIntegrator::Setup(const FiniteElementSpace &fes)
 {
    // Assuming the same element type
    Mesh *mesh = fes.GetMesh();

fem/bilininteg_vecdiffusion.cpp

@@ -0,0 +1,803 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// PA Vector Diffusion Integrator
+
+// PA Diffusion Assemble 2D kernel
+static void PAVectorDiffusionSetup2D(const int Q1D,
+                                     const int NE,
+                                     const Array<double> &w,
+                                     const Vector &j,
+                                     const double COEFF,
+                                     Vector &op)
+{
+   const int NQ = Q1D*Q1D;
+   auto W = w.Read();
+
+   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
+   auto y = Reshape(op.Write(), NQ, 3, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double c_detJ = W[q] * COEFF / ((J11*J22)-(J21*J12));
+         y(q,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
+         y(q,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
+         y(q,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
+      }
+   });
+}
+
+// PA Diffusion Assemble 3D kernel
+static void PAVectorDiffusionSetup3D(const int Q1D,
+                                     const int NE,
+                                     const Array<double> &w,
+                                     const Vector &j,
+                                     const double COEFF,
+                                     Vector &op)
+{
+   const int NQ = Q1D*Q1D*Q1D;
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   auto y = Reshape(op.Write(), NQ, 6, NE);
+   MFEM_FORALL(e, NE,
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double detJ = J11 * (J22 * J33 - J32 * J23) -
+         /* */               J21 * (J12 * J33 - J32 * J13) +
+         /* */               J31 * (J12 * J23 - J22 * J13);
+         const double c_detJ = W[q] * COEFF / detJ;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+         // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+         y(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+         y(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+         y(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+         y(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+         y(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+         y(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
+      }
+   });
+}
+
+static void PAVectorDiffusionSetup(const int dim,
+                                   const int D1D,
+                                   const int Q1D,
+                                   const int NE,
+                                   const Array<double> &W,
+                                   const Vector &J,
+                                   const double COEFF,
+                                   Vector &op)
+{
+   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PADiffusionSetup"); }
+   if (dim == 2)
+   {
+      PAVectorDiffusionSetup2D(Q1D, NE, W, J, COEFF, op);
+   }
+   if (dim == 3)
+   {
+      PAVectorDiffusionSetup3D(Q1D, NE, W, J, COEFF, op);
+   }
+}
+
+void VectorDiffusionIntegrator::Setup(const FiniteElementSpace &fes)
+{
+   // Assumes tensor-product elements
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement &el = *fes.GetFE(0);
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el);
+   const int dims = el.GetDim();
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   ne = fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   dofs1D = maps->ndof;
+   quad1D = maps->nqpt;
+   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
+   double coeff = 1.0;
+   if (Q)
+   {
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      coeff = cQ->constant;
+   }
+   PAVectorDiffusionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
+                          coeff, pa_data);
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAVectorDiffusionDiagonal2D(const int NE,
+                                                const Array<double> &b,
+                                                const Array<double> &g,
+                                                const Vector &op,
+                                                Vector &diag,
+                                                const int d1d = 0,
+                                                const int q1d = 0)
+{
+   // see eg PADiffusionApply2D
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 2;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   // note different shape for op, this is a (symmetric) matrix,
+   // we only store necessary entries
+   auto Q = Reshape(op.Read(), Q1D * Q1D, 3, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, VDIM, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+      double temp01[max_Q1D][max_D1D];
+      double temp02[max_Q1D][max_D1D];
+      double temp03[max_Q1D][max_D1D];
+      double temp04[max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            temp01[qx][dy] = 0.0;
+            temp02[qx][dy] = 0.0;
+            temp03[qx][dy] = 0.0;
+            temp04[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const int q = qx + qy * Q1D;
+               const double O11 = Q(q,0,e);
+               const double O12 = Q(q,1,e);
+               const double O22 = Q(q,2,e);
+               temp01[qx][dy]   += B(qy, dy) * B(qy, dy) * O11;
+               temp02[qx][dy]   += B(qy, dy) * G(qy, dy) * O12;
+               temp03[qx][dy] += G(qy, dy) * B(qy, dy) * O12;
+               temp04[qx][dy]  += G(qy, dy) * G(qy, dy) * O22;
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               Y(dx,dy,0,e) += G(qx, dx) * G(qx, dx) * temp01[qx][dy];
+               Y(dx,dy,0,e) += G(qx, dx) * B(qx, dx) * temp02[qx][dy];
+               Y(dx,dy,0,e) += B(qx, dx) * G(qx, dx) * temp03[qx][dy];
+               Y(dx,dy,0,e) += B(qx, dx) * B(qx, dx) * temp04[qx][dy];
+            }
+            Y(dx,dy,1,e) = Y(dx,dy,0,e);
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAVectorDiffusionDiagonal3D(const int NE,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &op,
+                                  Vector &y,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   // see eg PADiffusionApply3D
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int VDIM = 3;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Q = Reshape(op.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      // gradphi \cdot OP \gradphi has nine terms
+      // nine terms might be too many, but for proof of concept that's what I'll do
+      // (you could use symmetry to only have six?)
+
+      // nine terms:
+      // one   Gx By Bz O11 Gx By Bz;
+      // two   Gx By Bz O12 Bx Gy Bz;
+      // three Gx By Bz O13 Bx By Gz;
+      // four  Bx Gy Bz O21 Gx By Bz;
+      // five  Bx Gy Bz O22 Bx Gy Bz;
+      // six   Bx Gy Bz O23 Bx By Gz;
+      // seven Bx By Gz O31 Gx By Bz;
+      // eight Bx By Gz O32 Bx Gy Bz;
+      // nine  Bx By Gz O33 Bx By Gz;
+
+      double ztemp01[max_Q1D][max_Q1D][max_D1D];
+      double ztemp02[max_Q1D][max_Q1D][max_D1D];
+      double ztemp03[max_Q1D][max_Q1D][max_D1D];
+      double ztemp04[max_Q1D][max_Q1D][max_D1D];
+      double ztemp05[max_Q1D][max_Q1D][max_D1D];
+      double ztemp06[max_Q1D][max_Q1D][max_D1D];
+      double ztemp07[max_Q1D][max_Q1D][max_D1D];
+      double ztemp08[max_Q1D][max_Q1D][max_D1D];
+      double ztemp09[max_Q1D][max_Q1D][max_D1D];
+
+      // first tensor contraction, along z direction
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               ztemp01[qx][qy][dz] = 0.0;
+               ztemp02[qx][qy][dz] = 0.0;
+               ztemp03[qx][qy][dz] = 0.0;
+               ztemp04[qx][qy][dz] = 0.0;
+               ztemp05[qx][qy][dz] = 0.0;
+               ztemp06[qx][qy][dz] = 0.0;
+               ztemp07[qx][qy][dz] = 0.0;
+               ztemp08[qx][qy][dz] = 0.0;
+               ztemp09[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  const int q = qx + (qy + qz * Q1D) * Q1D;
+                  const double O11 = Q(q,0,e);
+                  const double O12 = Q(q,1,e);
+                  const double O13 = Q(q,2,e);
+                  const double O22 = Q(q,3,e);
+                  const double O23 = Q(q,4,e);
+                  const double O33 = Q(q,5,e);
+
+                  ztemp01[qx][qy][dz] += B(qz, dz) * B(qz, dz) * O11;
+                  ztemp02[qx][qy][dz] += B(qz, dz) * B(qz, dz) * O12;
+                  ztemp03[qx][qy][dz] += B(qz, dz) * G(qz, dz) * O13;
+                  ztemp04[qx][qy][dz] += B(qz, dz) * B(qz, dz) * O12;
+                  ztemp05[qx][qy][dz] += B(qz, dz) * B(qz, dz) * O22;
+                  ztemp06[qx][qy][dz] += B(qz, dz) * G(qz, dz) * O23;
+                  ztemp07[qx][qy][dz] += G(qz, dz) * B(qz, dz) * O13;
+                  ztemp08[qx][qy][dz] += G(qz, dz) * B(qz, dz) * O23;
+                  ztemp09[qx][qy][dz] += G(qz, dz) * G(qz, dz) * O33;
+               }
+            }
+         }
+      }
+
+      double ytemp01[max_Q1D][max_D1D][max_D1D];
+      double ytemp02[max_Q1D][max_D1D][max_D1D];
+      double ytemp03[max_Q1D][max_D1D][max_D1D];
+      double ytemp04[max_Q1D][max_D1D][max_D1D];
+      double ytemp05[max_Q1D][max_D1D][max_D1D];
+      double ytemp06[max_Q1D][max_D1D][max_D1D];
+      double ytemp07[max_Q1D][max_D1D][max_D1D];
+      double ytemp08[max_Q1D][max_D1D][max_D1D];
+      double ytemp09[max_Q1D][max_D1D][max_D1D];
+
+      // second tensor contraction, along y direction
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               ytemp01[qx][dy][dz] = 0.0;
+               ytemp02[qx][dy][dz] = 0.0;
+               ytemp03[qx][dy][dz] = 0.0;
+               ytemp04[qx][dy][dz] = 0.0;
+               ytemp05[qx][dy][dz] = 0.0;
+               ytemp06[qx][dy][dz] = 0.0;
+               ytemp07[qx][dy][dz] = 0.0;
+               ytemp08[qx][dy][dz] = 0.0;
+               ytemp09[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  ytemp01[qx][dy][dz] += B(qy, dy) * B(qy, dy) * ztemp01[qx][qy][dz];
+                  ytemp02[qx][dy][dz] += B(qy, dy) * G(qy, dy) * ztemp02[qx][qy][dz];
+                  ytemp03[qx][dy][dz] += B(qy, dy) * B(qy, dy) * ztemp03[qx][qy][dz];
+                  ytemp04[qx][dy][dz] += G(qy, dy) * B(qy, dy) * ztemp04[qx][qy][dz];
+                  ytemp05[qx][dy][dz] += G(qy, dy) * G(qy, dy) * ztemp05[qx][qy][dz];
+                  ytemp06[qx][dy][dz] += G(qy, dy) * B(qy, dy) * ztemp06[qx][qy][dz];
+                  ytemp07[qx][dy][dz] += B(qy, dy) * B(qy, dy) * ztemp07[qx][qy][dz];
+                  ytemp08[qx][dy][dz] += B(qy, dy) * G(qy, dy) * ztemp08[qx][qy][dz];
+                  ytemp09[qx][dy][dz] += B(qy, dy) * B(qy, dy) * ztemp09[qx][qy][dz];
+               }
+            }
+         }
+      }
+
+      // third tensor contraction, along x direction
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  Y(dx, dy, dz, 0, e) += G(qx, dx) * G(qx, dx) * ytemp01[qx][dy][dz];
+                  Y(dx, dy, dz, 0, e) += G(qx, dx) * B(qx, dx) * ytemp02[qx][dy][dz];
+                  Y(dx, dy, dz, 0, e) += G(qx, dx) * B(qx, dx) * ytemp03[qx][dy][dz];
+                  Y(dx, dy, dz, 0, e) += B(qx, dx) * G(qx, dx) * ytemp04[qx][dy][dz];
+                  Y(dx, dy, dz, 0, e) += B(qx, dx) * B(qx, dx) * ytemp05[qx][dy][dz];
+                  Y(dx, dy, dz, 0, e) += B(qx, dx) * B(qx, dx) * ytemp06[qx][dy][dz];
+                  Y(dx, dy, dz, 0, e) += B(qx, dx) * G(qx, dx) * ytemp07[qx][dy][dz];
+                  Y(dx, dy, dz, 0, e) += B(qx, dx) * B(qx, dx) * ytemp08[qx][dy][dz];
+                  Y(dx, dy, dz, 0, e) += B(qx, dx) * B(qx, dx) * ytemp09[qx][dy][dz];
+               }
+              Y(dx,dy,dz,1,e) = Y(dx,dy,dz,0,e);
+              Y(dx,dy,dz,2,e) = Y(dx,dy,dz,0,e);
+            }
+         }
+      }
+
+   });
+}
+
+
+static void PAVectorDiffusionAssembleDiagonal(const int dim,
+                                              const int D1D,
+                                              const int Q1D,
+                                              const int NE,
+                                              const Array<double> &B,
+                                              const Array<double> &Bt,
+                                              const Vector &op,
+                                              Vector &y)
+{
+   const int DQ = (D1D << 4) | Q1D;
+   if (dim == 2)
+   {
+      switch ((D1D << 4) | Q1D)
+      {
+      case 0x44:
+         return PAVectorDiffusionDiagonal2D<4, 4>(NE, B, Bt, op, y, D1D, Q1D);
+      case 0x66:
+         return PAVectorDiffusionDiagonal2D<6, 6>(NE, B, Bt, op, y, D1D, Q1D);
+      default:
+         printf("%x, %x(%d): %X", D1D, Q1D, Q1D, DQ);
+      }
+   }
+   else if (dim == 3)
+   {
+     switch ((D1D << 4) | Q1D)
+     {
+       case 0x34:
+         return PAVectorDiffusionDiagonal3D<3, 4>(NE, B, Bt, op, y, D1D, Q1D);
+       case 0x89:
+         return PAVectorDiffusionDiagonal3D<8, 9>(NE, B, Bt, op, y, D1D, Q1D);
+       default:
+         printf("%x, %x(%d): %X", D1D, Q1D, Q1D, DQ);
+     }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag) const
+{
+   PAVectorDiffusionAssembleDiagonal(dim,
+                                     dofs1D,
+                                     quad1D,
+                                     ne,
+                                     maps->B,
+                                     maps->Bt,
+                                     pa_data,
+                                     diag);
+}
+
+// PA Diffusion Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0> static
+void PAVectorDiffusionApply2D(const int NE,
+                              const Array<double> &b,
+                              const Array<double> &g,
+                              const Array<double> &bt,
+                              const Array<double> &gt,
+                              const Vector &_op,
+                              const Vector &_x,
+                              Vector &_y,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 2;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Gt = Reshape(gt.Read(), D1D, Q1D);
+   auto op = Reshape(_op.Read(), Q1D*Q1D, 3, NE);
+   auto x = Reshape(_x.Read(), D1D, D1D, VDIM, NE);
+   auto y = Reshape(_y.ReadWrite(), D1D, D1D, VDIM, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      for (int c = 0; c < VDIM; ++ c)
+      {
+         double grad[max_Q1D][max_Q1D][2];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               grad[qy][qx][0] = 0.0;
+               grad[qy][qx][1] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            double gradX[max_Q1D][2];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradX[qx][0] = 0.0;
+               gradX[qx][1] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double s = x(dx,dy,c,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradX[qx][0] += s * B(qx,dx);
+                  gradX[qx][1] += s * G(qx,dx);
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy  = B(qy,dy);
+               const double wDy = G(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  grad[qy][qx][0] += gradX[qx][1] * wy;
+                  grad[qy][qx][1] += gradX[qx][0] * wDy;
+               }
+            }
+         }
+         // Calculate Dxy, xDy in plane
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + qy * Q1D;
+
+               const double O11 = op(q,0,e);
+               const double O12 = op(q,1,e);
+               const double O22 = op(q,2,e);
+
+               const double gradX = grad[qy][qx][0];
+               const double gradY = grad[qy][qx][1];
+
+               grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
+               grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double gradX[max_D1D][2];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               gradX[dx][0] = 0;
+               gradX[dx][1] = 0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double gX = grad[qy][qx][0];
+               const double gY = grad[qy][qx][1];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const double wx  = Bt(dx,qx);
+                  const double wDx = Gt(dx,qx);
+                  gradX[dx][0] += gX * wDx;
+                  gradX[dx][1] += gY * wx;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double wy  = Bt(dy,qy);
+               const double wDy = Gt(dy,qy);
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  y(dx,dy,c,e) += ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy));
+               }
+            }
+         }
+      }
+   });
+}
+
+// PA Diffusion Apply 3D kernel
+template<const int T_D1D = 0,
+         const int T_Q1D = 0> static
+void PAVectorDiffusionApply3D(const int NE,
+                              const Array<double> &b,
+                              const Array<double> &g,
+                              const Array<double> &bt,
+                              const Array<double> &gt,
+                              const Vector &_op,
+                              const Vector &_x,
+                              Vector &_y,
+                              int d1d = 0, int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 3;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Gt = Reshape(gt.Read(), D1D, Q1D);
+   auto op = Reshape(_op.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto x = Reshape(_x.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(_y.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+      for (int c = 0; c < VDIM; ++ c)
+      {
+         double grad[max_Q1D][max_Q1D][max_Q1D][3];
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  grad[qz][qy][qx][0] = 0.0;
+                  grad[qz][qy][qx][1] = 0.0;
+                  grad[qz][qy][qx][2] = 0.0;
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            double gradXY[max_Q1D][max_Q1D][3];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradXY[qy][qx][0] = 0.0;
+                  gradXY[qy][qx][1] = 0.0;
+                  gradXY[qy][qx][2] = 0.0;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               double gradX[max_Q1D][2];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradX[qx][0] = 0.0;
+                  gradX[qx][1] = 0.0;
+               }
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const double s = x(dx,dy,dz,c,e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     gradX[qx][0] += s * B(qx,dx);
+                     gradX[qx][1] += s * G(qx,dx);
+                  }
+               }
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy  = B(qy,dy);
+                  const double wDy = G(qy,dy);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double wx  = gradX[qx][0];
+                     const double wDx = gradX[qx][1];
+                     gradXY[qy][qx][0] += wDx * wy;
+                     gradXY[qy][qx][1] += wx  * wDy;
+                     gradXY[qy][qx][2] += wx  * wy;
+                  }
+               }
+            }
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz  = B(qz,dz);
+               const double wDz = G(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+                     grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+                     grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+                  }
+               }
+            }
+         }
+         // Calculate Dxyz, xDyz, xyDz in plane
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const int q = qx + (qy + qz * Q1D) * Q1D;
+                  const double O11 = op(q,0,e);
+                  const double O12 = op(q,1,e);
+                  const double O13 = op(q,2,e);
+                  const double O22 = op(q,3,e);
+                  const double O23 = op(q,4,e);
+                  const double O33 = op(q,5,e);
+                  const double gradX = grad[qz][qy][qx][0];
+                  const double gradY = grad[qz][qy][qx][1];
+                  const double gradZ = grad[qz][qy][qx][2];
+                  grad[qz][qy][qx][0] = (O11*gradX)+(O12*gradY)+(O13*gradZ);
+                  grad[qz][qy][qx][1] = (O12*gradX)+(O22*gradY)+(O23*gradZ);
+                  grad[qz][qy][qx][2] = (O13*gradX)+(O23*gradY)+(O33*gradZ);
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            double gradXY[max_D1D][max_D1D][3];
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  gradXY[dy][dx][0] = 0;
+                  gradXY[dy][dx][1] = 0;
+                  gradXY[dy][dx][2] = 0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double gradX[max_D1D][3];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  gradX[dx][0] = 0;
+                  gradX[dx][1] = 0;
+                  gradX[dx][2] = 0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double gX = grad[qz][qy][qx][0];
+                  const double gY = grad[qz][qy][qx][1];
+                  const double gZ = grad[qz][qy][qx][2];
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     const double wx  = Bt(dx,qx);
+                     const double wDx = Gt(dx,qx);
+                     gradX[dx][0] += gX * wDx;
+                     gradX[dx][1] += gY * wx;
+                     gradX[dx][2] += gZ * wx;
+                  }
+               }
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  const double wy  = Bt(dy,qy);
+                  const double wDy = Gt(dy,qy);
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     gradXY[dy][dx][0] += gradX[dx][0] * wy;
+                     gradXY[dy][dx][1] += gradX[dx][1] * wDy;
+                     gradXY[dy][dx][2] += gradX[dx][2] * wy;
+                  }
+               }
+            }
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double wz  = Bt(dz,qz);
+               const double wDz = Gt(dz,qz);
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     y(dx,dy,dz,c,e) +=
+                        ((gradXY[dy][dx][0] * wz) +
+                         (gradXY[dy][dx][1] * wz) +
+                         (gradXY[dy][dx][2] * wDz));
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorDiffusionApply(const int dim,
+                                   const int D1D,
+                                   const int Q1D,
+                                   const int NE,
+                                   const Array<double> &B,
+                                   const Array<double> &G,
+                                   const Array<double> &Bt,
+                                   const Array<double> &Gt,
+                                   const Vector &op,
+                                   const Vector &x,
+                                   Vector &y)
+{
+   if (dim == 2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return PAVectorDiffusionApply2D<2,2>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x33: return PAVectorDiffusionApply2D<3,3>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x44: return PAVectorDiffusionApply2D<4,4>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x55: return PAVectorDiffusionApply2D<5,5>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x66: return PAVectorDiffusionApply2D<6,6>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x77: return PAVectorDiffusionApply2D<7,7>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x88: return PAVectorDiffusionApply2D<8,8>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x99: return PAVectorDiffusionApply2D<9,9>(NE,B,G,Bt,Gt,op,x,y);
+         default:   return PAVectorDiffusionApply2D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
+      }
+   }
+   if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: return PAVectorDiffusionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x34: return PAVectorDiffusionApply3D<3,4>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x45: return PAVectorDiffusionApply3D<4,5>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x56: return PAVectorDiffusionApply3D<5,6>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x67: return PAVectorDiffusionApply3D<6,7>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x78: return PAVectorDiffusionApply3D<7,8>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x89: return PAVectorDiffusionApply3D<8,9>(NE,B,G,Bt,Gt,op,x,y);
+         default:   return PAVectorDiffusionApply3D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+// PA Diffusion Apply kernel
+void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   PAVectorDiffusionApply(dim, dofs1D, quad1D, ne,
+                          maps->B, maps->G, maps->Bt, maps->Gt,
+                          pa_data, x, y);
+}
+
+} // namespace mfem

fem/bilininteg_vecmass.cpp

@@ -0,0 +1,560 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// PA Mass Integrator
+
+// PA Mass Assemble kernel
+void VectorMassIntegrator::Setup(const FiniteElementSpace &fes)
+{
+   // Assuming the same element type
+   Mesh *mesh = fes.GetMesh();
+   if (mesh->GetNE() == 0) { return; }
+   const FiniteElement &el = *fes.GetFE(0);
+   ElementTransformation *T = mesh->GetElementTransformation(0);
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T);
+   dim = mesh->Dimension();
+   ne = fes.GetMesh()->GetNE();
+   nq = ir->GetNPoints();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
+                                    GeometricFactors::JACOBIANS);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   dofs1D = maps->ndof;
+   quad1D = maps->nqpt;
+   pa_data.SetSize(ne*nq, Device::GetMemoryType());
+   double coeff = 1.0;
+   if (Q)
+   {
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      coeff = cQ->constant;
+   }
+   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
+   if (dim==2)
+   {
+      const double constant = coeff;
+      const int NE = ne;
+      const int NQ = nq;
+      auto w = ir->GetWeights().Read();
+      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
+      auto v = Reshape(pa_data.Write(), NQ, NE);
+      MFEM_FORALL(e, NE,
+      {
+         for (int q = 0; q < NQ; ++q)
+         {
+            const double J11 = J(q,0,0,e);
+            const double J12 = J(q,1,0,e);
+            const double J21 = J(q,0,1,e);
+            const double J22 = J(q,1,1,e);
+            const double detJ = (J11*J22)-(J21*J12);
+            v(q,e) =  w[q] * constant * detJ;
+         }
+      });
+   }
+   if (dim==3)
+   {
+      const double constant = coeff;
+      const int NE = ne;
+      const int NQ = nq;
+      auto W = ir->GetWeights().Read();
+      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
+      auto v = Reshape(pa_data.Write(), NQ,NE);
+      MFEM_FORALL(e, NE,
+      {
+         for (int q = 0; q < NQ; ++q)
+         {
+            const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e);
+            const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e);
+            const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e);
+            const double detJ = J11 * (J22 * J33 - J32 * J23) -
+            /* */               J21 * (J12 * J33 - J32 * J13) +
+            /* */               J31 * (J12 * J23 - J22 * J13);
+            v(q,e) = W[q] * constant * detJ;
+         }
+      });
+   }
+}
+
+template<const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal2D(const int NE,
+                                           const Array<double> &_B,
+                                           const Array<double> &_Bt,
+                                           const Vector &_op,
+                                           Vector &_diag,
+                                           const int d1d = 0,
+                                           const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 2;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(_B.Read(), Q1D, D1D);
+   auto op = Reshape(_op.Read(), Q1D, Q1D, NE);
+   auto y = Reshape(_diag.ReadWrite(), D1D, D1D, VDIM, NE);
+   MFEM_FORALL(e, NE, {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      double temp[max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            temp[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               temp[qx][dy] += B(qy, dy) * B(qy, dy) * op(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               y(dx, dy, 0, e) += B(qx, dx) * B(qx, dx) * temp[qx][dy];
+            }
+            y(dx, dy, 1, e) = y(dx, dy, 0, e);
+         }
+      }
+   });
+}
+
+template<const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal3D(const int NE,
+                                           const Array<double> &_B,
+                                           const Array<double> &_Bt,
+                                           const Vector &_op,
+                                           Vector &_diag,
+                                           const int d1d = 0,
+                                           const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 3;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(_B.Read(), Q1D, D1D);
+   // auto Bt = Reshape(_Bt.Read(), D1D, Q1D); // ?? (TODO atb@llnl.gov)
+   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, NE);
+   auto y = Reshape(_diag.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   MFEM_FORALL(e, NE, {
+      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      double temp[max_Q1D][max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               temp[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  temp[qx][qy][dz] += B(qz, dz) * B(qz, dz) * op(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      double temp2[max_Q1D][max_D1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               temp2[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  temp2[qx][dy][dz] += B(qy, dy) * B(qy, dy) * temp[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  y(dx, dy, dz, 0, e) += B(qx, dx) * B(qx, dx)
+                                         * temp2[qx][dy][dz];
+               }
+               y(dx, dy, dz, 1, e) = y(dx, dy, dz, 0, e);
+               y(dx, dy, dz, 2, e) = y(dx, dy, dz, 0, e);
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorMassAssembleDiagonal(const int dim,
+                                         const int D1D,
+                                         const int Q1D,
+                                         const int NE,
+                                         const Array<double> &B,
+                                         const Array<double> &Bt,
+                                         const Vector &op,
+                                         Vector &y)
+{
+   const int DQ = (D1D << 4) | Q1D;
+   if (dim == 2)
+   {
+      switch ((D1D << 4) | Q1D)
+      {
+      case 0x44:
+         return PAVectorMassAssembleDiagonal2D<4, 4>(NE, B, Bt, op, y, D1D, Q1D);
+      case 0x66:
+         return PAVectorMassAssembleDiagonal2D<6, 6>(NE, B, Bt, op, y, D1D, Q1D);
+      default:
+         printf("%x, %x(%d): %X", D1D, Q1D, Q1D, DQ);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4) | Q1D)
+      {
+      /* case 0x55: */
+      /*    return PAVectorMassAssembleDiagonal3D<5, 5>(NE, B, Bt, op, y, D1D, Q1D); */
+      /* case 0x88: */
+      /*    return PAVectorMassAssembleDiagonal3D<8, 8>(NE, B, Bt, op, y, D1D, Q1D); */
+      /* case 0x89: */
+      /*    return PAVectorMassAssembleDiagonal3D<8, 9>(NE, B, Bt, op, y, D1D, Q1D); */
+      default:
+         return PAVectorMassAssembleDiagonal3D(NE, B, Bt, op, y, D1D, Q1D);
+         printf("%x, %x(%d): %X", D1D, Q1D, Q1D, DQ);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag) const
+{
+   PAVectorMassAssembleDiagonal(dim,
+                                dofs1D,
+                                quad1D,
+                                ne,
+                                maps->B,
+                                maps->Bt,
+                                pa_data,
+                                diag);
+}
+
+template<const int T_D1D = 0,
+         const int T_Q1D = 0>
+static void PAVectorMassApply2D(const int NE,
+                                const Array<double> &_B,
+                                const Array<double> &_Bt,
+                                const Vector &_op,
+                                const Vector &_x,
+                                Vector &_y,
+                                const int d1d = 0,
+                                const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 2;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(_B.Read(), Q1D, D1D);
+   auto Bt = Reshape(_Bt.Read(), D1D, Q1D);
+   auto op = Reshape(_op.Read(), Q1D, Q1D, NE);
+   auto x = Reshape(_x.Read(), D1D, D1D, VDIM, NE);
+   auto y = Reshape(_y.ReadWrite(), D1D, D1D, VDIM, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+      double sol_xy[max_Q1D][max_Q1D];
+      for (int c = 0; c < VDIM; ++c)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            double sol_x[max_Q1D];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               sol_x[qy] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double s = x(dx,dy,c,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_x[qx] += B(qx,dx)* s;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double d2q = B(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_xy[qy][qx] += d2q * sol_x[qx];
+               }
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] *= op(qx,qy,e);
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double sol_x[max_D1D];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_x[dx] = 0.0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double s = sol_xy[qy][qx];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  sol_x[dx] += Bt(dx,qx) * s;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double q2d = Bt(dy,qy);
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  y(dx,dy,c,e) += q2d * sol_x[dx];
+               }
+            }
+         }
+      }
+   });
+}
+
+template<const int T_D1D = 0,
+         const int T_Q1D = 0>
+static void PAVectorMassApply3D(const int NE,
+                                const Array<double> &_B,
+                                const Array<double> &_Bt,
+                                const Vector &_op,
+                                const Vector &_x,
+                                Vector &_y,
+                                const int d1d = 0,
+                                const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 3;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(_B.Read(), Q1D, D1D);
+   auto Bt = Reshape(_Bt.Read(), D1D, Q1D);
+   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, NE);
+   auto x = Reshape(_x.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(_y.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
+      for (int c = 0; c < VDIM; ++ c)
+      {
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_xyz[qz][qy][qx] = 0.0;
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            double sol_xy[max_Q1D][max_Q1D];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_xy[qy][qx] = 0.0;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               double sol_x[max_Q1D];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_x[qx] = 0;
+               }
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const double s = x(dx,dy,dz,c,e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     sol_x[qx] += B(qx,dx) * s;
+                  }
+               }
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = B(qy,dy);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     sol_xy[qy][qx] += wy * sol_x[qx];
+                  }
+               }
+            }
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = B(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
+                  }
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_xyz[qz][qy][qx] *= op(qx,qy,qz,e);
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            double sol_xy[max_D1D][max_D1D];
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  sol_xy[dy][dx] = 0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double sol_x[max_D1D];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  sol_x[dx] = 0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double s = sol_xyz[qz][qy][qx];
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     sol_x[dx] += Bt(dx,qx) * s;
+                  }
+               }
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  const double wy = Bt(dy,qy);
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     sol_xy[dy][dx] += wy * sol_x[dx];
+                  }
+               }
+            }
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double wz = Bt(dz,qz);
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     y(dx,dy,dz,c,e) += wz * sol_xy[dy][dx];
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorMassApply(const int dim,
+                              const int D1D,
+                              const int Q1D,
+                              const int NE,
+                              const Array<double> &B,
+                              const Array<double> &Bt,
+                              const Vector &op,
+                              const Vector &x,
+                              Vector &y)
+{
+   if (dim == 2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return PAVectorMassApply2D<2,2>(NE, B, Bt, op, x, y);
+         case 0x33: return PAVectorMassApply2D<3,3>(NE, B, Bt, op, x, y);
+         case 0x44: return PAVectorMassApply2D<4,4>(NE, B, Bt, op, x, y);
+         case 0x55: return PAVectorMassApply2D<5,5>(NE, B, Bt, op, x, y);
+         case 0x66: return PAVectorMassApply2D<6,6>(NE, B, Bt, op, x, y);
+         case 0x77: return PAVectorMassApply2D<7,7>(NE, B, Bt, op, x, y);
+         case 0x88: return PAVectorMassApply2D<8,8>(NE, B, Bt, op, x, y);
+         case 0x99: return PAVectorMassApply2D<9,9>(NE, B, Bt, op, x, y);
+         default:   return PAVectorMassApply2D(NE, B, Bt, op, x, y, D1D, Q1D);
+      }
+   }
+   if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: return PAVectorMassApply3D<2,3>(NE, B, Bt, op, x, y);
+         case 0x34: return PAVectorMassApply3D<3,4>(NE, B, Bt, op, x, y);
+         case 0x45: return PAVectorMassApply3D<4,5>(NE, B, Bt, op, x, y);
+         case 0x56: return PAVectorMassApply3D<5,6>(NE, B, Bt, op, x, y);
+         case 0x67: return PAVectorMassApply3D<6,7>(NE, B, Bt, op, x, y);
+         case 0x78: return PAVectorMassApply3D<7,8>(NE, B, Bt, op, x, y);
+         case 0x89: return PAVectorMassApply3D<8,9>(NE, B, Bt, op, x, y);
+         default:   return PAVectorMassApply3D(NE, B, Bt, op, x, y, D1D, Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
+}
+
+} // namespace mfem

fem/coefficient.cpp

@@ -46,10 +46,27 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
    }
 }
 
-double GridFunctionCoefficient::Eval (ElementTransformation &T,
-                                      const IntegrationPoint &ip)
+double GridFunctionCoefficient::Eval(ElementTransformation &T,
+                                     const IntegrationPoint &ip)
 {
-   return GridF -> GetValue (T.ElementNo, ip, Component);
+   Mesh *mesh = GridF->FESpace()->GetMesh();
+   if (mesh->Dimension() == T.GetDimension())
+   {
+      return GridF->GetValue(T.ElementNo, ip, Component);
+   }
+   else // Assuming T is a boundary element transformation
+   {
+      int el_id, el_info;
+      mesh->GetBdrElementAdjacentElement(T.ElementNo, el_id, el_info);
+      IntegrationPointTransformation loc_T;
+      mesh->GetLocalFaceTransformation(mesh->GetBdrElementType(T.ElementNo),
+                                       mesh->GetElementType(el_id),
+                                       loc_T.Transf,
+                                       el_info);
+      IntegrationPoint eip;
+      loc_T.Transform(ip, eip);
+      return GridF->GetValue(el_id, eip, Component);
+   }
 }
 
 double TransformedCoefficient::Eval(ElementTransformation &T,
@@ -171,10 +188,28 @@ void VectorGridFunctionCoefficient::SetGridFunction(GridFunction *gf)
    GridFunc = gf; vdim = (gf) ? gf -> VectorDim() : 0;
 }
 
-void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
+void VectorGridFunctionCoefficient::Eval(Vector &V,
+                                         ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
-   GridFunc->GetVectorValue(T.ElementNo, ip, V);
+   Mesh *mesh = GridFunc->FESpace()->GetMesh();
+   if (mesh->Dimension() == T.GetDimension())
+   {
+      GridFunc->GetVectorValue(T.ElementNo, ip, V);
+   }
+   else // Assuming T is a boundary element transformation
+   {
+      int el_id, el_info;
+      mesh->GetBdrElementAdjacentElement(T.ElementNo, el_id, el_info);
+      IntegrationPointTransformation loc_T;
+      mesh->GetLocalFaceTransformation(mesh->GetBdrElementType(T.ElementNo),
+                                       mesh->GetElementType(el_id),
+                                       loc_T.Transf,
+                                       el_info);
+      IntegrationPoint eip;
+      loc_T.Transform(ip, eip);
+      GridFunc->GetVectorValue(el_id, eip, V);
+   }
 }
 
 void VectorGridFunctionCoefficient::Eval(

fem/fespace.cpp

@@ -801,8 +801,14 @@ const Operator *FiniteElementSpace::GetElementRestriction(
 {
    // Check if we have a discontinuous space using the FE collection:
    const L2_FECollection *dg_space = dynamic_cast<const L2_FECollection*>(fec);
-   if (dg_space) { return NULL; }
-   // TODO: support other DG collections.
+   if (dg_space)
+   {
+      if (L2E_nat.Ptr() == NULL)
+      {
+         L2E_nat.Reset(new L2ElementRestriction(*this));
+      }
+      return L2E_nat.Ptr();
+   }
    if (e_ordering == ElementDofOrdering::LEXICOGRAPHIC)
    {
       if (L2E_lex.Ptr() == NULL)
@@ -2691,6 +2697,36 @@ const Operator &L2ProjectionGridTransfer::BackwardOperator()
    return *B;
 }
 
+L2ElementRestriction::L2ElementRestriction(const FiniteElementSpace &fes)
+   : ne(fes.GetNE()),
+     vdim(fes.GetVDim()),
+     byvdim(fes.GetOrdering() == Ordering::byVDIM),
+     ndof(ne > 0 ? fes.GetFE(0)->GetDof() : 0)
+{ }
+
+void L2ElementRestriction::Mult(const Vector &x, Vector &y) const
+{
+   for (int iel=0; iel<ne; ++iel)
+   {
+      for (int vd=0; vd<vdim; ++vd)
+      {
+         for (int idof=0; idof<ndof; ++idof)
+         {
+            int yidx = iel*vdim*ndof + vd*ndof + idof;
+            int xidx;
+            if (byvdim)
+            {
+               xidx = iel*ndof*vdim + idof*vdim + vd;
+            }
+            else
+            {
+               xidx = vd*ne*ndof + iel*ndof + idof;
+            }
+            y[yidx] = x[xidx];
+         }
+      }
+   }
+}
 
 ElementRestriction::ElementRestriction(const FiniteElementSpace &f,
                                        ElementDofOrdering e_ordering)

fem/fespace.hpp

@@ -902,6 +902,16 @@ public:
    void MultTranspose(const Vector &x, Vector &y) const;
 };
 
+class L2ElementRestriction : public Operator
+{
+   const int ne;
+   const int vdim;
+   const bool byvdim;
+   const int ndof;
+public:
+   L2ElementRestriction(const FiniteElementSpace&);
+   void Mult(const Vector &x, Vector &y) const;
+};
 
 /** @brief A class that performs interpolation from an E-vector to quadrature
     point values and/or derivatives (Q-vectors). */
@@ -923,12 +933,13 @@ protected:
 
    mutable bool use_tensor_products;
 
-   static const int MAX_NQ2D = 100;
-   static const int MAX_ND2D = 100;
+   // TODO:WP:this is temporary
+   static const int MAX_NQ2D = 4000;
+   static const int MAX_ND2D = 1000;
    static const int MAX_VDIM2D = 2;
 
-   static const int MAX_NQ3D = 1000;
-   static const int MAX_ND3D = 1000;
+   static const int MAX_NQ3D = 240000;
+   static const int MAX_ND3D = 30000;
    static const int MAX_VDIM3D = 3;
 
 public:

fem/mgbilinearform.cpp

@@ -27,6 +27,32 @@ MultigridBilinearForm::MultigridBilinearForm(SpaceHierarchy& spaceHierarchy,
 {
    MFEM_VERIFY(bf.GetAssemblyLevel() == AssemblyLevel::PARTIAL,
                "Assembly level must be PARTIAL");
+   SetupPA(spaceHierarchy, bf, ess_bdr);
+}
+
+MultigridBilinearForm::MultigridBilinearForm(SpaceHierarchy& spaceHierarchy,
+                                             SparseMatrix& opr, Array<int>& ess_bdr)
+    : MultigridOperator()
+{
+   SetupFull(spaceHierarchy, opr, ess_bdr);
+}
+
+MultigridBilinearForm::~MultigridBilinearForm()
+{
+   for (int i = 0; i < bfs.Size(); ++i)
+   {
+      delete bfs[i];
+   }
+
+   for (int i = 0; i < essentialTrueDofs.Size(); ++i)
+   {
+      delete essentialTrueDofs[i];
+   }
+}
+
+void MultigridBilinearForm::SetupPA(SpaceHierarchy& spaceHierarchy,
+                                    BilinearForm& bf, Array<int>& ess_bdr)
+{
    BilinearForm* form = new BilinearForm(&spaceHierarchy.GetFESpaceAtLevel(0));
    // TODO: Copy all integrators
    Array<BilinearFormIntegrator*>& dbfi = *bf.GetDBFI();
@@ -108,17 +134,56 @@ MultigridBilinearForm::MultigridBilinearForm(SpaceHierarchy& spaceHierarchy,
    }
 }
 
-MultigridBilinearForm::~MultigridBilinearForm()
+void MultigridBilinearForm::SetupFull(SpaceHierarchy& spaceHierarchy,
+                                      SparseMatrix& opr, Array<int>& ess_bdr)
 {
-   for (int i = 0; i < bfs.Size(); ++i)
+   AddEmptyLevels(spaceHierarchy.GetNumLevels());
+
+   width = opr.Width();
+   height = opr.Height();
+
+   operators[spaceHierarchy.GetFinestLevelIndex()] = &opr;
+
+   for (int level = spaceHierarchy.GetFinestLevelIndex(); level > 0; --level)
    {
-      delete bfs[i];
+      smoothers[level] = new GSSmoother((SparseMatrix&)*operators[level]);
+
+      SparseMatrix* R =
+          spaceHierarchy.GetFESpaceAtLevel(level).H2L_GlobalRestrictionMatrix(
+              &spaceHierarchy.GetFESpaceAtLevel(level - 1));
+      SparseMatrix* P = mfem::Transpose(*R);
+      prolongations[level - 1] = P;
+
+
+      SparseMatrix* rap = mfem::RAP(*P, (SparseMatrix&)*operators[level], *P);
+
+      Array<int>* ess_tdof_list = new Array<int>();
+      essentialTrueDofs.Append(ess_tdof_list);
+      spaceHierarchy.GetFESpaceAtLevel(level-1).GetEssentialTrueDofs(
+          ess_bdr, *ess_tdof_list);
+
+      SparseMatrix* elim = new SparseMatrix(rap->Height());
+
+      for (int i = 0; i < ess_tdof_list->Size(); i++)
+      {
+         rap->EliminateRowCol((*ess_tdof_list)[i], *elim, Matrix::DiagonalPolicy::DIAG_ONE);
+      }
+
+      const int remove_zeros = 0;
+      rap->Finalize(remove_zeros);
+
+      operators[level - 1] = rap;
    }
 
-   for (int i = 0; i < essentialTrueDofs.Size(); ++i)
-   {
-      delete essentialTrueDofs[i];
-   }
+   CGSolver* pcg = new CGSolver();
+   GSSmoother* prec = new GSSmoother((SparseMatrix&)*operators[0]);
+   pcg->SetPrintLevel(-1);
+   pcg->SetMaxIter(50);
+   pcg->SetRelTol(sqrt(1e-4));
+   pcg->SetAbsTol(0.0);
+   pcg->SetOperator(*operators[0]);
+   pcg->SetPreconditioner(*prec);
+   smoothers[0] = pcg;
 }
 
 } // namespace mfem

fem/mgbilinearform.hpp

@@ -30,13 +30,21 @@ class MultigridBilinearForm : public MultigridOperator
    MultigridBilinearForm();
 
    /// Constructor for a multigrid bilinear form for a given SpaceHierarchy and
-   /// bilinear form. Uses Chebyshev accelerated smoothing. Only supports
-   /// partial assembly bilinear forms.
+   /// bilinear form. Uses Chebyshev accelerated smoothing.
    /// At the moment, only the DomainIntegrators of \p bf are copied.
    MultigridBilinearForm(SpaceHierarchy& spaceHierarchy, BilinearForm& bf,
                          Array<int>& ess_bdr);
 
+   /// Constructor for fully assembled systems
+   MultigridBilinearForm(SpaceHierarchy& spaceHierarchy, SparseMatrix& opr, Array<int>& ess_bdr);
+
    virtual ~MultigridBilinearForm();
+
+ private:
+   void SetupPA(SpaceHierarchy& spaceHierarchy, BilinearForm& bf,
+                Array<int>& ess_bdr);
+   void SetupFull(SpaceHierarchy& spaceHierarchy, SparseMatrix& opr,
+                  Array<int>& ess_bdr);
 };
 
 } // namespace mfem

fem/nonlinearform.cpp

@@ -14,6 +14,31 @@
 namespace mfem
 {
 
+void NonlinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
+{
+   if (ext)
+   {
+      MFEM_ABORT("the assembly level has already been set!");
+   }
+   assembly = assembly_level;
+   switch (assembly)
+   {
+      case AssemblyLevel::FULL:
+         // Use the original NonlinearForm implementation for now
+         break;
+      case AssemblyLevel::ELEMENT:
+         mfem_error("Element assembly not supported yet... stay tuned!");
+         break;
+      case AssemblyLevel::PARTIAL:
+         ext = new PANonlinearFormExtension(this);
+         break;
+      case AssemblyLevel::NONE:
+         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         break;
+      default:
+         mfem_error("Unknown assembly level");
+   }
+}
 void NonlinearForm::SetEssentialBC(const Array<int> &bdr_attr_is_ess,
                                    Vector *rhs)
 {
@@ -109,13 +134,22 @@ const Vector &NonlinearForm::Prolongate(const Vector &x) const
 
 void NonlinearForm::Mult(const Vector &x, Vector &y) const
 {
+   const Vector &px = Prolongate(x);
+   if (P) { aux2.SetSize(P->Height()); }
+
+   if (ext)
+   {
+      ext->Mult(px, aux2);
+      aux2.HostRead();
+      return;
+   }
+
    Array<int> vdofs;
    Vector el_x, el_y;
    const FiniteElement *fe;
    ElementTransformation *T;
    Mesh *mesh = fes->GetMesh();
-   const Vector &px = Prolongate(x);
-   Vector &py = P ? aux2.SetSize(P->Height()), aux2 : y;
+   Vector &py = P ? aux2 : y;
 
    py = 0.0;
 
@@ -232,6 +266,11 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
 
 Operator &NonlinearForm::GetGradient(const Vector &x) const
 {
+   if (ext)
+   {
+      MFEM_ABORT("Not yet implemented!");
+   }
+
    const int skip_zeros = 0;
    Array<int> vdofs;
    Vector el_x;
@@ -375,6 +414,8 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const
 
 void NonlinearForm::Update()
 {
+   if (ext) { MFEM_ABORT("Not yet implemented!"); }
+
    if (sequence == fes->GetSequence()) { return; }
 
    height = width = fes->GetTrueVSize();
@@ -387,6 +428,11 @@ void NonlinearForm::Update()
    cP = dynamic_cast<const SparseMatrix*>(P);
 }
 
+void NonlinearForm::Setup()
+{
+   if (ext) { return ext->Setup(); }
+}
+
 NonlinearForm::~NonlinearForm()
 {
    delete cGrad;
@@ -394,6 +440,7 @@ NonlinearForm::~NonlinearForm()
    for (int i = 0; i <  dnfi.Size(); i++) { delete  dnfi[i]; }
    for (int i = 0; i <  fnfi.Size(); i++) { delete  fnfi[i]; }
    for (int i = 0; i < bfnfi.Size(); i++) { delete bfnfi[i]; }
+   delete ext;
 }
 
 

fem/nonlinearform.hpp

@@ -14,6 +14,8 @@
 
 #include "../config/config.hpp"
 #include "nonlininteg.hpp"
+#include "nonlinearform_ext.hpp"
+#include "bilinearform.hpp"
 #include "gridfunc.hpp"
 
 namespace mfem
@@ -22,6 +24,13 @@ namespace mfem
 class NonlinearForm : public Operator
 {
 protected:
+   /// The form assembly level (full, partial, etc.)
+   AssemblyLevel assembly;
+
+   /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
+       Partial Assembly (PA), or Matrix Free assembly (MF). */
+   NonlinearFormExtension *ext;
+
    /// FE space on which the form lives.
    FiniteElementSpace *fes; // not owned
 
@@ -59,11 +68,16 @@ public:
    /** As an Operator, the NonlinearForm has input and output size equal to the
        number of true degrees of freedom, i.e. f->GetTrueVSize(). */
    NonlinearForm(FiniteElementSpace *f)
-      : Operator(f->GetTrueVSize()), fes(f), Grad(NULL), cGrad(NULL),
+      : Operator(f->GetTrueVSize()), assembly(AssemblyLevel::FULL),
+        ext(NULL), fes(f), Grad(NULL), cGrad(NULL),
         sequence(f->GetSequence()), P(f->GetProlongationMatrix()),
         cP(dynamic_cast<const SparseMatrix*>(P))
    { }
 
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
+   /** This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level);
+
    FiniteElementSpace *FESpace() { return fes; }
    const FiniteElementSpace *FESpace() const { return fes; }
 
@@ -71,6 +85,9 @@ public:
    void AddDomainIntegrator(NonlinearFormIntegrator *nlfi)
    { dnfi.Append(nlfi); }
 
+   /// Access all integrators added with AddDomainIntegrator().
+   Array<NonlinearFormIntegrator*> *GetDNFI() { return &dnfi; }
+
    /// Adds new Interior Face Integrator.
    void AddInteriorFaceIntegrator(NonlinearFormIntegrator *nlfi)
    { fnfi.Append(nlfi); }
@@ -144,6 +161,9 @@ public:
        set again. */
    virtual void Update();
 
+   /// Setup the NonlinearForm
+   virtual void Setup();
+
    /// Get the finite element space prolongation matrix
    virtual const Operator *GetProlongation() const { return P; }
    /// Get the finite element space restriction matrix

fem/nonlinearform_ext.cpp

@@ -0,0 +1,74 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+// Implementations of classes FABilinearFormExtension, EABilinearFormExtension,
+// PABilinearFormExtension and MFBilinearFormExtension.
+
+#include "nonlinearform.hpp"
+
+namespace mfem
+{
+
+NonlinearFormExtension::NonlinearFormExtension(NonlinearForm *form)
+   : Operator(form->FESpace()->GetTrueVSize()), n(form)
+{
+   // empty
+}
+
+PANonlinearFormExtension::PANonlinearFormExtension(NonlinearForm *form):
+   NonlinearFormExtension(form), fes(*form->FESpace())
+{
+   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   elem_restrict_lex = fes.GetElementRestriction(ordering);
+   if (elem_restrict_lex)
+   {
+      localX.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
+      localY.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
+      localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
+   }
+}
+
+void PANonlinearFormExtension::Setup()
+{
+   Array<NonlinearFormIntegrator*> &integrators = *n->GetDNFI();
+   const int Ni = integrators.Size();
+   for (int i = 0; i < Ni; ++i)
+   {
+      integrators[i]->Setup(*n->FESpace());
+   }
+}
+
+void PANonlinearFormExtension::Mult(const Vector &x, Vector &y) const
+{
+   Array<NonlinearFormIntegrator*> &integrators = *n->GetDNFI();
+   const int iSz = integrators.Size();
+   if (elem_restrict_lex)
+   {
+      elem_restrict_lex->Mult(x, localX);
+      localY = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->MultPA(localX, localY);
+      }
+      elem_restrict_lex->MultTranspose(localY, y);
+   }
+   else
+   {
+      y.UseDevice(true); // typically this is a large vector, so store on device
+      y = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->MultPA(x, y);
+      }
+   }
+}
+
+}

fem/nonlinearform_ext.hpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef NONLINEARFORM_EXT_HPP
+#define NONLINEARFORM_EXT_HPP
+
+#include "../config/config.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+class NonlinearForm;
+
+class NonlinearFormExtension : public Operator
+{
+protected:
+   NonlinearForm *n; ///< Not owned
+public:
+   NonlinearFormExtension(NonlinearForm *form);
+   virtual void Setup() = 0;
+};
+
+/// Data and methods for partially-assembled nonlinear forms
+class PANonlinearFormExtension : public NonlinearFormExtension
+{
+protected:
+   const FiniteElementSpace &fes; // Not owned
+   mutable Vector localX, localY;
+   const Operator *elem_restrict_lex; // Not owned
+public:
+   PANonlinearFormExtension(NonlinearForm*);
+   void Setup();
+   void Mult(const Vector &x, Vector &y) const;
+};
+}
+#endif // NONLINEARFORM_EXT_HPP

fem/nonlininteg.cpp

@@ -10,10 +10,35 @@
 // Software Foundation) version 2.1 dated February 1999.
 
 #include "fem.hpp"
+#include "../general/forall.hpp"
 
 namespace mfem
 {
 
+void NonlinearFormIntegrator::Setup(const FiniteElementSpace&)
+{
+   mfem_error ("NonlinearFormIntegrator::Setup(...)\n"
+               "   is not implemented for this class.");
+}
+
+void NonlinearFormIntegrator::Setup(const FiniteElementSpace&,
+                                    const FiniteElementSpace&)
+{
+   mfem_error ("NonlinearFormIntegrator::SetupAssembly(...)\n"
+               "   is not implemented for this class.");
+}
+
+void NonlinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
+{
+   mfem_error ("NonlinearFormIntegrator::AddMultPA(...)\n"
+               "   is not implemented for this class.");
+}
+
+void NonlinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
+{
+   mfem_error ("NonlinearFormIntegrator::AddMultTransposePA(...)\n"
+               "   is not implemented for this class.");
+}
 void NonlinearFormIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr,
    const Vector &elfun, Vector &elvect)
@@ -673,4 +698,606 @@ void IncompressibleNeoHookeanIntegrator::AssembleElementGrad(
 
 }
 
+const IntegrationRule&
+VectorConvectionNLFIntegrator::GetRule(const FiniteElement &fe,
+                                       ElementTransformation &T)
+{
+   const int order = 2 * fe.GetOrder() + T.OrderGrad(&fe);
+   return IntRules.Get(fe.GetGeomType(), order);
+}
+
+void VectorConvectionNLFIntegrator::AssembleElementVector(
+   const FiniteElement &el,
+   ElementTransformation &T,
+   const Vector &elfun,
+   Vector &elvect)
+{
+   const int nd = el.GetDof();
+   const int dim = el.GetDim();
+
+   shape.SetSize(nd);
+   dshape.SetSize(nd, dim);
+   elvect.SetSize(nd * dim);
+   gradEF.SetSize(dim);
+
+   EF.UseExternalData(elfun.GetData(), nd, dim);
+   ELV.UseExternalData(elvect.GetData(), nd, dim);
+
+   Vector vec1(dim), vec2(dim);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, T);
+   ELV = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      T.SetIntPoint(&ip);
+      el.CalcShape(ip, shape);
+      el.CalcPhysDShape(T, dshape);
+      double w = ip.weight * T.Weight();
+      if (Q) { w *= Q->Eval(T, ip); }
+      MultAtB(EF, dshape, gradEF);
+      EF.MultTranspose(shape, vec1);
+      gradEF.Mult(vec1, vec2);
+      vec2 *= w;
+      AddMultVWt(shape, vec2, ELV);
+   }
+}
+
+void VectorConvectionNLFIntegrator::Setup(const FiniteElementSpace &fes)
+{
+   MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
+               "PA Only supports Ordering::byNODES!");
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement &el = *fes.GetFE(0);
+   ElementTransformation &T = *mesh->GetElementTransformation(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, T);
+   dim = mesh->Dimension();
+   ne = fes.GetMesh()->GetNE();
+   nq = ir->GetNPoints();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   pa_data.SetSize(ne*nq*dim*dim, Device::GetMemoryType());
+   double COEFF = 1.0;
+   if (Q)
+   {
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      COEFF = cQ->constant;
+   }
+   const int NE = ne;
+   const int NQ = nq;
+   auto W = ir->GetWeights().Read();
+   if (dim==1) { MFEM_ABORT("dim==1 not supported!"); }
+   if (dim==2)
+   {
+      auto J = Reshape(geom->J.Read(), NQ, 2, 2,NE);
+      auto G = Reshape(pa_data.Write(), NQ, 2, 2, NE);
+      MFEM_FORALL(e, NE,
+      {
+         for (int q = 0; q < NQ; ++q)
+         {
+            const double J11 = J(q,0,0,e);
+            const double J12 = J(q,0,1,e);
+            const double J21 = J(q,1,0,e);
+            const double J22 = J(q,1,1,e);
+            // Store wq * Q * adj(J)
+            G(q,0,0,e) = W[q] * COEFF *  J22; // 1,1
+            G(q,0,1,e) = W[q] * COEFF * -J12; // 1,2
+            G(q,1,0,e) = W[q] * COEFF * -J21; // 2,1
+            G(q,1,1,e) = W[q] * COEFF *  J11; // 2,2
+         }
+      });
+   }
+   if (dim==3)
+   {
+      auto J = Reshape(geom->J.Read(), NQ, 3, 3,NE);
+      auto G = Reshape(pa_data.Write(), NQ, 3, 3, NE);
+      MFEM_FORALL(e, NE,
+      {
+         for (int q = 0; q < NQ; ++q)
+         {
+            const double J11 = J(q,0,0,e);
+            const double J21 = J(q,1,0,e);
+            const double J31 = J(q,2,0,e);
+            const double J12 = J(q,0,1,e);
+            const double J22 = J(q,1,1,e);
+            const double J32 = J(q,2,1,e);
+            const double J13 = J(q,0,2,e);
+            const double J23 = J(q,1,2,e);
+            const double J33 = J(q,2,2,e);
+            const double cw  = W[q] * COEFF;
+            // adj(J)
+            const double A11 = (J22 * J33) - (J23 * J32);
+            const double A12 = (J32 * J13) - (J12 * J33);
+            const double A13 = (J12 * J23) - (J22 * J13);
+            const double A21 = (J31 * J23) - (J21 * J33);
+            const double A22 = (J11 * J33) - (J13 * J31);
+            const double A23 = (J21 * J13) - (J11 * J23);
+            const double A31 = (J21 * J32) - (J31 * J22);
+            const double A32 = (J31 * J12) - (J11 * J32);
+            const double A33 = (J11 * J22) - (J12 * J21);
+            // Store wq * Q * adj(J)
+            G(q,0,0,e) = cw * A11; // 1,1
+            G(q,0,1,e) = cw * A12; // 1,2
+            G(q,0,2,e) = cw * A13; // 1,3
+            G(q,1,0,e) = cw * A21; // 2,1
+            G(q,1,1,e) = cw * A22; // 2,2
+            G(q,1,2,e) = cw * A23; // 2,3
+            G(q,2,0,e) = cw * A31; // 3,1
+            G(q,2,1,e) = cw * A32; // 3,2
+            G(q,2,2,e) = cw * A33; // 3,3
+         }
+      });
+   }
+}
+
+// PA Convection NL 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0> static
+void PAConvectionNLApply2D(const int NE,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const Array<double> &bt,
+                           const Vector &q_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D*Q1D, 2, 2, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, 2, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      double data[max_Q1D][max_Q1D][2];
+      double grad0[max_Q1D][max_Q1D][2];
+      double grad1[max_Q1D][max_Q1D][2];
+      double Z[max_Q1D][max_Q1D][2];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            data[qy][qx][0] = 0.0;
+            data[qy][qx][1] = 0.0;
+            grad0[qy][qx][0] = 0.0;
+            grad0[qy][qx][1] = 0.0;
+            grad1[qy][qx][0] = 0.0;
+            grad1[qy][qx][1] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         double dataX[max_Q1D][2];
+         double gradX0[max_Q1D][2];
+         double gradX1[max_Q1D][2];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            dataX[qx][0] = 0.0;
+            dataX[qx][1] = 0.0;
+            gradX0[qx][0] = 0.0;
+            gradX0[qx][1] = 0.0;
+            gradX1[qx][0] = 0.0;
+            gradX1[qx][1] = 0.0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double s0 = x(dx,dy,0,e);
+            const double s1 = x(dx,dy,1,e);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double Bx = B(qx,dx);
+               const double Gx = G(qx,dx);
+               dataX[qx][0] += s0 * Bx;
+               dataX[qx][1] += s1 * Bx;
+               gradX0[qx][0] += s0 * Gx;
+               gradX0[qx][1] += s0 * Bx;
+               gradX1[qx][0] += s1 * Gx;
+               gradX1[qx][1] += s1 * Bx;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double By = B(qy,dy);
+            const double Gy = G(qy,dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qy][qx][0] += dataX[qx][0] * By;
+               data[qy][qx][1] += dataX[qx][1] * By;
+               grad0[qy][qx][0] += gradX0[qx][0] * By;
+               grad0[qy][qx][1] += gradX0[qx][1] * Gy;
+               grad1[qy][qx][0] += gradX1[qx][0] * By;
+               grad1[qy][qx][1] += gradX1[qx][1] * Gy;
+            }
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const int q = qx + qy * Q1D;
+            const double u1 = data[qy][qx][0];
+            const double u2 = data[qy][qx][1];
+            const double grad00 = grad0[qy][qx][0];
+            const double grad01 = grad0[qy][qx][1];
+            const double grad10 = grad1[qy][qx][0];
+            const double grad11 = grad1[qy][qx][1];
+            const double Dxu1 = grad00*Q(q,0,0,e) + grad01*Q(q,1,0,e);
+            const double Dyu1 = grad00*Q(q,0,1,e) + grad01*Q(q,1,1,e);
+            const double Dxu2 = grad10*Q(q,0,0,e) + grad11*Q(q,1,0,e);
+            const double Dyu2 = grad10*Q(q,0,1,e) + grad11*Q(q,1,1,e);
+            Z[qy][qx][0] = u1 * Dxu1 + u2 * Dyu1;
+            Z[qy][qx][1] = u1 * Dxu2 + u2 * Dyu2;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         double Y[max_D1D][2];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y[dx][0] = 0.0;
+            Y[dx][1] = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double Btx = Bt(dx,qx);
+               Y[dx][0] += Btx * Z[qy][qx][0];
+               Y[dx][1] += Btx * Z[qy][qx][1];
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double Bty = Bt(dy,qy);
+               y(dx,dy,0,e) += Bty * Y[dx][0];
+               y(dx,dy,1,e) += Bty * Y[dx][1];
+            }
+         }
+      }
+   });
+}
+
+// PA Convection NL 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0> static
+void PAConvectionNLApply3D(const int NE,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const Array<double> &bt,
+                           const Vector &q_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D*Q1D*Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      constexpr int VDIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      double data[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      double grad0[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      double grad1[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      double grad2[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      double Z[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qz][qy][qx][0] = 0.0;
+               data[qz][qy][qx][1] = 0.0;
+               data[qz][qy][qx][2] = 0.0;
+
+               grad0[qz][qy][qx][0] = 0.0;
+               grad0[qz][qy][qx][1] = 0.0;
+               grad0[qz][qy][qx][2] = 0.0;
+
+               grad1[qz][qy][qx][0] = 0.0;
+               grad1[qz][qy][qx][1] = 0.0;
+               grad1[qz][qy][qx][2] = 0.0;
+
+               grad2[qz][qy][qx][0] = 0.0;
+               grad2[qz][qy][qx][1] = 0.0;
+               grad2[qz][qy][qx][2] = 0.0;
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         double dataXY[max_Q1D][max_Q1D][VDIM];
+         double gradXY0[max_Q1D][max_Q1D][VDIM];
+         double gradXY1[max_Q1D][max_Q1D][VDIM];
+         double gradXY2[max_Q1D][max_Q1D][VDIM];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataXY[qy][qx][0] = 0.0;
+               dataXY[qy][qx][1] = 0.0;
+               dataXY[qy][qx][2] = 0.0;
+
+               gradXY0[qy][qx][0] = 0.0;
+               gradXY0[qy][qx][1] = 0.0;
+               gradXY0[qy][qx][2] = 0.0;
+
+               gradXY1[qy][qx][0] = 0.0;
+               gradXY1[qy][qx][1] = 0.0;
+               gradXY1[qy][qx][2] = 0.0;
+
+               gradXY2[qy][qx][0] = 0.0;
+               gradXY2[qy][qx][1] = 0.0;
+               gradXY2[qy][qx][2] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            double dataX[max_Q1D][VDIM];
+            double gradX0[max_Q1D][VDIM];
+            double gradX1[max_Q1D][VDIM];
+            double gradX2[max_Q1D][VDIM];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataX[qx][0] = 0.0;
+               dataX[qx][1] = 0.0;
+               dataX[qx][2] = 0.0;
+
+               gradX0[qx][0] = 0.0;
+               gradX0[qx][1] = 0.0;
+               gradX0[qx][2] = 0.0;
+
+               gradX1[qx][0] = 0.0;
+               gradX1[qx][1] = 0.0;
+               gradX1[qx][2] = 0.0;
+
+               gradX2[qx][0] = 0.0;
+               gradX2[qx][1] = 0.0;
+               gradX2[qx][2] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double s0 = x(dx,dy,dz,0,e);
+               const double s1 = x(dx,dy,dz,1,e);
+               const double s2 = x(dx,dy,dz,2,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double Bx = B(qx,dx);
+                  const double Gx = G(qx,dx);
+
+                  dataX[qx][0] += s0 * Bx;
+                  dataX[qx][1] += s1 * Bx;
+                  dataX[qx][2] += s2 * Bx;
+
+                  gradX0[qx][0] += s0 * Gx;
+                  gradX0[qx][1] += s0 * Bx;
+                  gradX0[qx][2] += s0 * Bx;
+
+                  gradX1[qx][0] += s1 * Gx;
+                  gradX1[qx][1] += s1 * Bx;
+                  gradX1[qx][2] += s1 * Bx;
+
+                  gradX2[qx][0] += s2 * Gx;
+                  gradX2[qx][1] += s2 * Bx;
+                  gradX2[qx][2] += s2 * Bx;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double By = B(qy,dy);
+               const double Gy = G(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  dataXY[qy][qx][0] += dataX[qx][0] * By;
+                  dataXY[qy][qx][1] += dataX[qx][1] * By;
+                  dataXY[qy][qx][2] += dataX[qx][2] * By;
+
+                  gradXY0[qy][qx][0] += gradX0[qx][0] * By;
+                  gradXY0[qy][qx][1] += gradX0[qx][1] * Gy;
+                  gradXY0[qy][qx][2] += gradX0[qx][2] * By;
+
+                  gradXY1[qy][qx][0] += gradX1[qx][0] * By;
+                  gradXY1[qy][qx][1] += gradX1[qx][1] * Gy;
+                  gradXY1[qy][qx][2] += gradX1[qx][2] * By;
+
+                  gradXY2[qy][qx][0] += gradX2[qx][0] * By;
+                  gradXY2[qy][qx][1] += gradX2[qx][1] * Gy;
+                  gradXY2[qy][qx][2] += gradX2[qx][2] * By;
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const double Bz = B(qz,dz);
+            const double Gz = G(qz,dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  data[qz][qy][qx][0] += dataXY[qy][qx][0] * Bz;
+                  data[qz][qy][qx][1] += dataXY[qy][qx][1] * Bz;
+                  data[qz][qy][qx][2] += dataXY[qy][qx][2] * Bz;
+
+                  grad0[qz][qy][qx][0] += gradXY0[qy][qx][0] * Bz;
+                  grad0[qz][qy][qx][1] += gradXY0[qy][qx][1] * Bz;
+                  grad0[qz][qy][qx][2] += gradXY0[qy][qx][2] * Gz;
+
+                  grad1[qz][qy][qx][0] += gradXY1[qy][qx][0] * Bz;
+                  grad1[qz][qy][qx][1] += gradXY1[qy][qx][1] * Bz;
+                  grad1[qz][qy][qx][2] += gradXY1[qy][qx][2] * Gz;
+
+                  grad2[qz][qy][qx][0] += gradXY2[qy][qx][0] * Bz;
+                  grad2[qz][qy][qx][1] += gradXY2[qy][qx][1] * Bz;
+                  grad2[qz][qy][qx][2] += gradXY2[qy][qx][2] * Gz;
+               }
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + Q1D * (qy + qz * Q1D);
+
+               const double u1 = data[qz][qy][qx][0];
+               const double u2 = data[qz][qy][qx][1];
+               const double u3 = data[qz][qy][qx][2];
+
+               const double grad00 = grad0[qz][qy][qx][0];
+               const double grad01 = grad0[qz][qy][qx][1];
+               const double grad02 = grad0[qz][qy][qx][2];
+
+               const double grad10 = grad1[qz][qy][qx][0];
+               const double grad11 = grad1[qz][qy][qx][1];
+               const double grad12 = grad1[qz][qy][qx][2];
+
+               const double grad20 = grad2[qz][qy][qx][0];
+               const double grad21 = grad2[qz][qy][qx][1];
+               const double grad22 = grad2[qz][qy][qx][2];
+
+               const double Dxu1 = grad00*Q(q,0,0,e) + grad01*Q(q,1,0,e) + grad02*Q(q,2,0,e);
+               const double Dyu1 = grad00*Q(q,0,1,e) + grad01*Q(q,1,1,e) + grad02*Q(q,2,1,e);
+               const double Dzu1 = grad00*Q(q,0,2,e) + grad01*Q(q,1,2,e) + grad02*Q(q,2,2,e);
+
+               const double Dxu2 = grad10*Q(q,0,0,e) + grad11*Q(q,1,0,e) + grad12*Q(q,2,0,e);
+               const double Dyu2 = grad10*Q(q,0,1,e) + grad11*Q(q,1,1,e) + grad12*Q(q,2,1,e);
+               const double Dzu2 = grad10*Q(q,0,2,e) + grad11*Q(q,1,2,e) + grad12*Q(q,2,2,e);
+
+               const double Dxu3 = grad20*Q(q,0,0,e) + grad21*Q(q,1,0,e) + grad22*Q(q,2,0,e);
+               const double Dyu3 = grad20*Q(q,0,1,e) + grad21*Q(q,1,1,e) + grad22*Q(q,2,1,e);
+               const double Dzu3 = grad20*Q(q,0,2,e) + grad21*Q(q,1,2,e) + grad22*Q(q,2,2,e);
+
+               Z[qz][qy][qx][0] = u1 * Dxu1 + u2 * Dyu1 + u3 * Dzu1;
+               Z[qz][qy][qx][1] = u1 * Dxu2 + u2 * Dyu2 + u3 * Dzu2;
+               Z[qz][qy][qx][2] = u1 * Dxu3 + u2 * Dyu3 + u3 * Dzu3;
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double opXY[max_D1D][max_D1D][VDIM];
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opXY[dy][dx][0] = 0.0;
+               opXY[dy][dx][1] = 0.0;
+               opXY[dy][dx][2] = 0.0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double opX[max_D1D][VDIM];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opX[dx][0] = 0.0;
+               opX[dx][1] = 0.0;
+               opX[dx][2] = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double Btx = Bt(dx,qx);
+                  opX[dx][0] += Btx * Z[qz][qy][qx][0];
+                  opX[dx][1] += Btx * Z[qz][qy][qx][1];
+                  opX[dx][2] += Btx * Z[qz][qy][qx][2];
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const double Bty = Bt(dy,qy);
+                  opXY[dy][dx][0] += Bty * opX[dx][0];
+                  opXY[dy][dx][1] += Bty * opX[dx][1];
+                  opXY[dy][dx][2] += Bty * opX[dx][2];
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const double Btz = Bt(dz,qz);
+                  y(dx,dy,dz,0,e) += Btz * opXY[dy][dx][0];
+                  y(dx,dy,dz,1,e) += Btz * opXY[dy][dx][1];
+                  y(dx,dy,dz,2,e) += Btz * opXY[dy][dx][2];
+               }
+            }
+         }
+      }
+   });
+}
+
+void VectorConvectionNLFIntegrator::MultPA(const Vector &x, Vector &y) const
+{
+   const int NE = ne;
+   const int D1D = maps->ndof;
+   const int Q1D = maps->nqpt;
+   const Vector &Q = pa_data;
+   const Array<double> &B = maps->B;
+   const Array<double> &G = maps->G;
+   const Array<double> &Bt = maps->Bt;
+   const int DQ = (D1D << 4) | Q1D;
+   if (dim == 2)
+   {
+      switch (DQ)
+      {
+         case 0x22: return PAConvectionNLApply2D<2,2>(NE,B,G,Bt,Q,x,y);
+         case 0x34: return PAConvectionNLApply2D<3,4>(NE,B,G,Bt,Q,x,y);
+         case 0x45: return PAConvectionNLApply2D<4,5>(NE,B,G,Bt,Q,x,y);
+         case 0x57: return PAConvectionNLApply2D<5,7>(NE,B,G,Bt,Q,x,y);
+         case 0x68: return PAConvectionNLApply2D<6,8>(NE,B,G,Bt,Q,x,y);
+         case 0x7A: return PAConvectionNLApply2D<7,10>(NE,B,G,Bt,Q,x,y);
+         case 0x8B: return PAConvectionNLApply2D<8,11>(NE,B,G,Bt,Q,x,y);
+         case 0x9D: return PAConvectionNLApply2D<9,13>(NE,B,G,Bt,Q,x,y);
+         default: return PAConvectionNLApply2D(NE,B,G,Bt,Q,x,y,D1D,Q1D);
+      }
+   }
+   if (dim == 3)
+   {
+      switch (DQ)
+      {
+         case 0x23: return PAConvectionNLApply3D<2,3>(NE,B,G,Bt,Q,x,y);
+         case 0x34: return PAConvectionNLApply3D<3,4>(NE,B,G,Bt,Q,x,y);
+         case 0x35: return PAConvectionNLApply3D<3,5>(NE,B,G,Bt,Q,x,y);
+         case 0x48: return PAConvectionNLApply3D<4,8>(NE,B,G,Bt,Q,x,y);
+         case 0x5A: return PAConvectionNLApply3D<5,10>(NE,B,G,Bt,Q,x,y);
+         case 0x6D: return PAConvectionNLApply3D<6,13>(NE,B,G,Bt,Q,x,y);
+         case 0x7F: return PAConvectionNLApply3D<7,15>(NE,B,G,Bt,Q,x,y);
+         case 0x92: return PAConvectionNLApply3D<8,18>(NE,B,G,Bt,Q,x,y);
+         case 0x94: return PAConvectionNLApply3D<9,20>(NE,B,G,Bt,Q,x,y);
+         case 0x8C: return PAConvectionNLApply3D<8,12>(NE,B,G,Bt,Q,x,y);
+         default: printf ("%x, %x(%d): %X", D1D, Q1D, Q1D, DQ);
+      }
+   }
+   MFEM_ABORT("Not yet implemented!");
+}
+
 }

fem/nonlininteg.hpp

@@ -15,6 +15,7 @@
 #include "../config/config.hpp"
 #include "fe.hpp"
 #include "coefficient.hpp"
+#include "fespace.hpp"
 
 namespace mfem
 {
@@ -68,6 +69,50 @@ public:
                                    ElementTransformation &Tr,
                                    const Vector &elfun);
 
+   // TODO: add support for other assembly levels (in addition to PA) and their
+   // actions.
+
+   // TODO: for mixed meshes the quadrature rules to be used by methods like
+   // Setup() can be given as a QuadratureSpace, e.g. using a new method:
+   // SetQuadratureSpace().
+
+   // TODO: the methods for the various assembly levels make sense even in the
+   // base class NonlinearFormIntegrator, except that not all assembly levels
+   // make sense for the action of the nonlinear operator (but they all make
+   // sense for its Jacobian).
+
+   /// Method defining partial assembly.
+   /** The result of the partial assembly is stored internally so that it can be
+       used later in the methods AddMultPA() and AddMultTransposePA(). */
+   virtual void Setup(const FiniteElementSpace &fes);
+
+   /** The result of the partial assembly is stored internally so that it can be
+       used later in the methods AddMultPA() and AddMultTransposePA().
+       Used with BilinearFormIntegrators that have different spaces. */
+   virtual void Setup(const FiniteElementSpace &trial_fes,
+                      const FiniteElementSpace &test_fes);
+
+   /// Method for partially assembled action.
+   /** Perform the action of integrator on the input @a x and add the result to
+       the output @a y. Both @a x and @a y are E-vectors, i.e. they represent
+       the element-wise discontinuous version of the FE space.
+
+       This method can be called only after the method Setup() has been
+       called. */
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+
+   /// Method for partially assembled transposed action.
+   /** Perform the transpose action of integrator on the input @a x and add the
+       result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
+       represent the element-wise discontinuous version of the FE space.
+
+       This method can be called only after the method Setup() has been
+       called. */
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+   /// Method for partially assembled action. */
+   virtual void MultPA(const Vector &x, Vector &y) const {}
+
    virtual ~NonlinearFormIntegrator() { }
 };
 
@@ -285,6 +330,31 @@ public:
                                     const Array2D<DenseMatrix *> &elmats);
 };
 
+class VectorConvectionNLFIntegrator : public NonlinearFormIntegrator
+{
+private:
+   Coefficient *Q{};
+   DenseMatrix dshape, EF, gradEF, ELV;
+   Vector shape;
+   // PA extension
+   Vector pa_data;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq;
+public:
+   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { }
+   VectorConvectionNLFIntegrator() = default;
+   static const IntegrationRule &GetRule(const FiniteElement &fe,
+                                         ElementTransformation &T);
+   virtual void AssembleElementVector(const FiniteElement &el,
+                                      ElementTransformation &trans,
+                                      const Vector &elfun,
+                                      Vector &elvect);
+   using NonlinearFormIntegrator::Setup;
+   virtual void Setup(const FiniteElementSpace &fes);
+   virtual void MultPA(const Vector &x, Vector &y) const;
+};
+
 }
 
 #endif

fem/pbilinearform.cpp

@@ -496,6 +496,69 @@ void ParMixedBilinearForm::TrueAddMult(const Vector &x, Vector &y,
    test_pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Y, 1.0, y);
 }
 
+void ParMixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
+                                                       &trial_tdof_list,
+                                                       const Array<int> &test_tdof_list,
+                                                       OperatorHandle &A)
+{
+   if (ext)
+   {
+      ext->FormRectangularSystemOperator(trial_tdof_list, test_tdof_list, A);
+      return;
+   }
+
+   if (mat)
+   {
+      Finalize();
+      ParallelAssemble(p_mat);
+      delete mat;
+      mat = NULL;
+      delete mat_e;
+      mat_e = NULL;
+      HypreParMatrix *temp = p_mat.As<HypreParMatrix>()->EliminateCols(
+                                trial_tdof_list);
+      p_mat.As<HypreParMatrix>()->EliminateRows(test_tdof_list);
+      p_mat_e.Reset(temp, true);
+   }
+
+   A = p_mat;
+}
+
+void ParMixedBilinearForm::FormRectangularLinearSystem(const Array<int>
+                                                       &trial_tdof_list,
+                                                       const Array<int> &test_tdof_list, Vector &x,
+                                                       Vector &b, OperatorHandle &A, Vector &X,
+                                                       Vector &B)
+{
+   if (ext)
+   {
+      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, A, X,
+                                       B);
+      return;
+   }
+
+   FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list, A);
+
+   const Operator *test_P = test_pfes->GetProlongationMatrix();
+   const SparseMatrix *trial_R = trial_pfes->GetRestrictionMatrix();
+
+   X.SetSize(trial_pfes->TrueVSize());
+   B.SetSize(test_pfes->TrueVSize());
+   test_P->MultTranspose(b, B);
+   trial_R->Mult(x, X);
+
+   p_mat_e.As<HypreParMatrix>()->Mult(-1.0, X, 1.0, B);
+   B.SetSubVector(test_tdof_list, 0.0);
+}
+
+void ParMixedBilinearForm::Update()
+{
+   MixedBilinearForm::Update();
+
+   p_mat.Clear();
+   p_mat_e.Clear();
+}
+
 
 HypreParMatrix* ParDiscreteLinearOperator::ParallelAssemble() const
 {

fem/pbilinearform.hpp

@@ -198,6 +198,9 @@ protected:
    /// Auxiliary objects used in TrueAddMult().
    mutable ParGridFunction X, Y;
 
+   /// Matrix and eliminated matrix
+   OperatorHandle p_mat, p_mat_e;
+
 private:
    /// Copy construction is not supported; body is undefined.
    ParMixedBilinearForm(const ParMixedBilinearForm &);
@@ -212,7 +215,8 @@ public:
        constructed object. */
    ParMixedBilinearForm(ParFiniteElementSpace *trial_fes,
                         ParFiniteElementSpace *test_fes)
-      : MixedBilinearForm(trial_fes, test_fes)
+      : MixedBilinearForm(trial_fes, test_fes),
+        p_mat(Operator::Hypre_ParCSR), p_mat_e(Operator::Hypre_ParCSR)
    {
       trial_pfes = trial_fes;
       test_pfes  = test_fes;
@@ -230,7 +234,8 @@ public:
    ParMixedBilinearForm(ParFiniteElementSpace *trial_fes,
                         ParFiniteElementSpace *test_fes,
                         ParMixedBilinearForm * mbf)
-      : MixedBilinearForm(trial_fes, test_fes, mbf)
+      : MixedBilinearForm(trial_fes, test_fes, mbf),
+        p_mat(Operator::Hypre_ParCSR), p_mat_e(Operator::Hypre_ParCSR)
    {
       trial_pfes = trial_fes;
       test_pfes  = test_fes;
@@ -244,6 +249,17 @@ public:
        @a A. */
    void ParallelAssemble(OperatorHandle &A);
 
+   virtual void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
+                                            const Array<int> &test_tdof_list,
+                                            OperatorHandle &A);
+
+   virtual void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                            const Array<int> &test_tdof_list, Vector &x,
+                                            Vector &b, OperatorHandle &A, Vector &X,
+                                            Vector &B);
+
+   void Update();
+
    /// Compute y += a (P^t A P) x, where x and y are vectors on the true dofs
    void TrueAddMult(const Vector &x, Vector &y, const double a = 1.0) const;
 

fem/pfespace.cpp

@@ -2926,7 +2926,6 @@ void ConformingProlongationOperator::MultTranspose(
 {
    MFEM_ASSERT(x.Size() == Height(), "");
    MFEM_ASSERT(y.Size() == Width(), "");
-
    const double *xdata = x.HostRead();
    double *ydata = y.HostWrite();
    const int m = external_ldofs.Size();

fem/pnonlinearform.cpp

@@ -45,9 +45,9 @@ double ParNonlinearForm::GetParGridFunctionEnergy(const Vector &x) const
 
 void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
+   //NonlinearForm::Mult(Xtmp, y); // x --(P)--> aux1 --(A_local)--> aux2
    NonlinearForm::Mult(x, y); // x --(P)--> aux1 --(A_local)--> aux2
    Y.SetData(aux2.GetData()); // aux2 contains A_local.P.x
-
    if (fnfi.Size())
    {
       // Terms over shared interior faces in parallel.
@@ -82,7 +82,6 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
          }
       }
    }
-
    P->MultTranspose(Y, y);
 
    for (int i = 0; i < ess_tdof_list.Size(); i++)

general/forall.hpp

@@ -31,6 +31,7 @@
 namespace mfem
 {
 
+// TODO:WP:this is temporary
 // Maximum size of dofs and quads in 1D.
 const int MAX_D1D = 16;
 const int MAX_Q1D = 16;

general/mem_manager.hpp

@@ -203,6 +203,9 @@ public:
    void ClearOwnerFlags() const
    { flags = flags & ~(OWNS_HOST | OWNS_DEVICE | OWNS_INTERNAL); }
 
+   void SetInternalOwner() const
+   { flags = flags | OWNS_INTERNAL; };
+
    /// Read the internal device flag.
    bool UseDevice() const { return flags & USE_DEVICE; }
 

linalg/blockoperator.cpp

@@ -69,18 +69,26 @@ void BlockOperator::Mult (const Vector & x, Vector & y) const
 {
    MFEM_ASSERT(x.Size() == width, "incorrect input Vector size");
    MFEM_ASSERT(y.Size() == height, "incorrect output Vector size");
-
-   yblock.Update(y.GetData(),row_offsets);
-   xblock.Update(x.GetData(),col_offsets);
+   
+   const bool use_dev = x.UseDevice() || y.UseDevice();
+   yblock.Update(y,row_offsets);
+   yblock.UseDevice(use_dev);
+   yblock.ReadWrite(use_dev);
+   xblock.Update(x,col_offsets);
+   xblock.UseDevice(use_dev);
+   xblock.Read(use_dev);
+   tmp.UseDevice(use_dev);
 
    y = 0.0;
    for (int iRow=0; iRow < nRowBlocks; ++iRow)
    {
+      yblock.GetBlock(iRow).SyncAliasMemory(yblock);
       tmp.SetSize(row_offsets[iRow+1] - row_offsets[iRow]);
       for (int jCol=0; jCol < nColBlocks; ++jCol)
       {
          if (op(iRow,jCol))
          {
+            xblock.GetBlock(jCol).SyncAliasMemory(xblock); // TODO: This shouldn't have to be done...
             op(iRow,jCol)->Mult(xblock.GetBlock(jCol), tmp);
             yblock.GetBlock(iRow).Add(coef(iRow,jCol), tmp);
          }
@@ -96,16 +104,24 @@ void BlockOperator::MultTranspose (const Vector & x, Vector & y) const
 
    y = 0.0;
 
-   xblock.Update(x.GetData(),row_offsets);
-   yblock.Update(y.GetData(),col_offsets);
+   const bool use_dev = x.UseDevice() || y.UseDevice();
+   xblock.Update(x,row_offsets);
+   xblock.UseDevice(use_dev);
+   xblock.Read(use_dev);
+   yblock.Update(y,col_offsets);
+   yblock.UseDevice(use_dev);
+   yblock.ReadWrite(use_dev);
+   tmp.UseDevice(use_dev);
 
    for (int iRow=0; iRow < nColBlocks; ++iRow)
    {
+      yblock.GetBlock(iRow).SyncAliasMemory(yblock);
       tmp.SetSize(col_offsets[iRow+1] - col_offsets[iRow]);
       for (int jCol=0; jCol < nRowBlocks; ++jCol)
       {
          if (op(jCol,iRow))
          {
+            xblock.GetBlock(jCol).SyncAliasMemory(xblock);
             op(jCol,iRow)->MultTranspose(xblock.GetBlock(jCol), tmp);
             yblock.GetBlock(iRow).Add(coef(jCol,iRow), tmp);
          }
@@ -157,11 +173,18 @@ void BlockDiagonalPreconditioner::Mult (const Vector & x, Vector & y) const
    MFEM_ASSERT(x.Size() == width, "incorrect input Vector size");
    MFEM_ASSERT(y.Size() == height, "incorrect output Vector size");
 
-   yblock.Update(y.GetData(), offsets);
-   xblock.Update(x.GetData(), offsets);
+   const bool use_dev = x.UseDevice() || y.UseDevice();
+   yblock.Update(y, offsets);
+   yblock.UseDevice(use_dev);
+   yblock.ReadWrite(use_dev);
+   xblock.Update(x, offsets);
+   xblock.UseDevice(use_dev);
+   xblock.Read(use_dev);
 
    for (int i=0; i<nBlocks; ++i)
    {
+      yblock.GetBlock(i).SyncAliasMemory(yblock);
+      xblock.GetBlock(i).SyncAliasMemory(xblock); // TODO: This shouldn't have to be done...
       if (op[i])
       {
          op[i]->Mult(xblock.GetBlock(i), yblock.GetBlock(i));
@@ -180,11 +203,18 @@ void BlockDiagonalPreconditioner::MultTranspose (const Vector & x,
    MFEM_ASSERT(x.Size() == height, "incorrect input Vector size");
    MFEM_ASSERT(y.Size() == width, "incorrect output Vector size");
 
-   yblock.Update(y.GetData(), offsets);
-   xblock.Update(x.GetData(), offsets);
+   const bool use_dev = x.UseDevice() || y.UseDevice();
+   yblock.Update(y, offsets);
+   yblock.UseDevice(use_dev);
+   yblock.ReadWrite(use_dev);
+   xblock.Update(x, offsets);
+   xblock.UseDevice(use_dev);
+   xblock.Read(use_dev);
 
    for (int i=0; i<nBlocks; ++i)
    {
+      yblock.GetBlock(i).SyncAliasMemory(yblock);
+      xblock.GetBlock(i).SyncAliasMemory(xblock); // TODO: This shouldn't have to be done
       if (op[i])
       {
          (op[i])->MultTranspose(xblock.GetBlock(i), yblock.GetBlock(i));
@@ -247,12 +277,20 @@ void BlockLowerTriangularPreconditioner::Mult (const Vector & x,
    MFEM_ASSERT(x.Size() == width, "incorrect input Vector size");
    MFEM_ASSERT(y.Size() == height, "incorrect output Vector size");
 
-   yblock.Update(y.GetData(),offsets);
-   xblock.Update(x.GetData(),offsets);
+   const bool use_dev = x.UseDevice() || y.UseDevice();
+   yblock.Update(y,offsets);
+   yblock.UseDevice(use_dev);
+   yblock.ReadWrite(use_dev);
+   xblock.Update(x,offsets);
+   xblock.UseDevice(use_dev);
+   xblock.Read(use_dev);
+   tmp.UseDevice(use_dev);
+   tmp2.UseDevice(use_dev);
 
    y = 0.0;
    for (int iRow=0; iRow < nBlocks; ++iRow)
    {
+      xblock.GetBlock(iRow).SyncAliasMemory(xblock); // TODO: This shouldn't have to be done...
       tmp.SetSize(offsets[iRow+1] - offsets[iRow]);
       tmp2.SetSize(offsets[iRow+1] - offsets[iRow]);
       tmp2 = 0.0;
@@ -261,10 +299,12 @@ void BlockLowerTriangularPreconditioner::Mult (const Vector & x,
       {
          if (op(iRow,jCol))
          {
+            yblock.GetBlock(jCol).SyncAliasMemory(yblock);
             op(iRow,jCol)->Mult(yblock.GetBlock(jCol), tmp);
             tmp2 -= tmp;
          }
       }
+      yblock.GetBlock(iRow).SyncAliasMemory(yblock);
       if (op(iRow,iRow))
       {
          op(iRow,iRow)->Mult(tmp2, yblock.GetBlock(iRow));
@@ -283,12 +323,20 @@ void BlockLowerTriangularPreconditioner::MultTranspose (const Vector & x,
    MFEM_ASSERT(x.Size() == height, "incorrect input Vector size");
    MFEM_ASSERT(y.Size() == width, "incorrect output Vector size");
 
-   yblock.Update(y.GetData(),offsets);
-   xblock.Update(x.GetData(),offsets);
+   const bool use_dev = x.UseDevice() || y.UseDevice();
+   yblock.Update(y,offsets);
+   yblock.UseDevice(use_dev);
+   yblock.ReadWrite(use_dev);
+   xblock.Update(x,offsets);
+   xblock.UseDevice(use_dev);
+   xblock.Read(use_dev);
+   tmp.UseDevice(use_dev);
+   tmp2.UseDevice(use_dev);
 
    y = 0.0;
    for (int iRow=nBlocks-1; iRow >=0; --iRow)
    {
+      xblock.GetBlock(iRow).SyncAliasMemory(xblock); // TODO: This shouldn't have to be done...
       tmp.SetSize(offsets[iRow+1] - offsets[iRow]);
       tmp2.SetSize(offsets[iRow+1] - offsets[iRow]);
       tmp2 = 0.0;
@@ -297,10 +345,12 @@ void BlockLowerTriangularPreconditioner::MultTranspose (const Vector & x,
       {
          if (op(jCol,iRow))
          {
+            yblock.GetBlock(jCol).SyncAliasMemory(yblock);
             op(jCol,iRow)->MultTranspose(yblock.GetBlock(jCol), tmp);
             tmp2 -= tmp;
          }
       }
+      yblock.GetBlock(iRow).SyncAliasMemory(yblock);
       if (op(iRow,iRow))
       {
          op(iRow,iRow)->MultTranspose(tmp2, yblock.GetBlock(iRow));

linalg/blockvector.cpp

@@ -74,6 +74,19 @@ BlockVector::BlockVector(double *data, const Array<int> & bOffsets):
    SetBlocks();
 }
 
+void BlockVector::Update(const Vector& data, const Array<int> &bOffsets)
+{
+   NewMemoryAndSize(data.GetMemory(), bOffsets.Last(), false);
+   blockOffsets = bOffsets.GetData();
+   if (numBlocks != bOffsets.Size()-1)
+   {
+      delete [] blocks;
+      numBlocks = bOffsets.Size()-1;
+      blocks = new Vector[numBlocks];
+   }
+   SetBlocks();
+}
+
 void BlockVector::Update(double *data, const Array<int> & bOffsets)
 {
    NewDataAndSize(data, bOffsets.Last());
@@ -166,4 +179,14 @@ void BlockVector::GetBlockView(int i, Vector & blockView)
       BlockSize(i), true);
 }
 
+
+void BlockVector::UseDevice(bool use_dev)
+{
+   Vector::UseDevice(use_dev);
+   for (int i = 0; i < numBlocks; ++i)
+   {
+      blocks[i].UseDevice(use_dev);
+   }
+}
+
 }

linalg/blockvector.hpp

@@ -91,6 +91,14 @@ public:
    int BlockSize(int i) { return blockOffsets[i+1] - blockOffsets[i]; }
 
    //! Update method
+
+   /**
+      data is another vector that contains all the values of the monolithic vector. 
+      bOffsets is an array of integers (length nBlocks + 1) that tells the offsets
+      of each block start. Does not take ownership of data.
+    */
+   void Update(const Vector& data, const Array<int> &bOffsets);
+
    /**
     * data is an array of double of length at least blockOffsets[numBlocks] that
     * contain all the values of the monolithic vector.  bOffsets is an array of
@@ -112,6 +120,12 @@ public:
        - currently, the block-vector does not own its data, or
        - currently, the block-vector does not use MemoryType @a mt. */
    void Update(const Array<int> &bOffsets, MemoryType mt);
+
+   /**
+      Overload the internal UseDevice and propogate to blocks too
+   */
+   virtual void UseDevice(bool use_dev);
+   using Vector::UseDevice; // Ensure we call Vector::UseDevice() without inputs
 };
 
 }

linalg/hypre.cpp

@@ -1415,6 +1415,26 @@ HypreParMatrix* HypreParMatrix::EliminateRowsCols(const Array<int> &rows_cols)
    return new HypreParMatrix(Ae);
 }
 
+HypreParMatrix* HypreParMatrix::EliminateCols(const Array<int> &cols)
+{
+   Array<HYPRE_Int> rc_sorted;
+   get_sorted_rows_cols(cols, rc_sorted);
+
+   hypre_ParCSRMatrix* Ae;
+   internal::hypre_ParCSRMatrixEliminateAAe(
+      A, &Ae, rc_sorted.Size(), rc_sorted.GetData(), 1);
+
+   return new HypreParMatrix(Ae);
+}
+
+void HypreParMatrix::EliminateRows(const Array<int> &rows)
+{
+   if (rows.Size() > 0)
+   {
+      internal::hypre_ParCSRMatrixEliminateRows(A, rows.Size(), rows.GetData());
+   }
+}
+
 void HypreParMatrix::Print(const char *fname, HYPRE_Int offi, HYPRE_Int offj)
 {
    hypre_ParCSRMatrixPrintIJ(A,offi,offj,fname);

linalg/hypre.hpp

@@ -525,6 +525,13 @@ public:
        Ae sum to the original matrix. */
    HypreParMatrix* EliminateRowsCols(const Array<int> &rows_cols);
 
+   /** Eliminate columns from the matrix and store the eliminated
+       elements in a new matrix Ae (returned) so that the modified
+       matrix and Ae sum to the original matrix. */
+   HypreParMatrix* EliminateCols(const Array<int> &cols);
+
+   void EliminateRows(const Array<int> &rows);
+
    /// Prints the locally owned rows in parallel
    void Print(const char *fname, HYPRE_Int offi = 0, HYPRE_Int offj = 0);
    /// Reads the matrix from a file

linalg/hypre_parcsr.cpp

@@ -459,6 +459,31 @@ void hypre_CSRMatrixEliminateRowsCols(hypre_CSRMatrix *A,
    }
 }
 
+/*
+  Eliminate rows of A, setting all entries in the eliminated rows to zero.
+*/
+void hypre_CSRMatrixEliminateRows(hypre_CSRMatrix *A,
+                                  HYPRE_Int nrows, const HYPRE_Int *rows)
+{
+   HYPRE_Int  irow, i, j;
+   HYPRE_Int  A_beg, A_end;
+
+   HYPRE_Int  *A_i     = hypre_CSRMatrixI(A);
+   HYPRE_Real *A_data  = hypre_CSRMatrixData(A);
+
+   for (i = 0; i < nrows; i++)
+   {
+      irow = rows[i];
+      A_beg = A_i[irow];
+      A_end = A_i[irow+1];
+      /* eliminate row */
+      for (j = A_beg; j < A_end; j++)
+      {
+         A_data[j] = 0.0;
+      }
+   }
+}
+
 
 /*
   Function:  hypre_ParCSRMatrixEliminateAAe
@@ -478,13 +503,14 @@ void hypre_CSRMatrixEliminateRowsCols(hypre_CSRMatrix *A,
 void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
                                     hypre_ParCSRMatrix **Ae,
                                     HYPRE_Int num_rowscols_to_elim,
-                                    HYPRE_Int *rowscols_to_elim)
+                                    HYPRE_Int *rowscols_to_elim,
+                                    int ignore_rows)
 {
    HYPRE_Int i, j, k;
 
    hypre_CSRMatrix *A_diag = hypre_ParCSRMatrixDiag(A);
    hypre_CSRMatrix *A_offd = hypre_ParCSRMatrixOffd(A);
-   HYPRE_Int A_diag_nrows  = hypre_CSRMatrixNumRows(A_diag);
+   HYPRE_Int A_diag_ncols  = hypre_CSRMatrixNumCols(A_diag);
    HYPRE_Int A_offd_ncols  = hypre_CSRMatrixNumCols(A_offd);
 
    *Ae = hypre_ParCSRMatrixCreate(hypre_ParCSRMatrixComm(A),
@@ -517,8 +543,8 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
       HYPRE_Int num_sends, *int_buf_data;
       HYPRE_Int index, start;
 
-      HYPRE_Int *eliminate_row = mfem_hypre_CTAlloc(HYPRE_Int, A_diag_nrows);
-      HYPRE_Int *eliminate_col = mfem_hypre_CTAlloc(HYPRE_Int, A_offd_ncols);
+      HYPRE_Int *eliminate_diag_col = mfem_hypre_CTAlloc(HYPRE_Int, A_diag_ncols);
+      HYPRE_Int *eliminate_offd_col = mfem_hypre_CTAlloc(HYPRE_Int, A_offd_ncols);
 
       /* make sure A has a communication package */
       comm_pkg = hypre_ParCSRMatrixCommPkg(A);
@@ -529,13 +555,13 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
       }
 
       /* which of the local rows are to be eliminated */
-      for (i = 0; i < A_diag_nrows; i++)
+      for (i = 0; i < A_diag_ncols; i++)
       {
-         eliminate_row[i] = 0;
+         eliminate_diag_col[i] = 0;
       }
       for (i = 0; i < num_rowscols_to_elim; i++)
       {
-         eliminate_row[rowscols_to_elim[i]] = 1;
+         eliminate_diag_col[rowscols_to_elim[i]] = 1;
       }
 
       /* use a Matvec communication pattern to find (in eliminate_col)
@@ -551,22 +577,38 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
          for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
          {
             k = hypre_ParCSRCommPkgSendMapElmt(comm_pkg, j);
-            int_buf_data[index++] = eliminate_row[k];
+            int_buf_data[index++] = eliminate_diag_col[k];
          }
       }
       comm_handle = hypre_ParCSRCommHandleCreate(11, comm_pkg,
-                                                 int_buf_data, eliminate_col);
+                                                 int_buf_data, eliminate_offd_col);
 
       /* eliminate diagonal part, overlapping it with communication */
-      hypre_CSRMatrixElimCreate(A_diag, Ae_diag,
-                                num_rowscols_to_elim, rowscols_to_elim,
-                                num_rowscols_to_elim, rowscols_to_elim,
-                                NULL);
+      if (ignore_rows)
+      {
+         hypre_CSRMatrixElimCreate(A_diag, Ae_diag,
+                                   0, nullptr,
+                                   num_rowscols_to_elim, rowscols_to_elim,
+                                   NULL);
+
+         hypre_CSRMatrixEliminateRowsCols(A_diag, Ae_diag,
+                                          0, nullptr,
+                                          num_rowscols_to_elim, rowscols_to_elim,
+                                          1, NULL);
+      }
+      else
+      {
+         hypre_CSRMatrixElimCreate(A_diag, Ae_diag,
+                                   num_rowscols_to_elim, rowscols_to_elim,
+                                   num_rowscols_to_elim, rowscols_to_elim,
+                                   NULL);
+
+         hypre_CSRMatrixEliminateRowsCols(A_diag, Ae_diag,
+                                          num_rowscols_to_elim, rowscols_to_elim,
+                                          num_rowscols_to_elim, rowscols_to_elim,
+                                          1, NULL);
+      }
 
-      hypre_CSRMatrixEliminateRowsCols(A_diag, Ae_diag,
-                                       num_rowscols_to_elim, rowscols_to_elim,
-                                       num_rowscols_to_elim, rowscols_to_elim,
-                                       1, NULL);
       hypre_CSRMatrixReorder(Ae_diag);
 
       /* finish the communication */
@@ -576,7 +618,7 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
       num_offd_cols_to_elim = 0;
       for (i = 0; i < A_offd_ncols; i++)
       {
-         if (eliminate_col[i]) { num_offd_cols_to_elim++; }
+         if (eliminate_offd_col[i]) { num_offd_cols_to_elim++; }
       }
 
       offd_cols_to_elim = mfem_hypre_CTAlloc(HYPRE_Int, num_offd_cols_to_elim);
@@ -585,35 +627,55 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
       num_offd_cols_to_elim = 0;
       for (i = 0; i < A_offd_ncols; i++)
       {
-         if (eliminate_col[i])
+         if (eliminate_offd_col[i])
          {
             offd_cols_to_elim[num_offd_cols_to_elim++] = i;
          }
       }
 
       mfem_hypre_TFree(int_buf_data);
-      mfem_hypre_TFree(eliminate_col);
-      mfem_hypre_TFree(eliminate_row);
+      mfem_hypre_TFree(eliminate_offd_col);
+      mfem_hypre_TFree(eliminate_diag_col);
    }
 
    /* eliminate the off-diagonal part */
    col_mark = mfem_hypre_CTAlloc(HYPRE_Int, A_offd_ncols);
    col_remap = mfem_hypre_CTAlloc(HYPRE_Int, A_offd_ncols);
 
-   hypre_CSRMatrixElimCreate(A_offd, Ae_offd,
-                             num_rowscols_to_elim, rowscols_to_elim,
-                             num_offd_cols_to_elim, offd_cols_to_elim,
-                             col_mark);
-
-   for (i = k = 0; i < A_offd_ncols; i++)
+   if (ignore_rows)
    {
-      if (col_mark[i]) { col_remap[i] = k++; }
-   }
+      hypre_CSRMatrixElimCreate(A_offd, Ae_offd,
+                                0, nullptr,
+                                num_offd_cols_to_elim, offd_cols_to_elim,
+                                col_mark);
 
-   hypre_CSRMatrixEliminateRowsCols(A_offd, Ae_offd,
-                                    num_rowscols_to_elim, rowscols_to_elim,
-                                    num_offd_cols_to_elim, offd_cols_to_elim,
-                                    0, col_remap);
+      for (i = k = 0; i < A_offd_ncols; i++)
+      {
+         if (col_mark[i]) { col_remap[i] = k++; }
+      }
+
+      hypre_CSRMatrixEliminateRowsCols(A_offd, Ae_offd,
+                                       0, nullptr,
+                                       num_offd_cols_to_elim, offd_cols_to_elim,
+                                       0, col_remap);
+   }
+   else
+   {
+      hypre_CSRMatrixElimCreate(A_offd, Ae_offd,
+                                num_rowscols_to_elim, rowscols_to_elim,
+                                num_offd_cols_to_elim, offd_cols_to_elim,
+                                col_mark);
+
+      for (i = k = 0; i < A_offd_ncols; i++)
+      {
+         if (col_mark[i]) { col_remap[i] = k++; }
+      }
+
+      hypre_CSRMatrixEliminateRowsCols(A_offd, Ae_offd,
+                                       num_rowscols_to_elim, rowscols_to_elim,
+                                       num_offd_cols_to_elim, offd_cols_to_elim,
+                                       0, col_remap);
+   }
 
    /* create col_map_offd for Ae */
    Ae_offd_ncols = 0;
@@ -645,6 +707,18 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
 }
 
 
+// Eliminate rows from the diagonal and off-diagonal blocks of the matrix
+void hypre_ParCSRMatrixEliminateRows(hypre_ParCSRMatrix *A,
+                                     HYPRE_Int num_rows_to_elim,
+                                     const HYPRE_Int *rows_to_elim)
+{
+   hypre_CSRMatrix *A_diag = hypre_ParCSRMatrixDiag(A);
+   hypre_CSRMatrix *A_offd = hypre_ParCSRMatrixOffd(A);
+   hypre_CSRMatrixEliminateRows(A_diag, num_rows_to_elim, rows_to_elim);
+   hypre_CSRMatrixEliminateRows(A_offd, num_rows_to_elim, rows_to_elim);
+}
+
+
 /*--------------------------------------------------------------------------
  *                               Split
  *--------------------------------------------------------------------------*/
@@ -1484,7 +1558,7 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A,
                                    0, 0, 0);
 
       /* split C into diag and off-diag portions */
-      /* TODO: GenerateDiagAndOffd() uses an int array of size equal to the
+      /* FIXME: GenerateDiagAndOffd() uses an int array of size equal to the
          number of columns in csr_C_temp which is the global number of columns
          in A and B. This does not scale well. */
       ierr += GenerateDiagAndOffd(csr_C_temp, C,

linalg/hypre_parcsr.hpp

@@ -45,7 +45,14 @@ void hypre_ParCSRMatrixEliminateAXB(hypre_ParCSRMatrix *A,
 void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
                                     hypre_ParCSRMatrix **Ae,
                                     HYPRE_Int num_rowscols_to_elim,
-                                    HYPRE_Int *rowscols_to_elim);
+                                    HYPRE_Int *rowscols_to_elim,
+                                    int ignore_rows = 0);
+
+/** Eliminate rows from a hypre ParCSRMatrix, setting all entries in the listed
+    rows of the matrix to zero. */
+void hypre_ParCSRMatrixEliminateRows(hypre_ParCSRMatrix *A,
+                                     HYPRE_Int num_rows_to_elim,
+                                     const HYPRE_Int *rows_to_elim);
 
 /** Split matrix 'A' into nr x nc blocks, return nr x nc pointers to
     new parallel matrices. The array 'blocks' needs to be preallocated to hold

linalg/multigrid.cpp

@@ -56,8 +56,11 @@ void MultigridOperator::AddCoarsestLevel(Operator* opr, Solver* solver,
    smoothers.Append(solver);
    ownedOperators.Append(ownOperator);
    ownedSmoothers.Append(ownSolver);
-   width = opr->Width();
-   height = opr->Height();
+   if (opr)
+   {
+      width = opr->Width();
+      height = opr->Height();
+   }
 }
 
 void MultigridOperator::AddLevel(Operator* opr, Solver* smoother,
@@ -71,8 +74,11 @@ void MultigridOperator::AddLevel(Operator* opr, Solver* smoother,
    ownedOperators.Append(ownOperator);
    ownedSmoothers.Append(ownSmoother);
    ownedProlongations.Append(ownProlongation);
-   width = opr->Width();
-   height = opr->Height();
+   if (opr)
+   {
+      width = opr->Width();
+      height = opr->Height();
+   }
 }
 
 unsigned MultigridOperator::NumLevels() const { return operators.Size(); }
@@ -144,6 +150,15 @@ Solver* MultigridOperator::GetSmootherAtLevel(unsigned level)
    return smoothers[level];
 }
 
+void MultigridOperator::AddEmptyLevels(unsigned levels)
+{
+   AddCoarsestLevel(nullptr, nullptr, true, true);
+   for (unsigned i = 1; i < levels; ++i)
+   {
+      AddLevel(nullptr, nullptr, nullptr, true, true, true);
+   }
+}
+
 TimedMultigridOperator::TimedMultigridOperator() : MultigridOperator() {}
 
 TimedMultigridOperator::TimedMultigridOperator(Operator* opr,

linalg/multigrid.hpp

@@ -99,6 +99,9 @@ class MultigridOperator : public Operator
 
    /// Returns smoother at given level
    Solver* GetSmootherAtLevel(unsigned level);
+
+protected:
+   void AddEmptyLevels(unsigned levels);
 };
 
 class TimedMultigridOperator : public MultigridOperator

linalg/operator.cpp

@@ -20,40 +20,70 @@
 namespace mfem
 {
 
+void Operator::InitTVectors(const Operator *Po, const Operator *Ri,
+                            Vector &x, Vector &b,
+                            Vector &X, Vector &B) const
+{
+   if (Po)
+   {
+      // Variational restriction with Po
+      B.SetSize(Po->Width(), b);
+      Po->MultTranspose(b, B);
+   }
+   else
+   {
+      // B points to same data as b
+      B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+   }
+   if (Ri)
+   {
+      // Variational restriction with Ri
+      X.SetSize(Ri->Height(), x);
+      Ri->Mult(x, X);
+   }
+   else
+   {
+      // X points to same data as x
+      X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
+   }
+}
+
 void Operator::FormLinearSystem(const Array<int> &ess_tdof_list,
                                 Vector &x, Vector &b,
                                 Operator* &Aout, Vector &X, Vector &B,
                                 int copy_interior)
 {
-   ConstrainedOperator *constrainedA;
-   FormConstrainedSystemOperator(ess_tdof_list, constrainedA);
-
    const Operator *P = this->GetProlongation();
    const Operator *R = this->GetRestriction();
-
-   if (P)
-   {
-      // Variational restriction with P
-      B.SetSize(P->Width(), b);
-      P->MultTranspose(b, B);
-      X.SetSize(R->Height(), x);
-      R->Mult(x, X);
-   }
-   else
-   {
-      // rap, X and B point to the same data as this, x and b, respectively
-      X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-      B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
-   }
+   InitTVectors(P, R, x, b, X, B);
 
    if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
 
+   ConstrainedOperator *constrainedA;
+   FormConstrainedSystemOperator(ess_tdof_list, constrainedA);
+   constrainedA->EliminateRHS(X, B);
+   Aout = constrainedA;
+}
+
+void Operator::FormRectangularLinearSystem(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list, Vector &x, Vector &b,
+   Operator* &Aout, Vector &X, Vector &B)
+{
+   const Operator *Po = this->GetOutputProlongation();
+   const Operator *Ri = this->GetRestriction();
+   InitTVectors(Po, Ri, x, b, X, B);
+
+   RectangularConstrainedOperator *constrainedA;
+   FormRectangularConstrainedSystemOperator(trial_tdof_list, test_tdof_list,
+                                            constrainedA);
    constrainedA->EliminateRHS(X, B);
    Aout = constrainedA;
 }
 
 void Operator::RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x)
 {
+   // Same for Rectangular and Square operators
    const Operator *P = this->GetProlongation();
    if (P)
    {
@@ -71,21 +101,40 @@ void Operator::RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x)
    }
 }
 
+Operator * Operator::SetupRAP(const Operator *Pi, const Operator *Po)
+{
+   Operator *rap;
+   if (Pi)
+   {
+      if (Po)
+      {
+         rap = new RAPOperator(*Po, *this, *Pi);
+      }
+      else
+      {
+         rap = new ProductOperator(this, Pi, false,false);
+      }
+   }
+   else
+   {
+      if (Po)
+      {
+         TransposeOperator * PoT = new TransposeOperator(Po);
+         rap = new ProductOperator(PoT, this, true,false);
+      }
+      else
+      {
+         rap = this;
+      }
+   }
+   return rap;
+}
+
 void Operator::FormConstrainedSystemOperator(
    const Array<int> &ess_tdof_list, ConstrainedOperator* &Aout)
 {
    const Operator *P = this->GetProlongation();
-   Operator *rap;
-
-   if (P)
-   {
-      // Variational restriction with P
-      rap = new RAPOperator(*P, *this, *P);
-   }
-   else
-   {
-      rap = this;
-   }
+   Operator *rap = SetupRAP(P, P);
 
    // Impose the boundary conditions through a ConstrainedOperator, which owns
    // the rap operator when P and R are non-trivial
@@ -94,6 +143,23 @@ void Operator::FormConstrainedSystemOperator(
    Aout = A;
 }
 
+void Operator::FormRectangularConstrainedSystemOperator(
+   const Array<int> &trial_tdof_list, const Array<int> &test_tdof_list,
+   RectangularConstrainedOperator* &Aout)
+{
+   const Operator *Pi = this->GetProlongation();
+   const Operator *Po = this->GetOutputProlongation();
+   Operator *rap = SetupRAP(Pi, Po);
+
+   // Impose the boundary conditions through a RectangularConstrainedOperator,
+   // which owns the rap operator when P and R are non-trivial
+   RectangularConstrainedOperator *A
+      = new RectangularConstrainedOperator(rap,
+                                           trial_tdof_list, test_tdof_list,
+                                           rap != this);
+   Aout = A;
+}
+
 void Operator::FormSystemOperator(const Array<int> &ess_tdof_list,
                                   Operator* &Aout)
 {
@@ -102,6 +168,15 @@ void Operator::FormSystemOperator(const Array<int> &ess_tdof_list,
    Aout = A;
 }
 
+void Operator::FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
+                                             const Array<int> &test_tdof_list,
+                                             Operator* &Aout)
+{
+   RectangularConstrainedOperator *A;
+   FormRectangularConstrainedSystemOperator(trial_tdof_list, test_tdof_list, A);
+   Aout = A;
+}
+
 void Operator::FormDiscreteOperator(Operator* &Aout)
 {
    const Operator *Pin  = this->GetProlongation();
@@ -226,9 +301,10 @@ void ConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
       d_w[id] = d_x[id];
    });
 
+   // A.AddMult(w, b, -1.0); // if available to all Operators
    A->Mult(w, z);
-
    b -= z;
+
    // Use read+write access - we are modifying sub-vector of b
    auto d_b = b.ReadWrite();
    MFEM_FORALL(i, csz,
@@ -266,4 +342,76 @@ void ConstrainedOperator::Mult(const Vector &x, Vector &y) const
    });
 }
 
+RectangularConstrainedOperator::RectangularConstrainedOperator(
+   Operator *A,
+   const Array<int> &trial_list,
+   const Array<int> &test_list,
+   bool _own_A)
+   : Operator(A->Height(), A->Width()), A(A), own_A(_own_A)
+{
+   // 'mem_class' should work with A->Mult() and MFEM_FORALL():
+   mem_class = A->GetMemoryClass()*Device::GetMemoryClass();
+   MemoryType mem_type = GetMemoryType(mem_class);
+   trial_list.Read(); // TODO: just ensure 'list' is registered, no need to copy it
+   test_list.Read(); // TODO: just ensure 'list' is registered, no need to copy it
+   trial_constraints.MakeRef(trial_list);
+   test_constraints.MakeRef(test_list);
+   // typically z and w are large vectors, so store them on the device
+   z.SetSize(height, mem_type); z.UseDevice(true);
+   w.SetSize(width, mem_type); w.UseDevice(true);
+}
+
+void RectangularConstrainedOperator::EliminateRHS(const Vector &x,
+                                                  Vector &b) const
+{
+   w = 0.0;
+   const int trial_csz = trial_constraints.Size();
+   auto trial_idx = trial_constraints.Read();
+   auto d_x = x.Read();
+   // Use read+write access - we are modifying sub-vector of w
+   auto d_w = w.ReadWrite();
+   MFEM_FORALL(i, trial_csz,
+   {
+      const int id = trial_idx[i];
+      d_w[id] = d_x[id];
+   });
+
+   // A.AddMult(w, b, -1.0); // if available to all Operators
+   A->Mult(w, z);
+   b -= z;
+
+   const int test_csz = test_constraints.Size();
+   auto test_idx = test_constraints.Read();
+   auto d_b = b.ReadWrite();
+   MFEM_FORALL(i, test_csz, d_b[test_idx[i]] = 0.0;);
+}
+
+void RectangularConstrainedOperator::Mult(const Vector &x, Vector &y) const
+{
+   const int trial_csz = trial_constraints.Size();
+   const int test_csz = test_constraints.Size();
+   if (trial_csz == 0)
+   {
+      A->Mult(x, y);
+   }
+   else
+   {
+      w = x;
+
+      auto idx = trial_constraints.Read();
+      // Use read+write access - we are modifying sub-vector of w
+      auto d_w = w.ReadWrite();
+      MFEM_FORALL(i, trial_csz, d_w[idx[i]] = 0.0;);
+
+      A->Mult(w, y);
+   }
+
+   if (test_csz != 0)
+   {
+      auto idx = test_constraints.Read();
+      auto d_y = y.ReadWrite();
+      MFEM_FORALL(i, test_csz, d_y[idx[i]] = 0.0;);
+   }
+}
+
 }

linalg/operator.hpp

@@ -18,6 +18,7 @@ namespace mfem
 {
 
 class ConstrainedOperator;
+class RectangularConstrainedOperator;
 
 /// Abstract operator
 class Operator
@@ -30,7 +31,21 @@ protected:
    void FormConstrainedSystemOperator(
       const Array<int> &ess_tdof_list, ConstrainedOperator* &Aout);
 
+   /// see FormRectangularSystemOperator()
+   void FormRectangularConstrainedSystemOperator(
+      const Array<int> &trial_tdof_list,
+      const Array<int> &test_tdof_list,
+      RectangularConstrainedOperator* &Aout);
+
+   /// Returns RAP Operator of this, taking in input/output Prolongation matrices
+   Operator *SetupRAP(const Operator *Pi, const Operator *Po);
+
 public:
+   /// Initializes memory for true vectors of linear system
+   void InitTVectors(const Operator *Po, const Operator *Ri,
+                     Vector &x, Vector &b,
+                     Vector &X, Vector &B) const;
+
    /// Construct a square Operator with given size s (default 0).
    explicit Operator(int s = 0) { height = width = s; }
 
@@ -83,9 +98,18 @@ public:
    /** @brief Restriction operator from input vectors for the operator to linear
        algebra (linear system) vectors. `NULL` means identity. */
    virtual const Operator *GetRestriction() const  { return NULL; }
+   /** @brief Prolongation operator from linear algebra (linear system) vectors,
+       to output vectors for the operator. `NULL` means identity. */
+   virtual const Operator *GetOutputProlongation() const
+   {
+      return GetProlongation(); // Assume square unless specialized
+   }
    /** @brief Restriction operator from output vectors for the operator to linear
        algebra (linear system) vectors. `NULL` means identity. */
-   virtual const Operator *GetOutputRestriction() const  { return NULL; }
+   virtual const Operator *GetOutputRestriction() const
+   {
+      return GetRestriction(); // Assume square unless specialized
+   }
 
    /** @brief Form a constrained linear system using a matrix-free approach.
 
@@ -122,9 +146,40 @@ public:
                          Operator* &A, Vector &X, Vector &B,
                          int copy_interior = 0);
 
+   /** @brief Form a column-constrained linear system using a matrix-free approach.
+
+       Form the operator linear system `A(X)=B`
+       corresponding to it and the right-hand side @a b, by applying any
+       necessary transformations such as: parallel assembly, conforming
+       constraints for non-conforming AMR and eliminating boundary conditions.
+       @note Static condensation and hybridization are not supported for general
+       operators (cf. the method MixedBilinearForm::FormRectangularLinearSystem())
+
+       The constraints are specified through the input prolongation Pi from
+       GetProlongation(), and output restriction Ro from GetOutputRestriction()
+       methods, which are e.g. available through the (parallel) finite element
+       spaces of any (parallel) mixed bilinear form operator. So we have:
+       `A(X)=[Ro (*this) Pi](X)`, `B=Ro(b)`, and `X=Pi^T(x)`.
+
+       The vector @a x must contain the essential boundary condition values.
+       The "columns" in this operator corresponding to these values are
+       eliminated through the RectangularConstrainedOperator class.
+
+       After solving the system `A(X)=B`, the (finite element) solution @a x can
+       be recovered by calling Operator::RecoverFEMSolution() with the same
+       vectors @a X, @a b, and @a x.
+
+       @note The caller is responsible for destroying the output operator @a A!
+       @note If there are no transformations, @a X simply reuses the data of @a
+       x. */
+   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list,
+                                    Vector &x, Vector &b,
+                                    Operator* &A, Vector &X, Vector &B);
+
    /** @brief Reconstruct a solution vector @a x (e.g. a GridFunction) from the
        solution @a X of a constrained linear system obtained from
-       Operator::FormLinearSystem().
+       Operator::FormLinearSystem() or Operator::FormRectangularLinearSystem().
 
        Call this method after solving a linear system constructed using
        Operator::FormLinearSystem() to recover the solution as an input vector,
@@ -141,6 +196,15 @@ public:
    void FormSystemOperator(const Array<int> &ess_tdof_list,
                            Operator* &A);
 
+   /** @brief Return in @a A a parallel (on truedofs) version of this
+       rectangular operator (including constraints).
+
+       This returns the same operator as FormRectangularLinearSystem(), but does without
+       the transformations of the right-hand side. */
+   void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
+                                      const Array<int> &test_tdof_list,
+                                      Operator* &A);
+
    /** @brief Return in @a A a parallel (on truedofs) version of this
        rectangular operator.
 
@@ -467,6 +531,7 @@ public:
        when this object is destroyed. */
    ConstrainedOperator(Operator *A, const Array<int> &list, bool own_A = false);
 
+   /// Returns the type of memory in which the solution and temporaries are stored.
    virtual MemoryClass GetMemoryClass() const { return mem_class; }
 
    /** @brief Eliminate "essential boundary condition" values specified in @a x
@@ -494,6 +559,57 @@ public:
    virtual ~ConstrainedOperator() { if (own_A) { delete A; } }
 };
 
+/** @brief Rectangular Operator for imposing essential boundary conditions on the
+    input space using only the action, Mult(), of a given unconstrained Operator.
+
+    Rectangular operator constrained by fixing certain entries in the solution to
+    given "essential boundary condition" values. This class is used by the
+    general, matrix-free system formulation of Operator::FormRectangularLinearSystem. */
+class RectangularConstrainedOperator : public Operator
+{
+protected:
+   Array<int> trial_constraints, test_constraints;
+   Operator *A;
+   bool own_A;
+   mutable Vector z, w;
+   MemoryClass mem_class;
+
+public:
+   /** @brief Constructor from a general Operator and a list of essential
+       indices/dofs.
+
+       Specify the unconstrained operator @a *A and two lists of indices to
+       constrain, i.e. each entry @a trial_list[i] represents an essential
+       trial dof. If the ownership flag @a own_A is true, the operator @a *A
+       will be destroyed when this object is destroyed. */
+   RectangularConstrainedOperator(Operator *A, const Array<int> &trial_list,
+                                  const Array<int> &test_list, bool own_A = false);
+   /// Returns the type of memory in which the solution and temporaries are stored.
+   virtual MemoryClass GetMemoryClass() const { return mem_class; }
+   /** @brief Eliminate columns corresponding to "essential boundary condition"
+       values specified in @a x from the given right-hand side @a b.
+
+       Performs the following steps:
+
+           b -= A((0,x_b));
+           b_j = 0
+
+       where the "_b" subscripts denote the essential (boundary) indices
+       and the "_j" subscript denotes the essential test indices */
+   void EliminateRHS(const Vector &x, Vector &b) const;
+   /** @brief Rectangular-constrained operator action.
+
+       Performs the following steps:
+
+           y = A((x_i,0));
+           y_j = 0
+
+       where the "_i" subscripts denote all the nonessential (boundary) trial indices
+       and the "_j" subscript denotes the essential test indices */
+   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual ~RectangularConstrainedOperator() { if (own_A) { delete A; } }
+};
+
 }
 
 #endif

linalg/solvers.cpp

@@ -92,6 +92,11 @@ void IterativeSolver::SetPreconditioner(Solver &pr)
    prec->iterative_mode = false;
 }
 
+void IterativeSolver::ClearPreconditioner()
+{
+   prec = NULL;
+}
+
 void IterativeSolver::SetOperator(const Operator &op)
 {
    oper = &op;

linalg/solvers.hpp

@@ -69,6 +69,9 @@ public:
 
    /// This should be called before SetOperator
    virtual void SetPreconditioner(Solver &pr);
+   /// Clear the Preconditioner currently stored to NULL.
+   /// Does NOT delete the underlying preconditioner.
+   void ClearPreconditioner();
 
    /// Also calls SetOperator for the preconditioner if there is one
    virtual void SetOperator(const Operator &op);

linalg/sparsemat.cpp

@@ -35,7 +35,8 @@ SparseMatrix::SparseMatrix(int nrows, int ncols)
      ColPtrJ(NULL),
      ColPtrNode(NULL),
      At(NULL),
-     isSorted(false)
+     isSorted(false),
+     use_dev(true)
 {
    // We probably do not need to set the ownership flags here.
    I.Reset(); I.SetHostPtrOwner(true);
@@ -58,7 +59,8 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
      ColPtrJ(NULL),
      ColPtrNode(NULL),
      At(NULL),
-     isSorted(false)
+     isSorted(false),
+     use_dev(true)
 {
    I.Wrap(i, height+1, true);
    J.Wrap(j, I[height], true);
@@ -76,7 +78,8 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
      ColPtrJ(NULL),
      ColPtrNode(NULL),
      At(NULL),
-     isSorted(issorted)
+     isSorted(issorted),
+     use_dev(true)
 {
    I.Wrap(i, height+1, ownij);
    J.Wrap(j, I[height], ownij);
@@ -98,12 +101,13 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
 }
 
 SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
-   : AbstractSparseMatrix(nrows, ncols)
-   , Rows(NULL)
-   , ColPtrJ(NULL)
-   , ColPtrNode(NULL)
-   , At(NULL)
-   , isSorted(false)
+   : AbstractSparseMatrix(nrows, ncols),
+     Rows(NULL),
+     ColPtrJ(NULL),
+     ColPtrNode(NULL),
+     At(NULL),
+     isSorted(false),
+     use_dev(true)
 {
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = NULL;
@@ -181,15 +185,17 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
    ColPtrNode = NULL;
    At = NULL;
    isSorted = mat.isSorted;
+   use_dev = mat.UseDevice();
 }
 
 SparseMatrix::SparseMatrix(const Vector &v)
-   : AbstractSparseMatrix(v.Size(), v.Size())
-   , Rows(NULL)
-   , ColPtrJ(NULL)
-   , ColPtrNode(NULL)
-   , At(NULL)
-   , isSorted(true)
+   : AbstractSparseMatrix(v.Size(), v.Size()),
+     Rows(NULL),
+     ColPtrJ(NULL),
+     ColPtrNode(NULL),
+     At(NULL),
+     isSorted(true),
+     use_dev(true)
 {
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = NULL;
@@ -230,6 +236,7 @@ void SparseMatrix::MakeRef(const SparseMatrix &master)
    J = master.J; J.ClearOwnerFlags();
    A = master.A; A.ClearOwnerFlags();
    isSorted = master.isSorted;
+   use_dev = master.UseDevice();
 }
 
 void SparseMatrix::SetEmpty()
@@ -544,7 +551,7 @@ void SparseMatrix::ToDenseMatrix(DenseMatrix & B) const
 
 void SparseMatrix::Mult(const Vector &x, Vector &y) const
 {
-   if (Finalized()) { y.UseDevice(true); }
+   if (Finalized()) { y.UseDevice(UseDevice()); }
    y = 0.0;
    AddMult(x, y);
 }
@@ -579,12 +586,12 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
 #ifndef MFEM_USE_LEGACY_OPENMP
    const int height = this->height;
    const int nnz = J.Capacity();
-   auto d_I = Read(I, height+1);
-   auto d_J = Read(J, nnz);
-   auto d_A = Read(A, nnz);
-   auto d_x = x.Read();
-   auto d_y = y.ReadWrite();
-   MFEM_FORALL(i, height,
+   auto d_I = Read(I, height+1, UseDevice());
+   auto d_J = Read(J, nnz, UseDevice());
+   auto d_A = Read(A, nnz, UseDevice());
+   auto d_x = x.Read(UseDevice());
+   auto d_y = y.ReadWrite(UseDevice());
+   MFEM_FORALL_SWITCH(UseDevice(), i, height,
    {
       double d = 0.0;
       const int end = d_I[i+1];
@@ -615,7 +622,7 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
 
 void SparseMatrix::MultTranspose(const Vector &x, Vector &y) const
 {
-   if (Finalized()) { y.UseDevice(true); }
+   if (Finalized()) { y.UseDevice(UseDevice()); }
    y = 0.0;
    AddMultTranspose(x, y);
 }
@@ -686,13 +693,13 @@ void SparseMatrix::PartMult(
 
    const int n = rows.Size();
    const int nnz = J.Capacity();
-   auto d_rows = rows.Read();
-   auto d_I = Read(I, height+1);
-   auto d_J = Read(J, nnz);
-   auto d_A = Read(A, nnz);
-   auto d_x = x.Read();
-   auto d_y = y.Write();
-   MFEM_FORALL(i, n,
+   auto d_rows = rows.Read(UseDevice());
+   auto d_I = Read(I, height+1, UseDevice());
+   auto d_J = Read(J, nnz, UseDevice());
+   auto d_A = Read(A, nnz, UseDevice());
+   auto d_x = x.Read(UseDevice());
+   auto d_y = y.Write(UseDevice());
+   MFEM_FORALL_SWITCH(UseDevice(), i, n,
    {
       const int r = d_rows[i];
       const int end = d_I[r + 1];
@@ -729,15 +736,15 @@ void SparseMatrix::BooleanMult(const Array<int> &x, Array<int> &y) const
    MFEM_ASSERT(x.Size() == Width(), "Input vector size (" << x.Size()
                << ") must match matrix width (" << Width() << ")");
 
-   y.SetSize(Height(), Device::GetMemoryType());
+   y.SetSize(Height(), UseDevice() ? Device::GetMemoryType() : MemoryType::HOST );
 
    const int height = Height();
    const int nnz = J.Capacity();
-   auto d_I = Read(I, height+1);
-   auto d_J = Read(J, nnz);
-   auto d_x = Read(x.GetMemory(), x.Size());
-   auto d_y = Write(y.GetMemory(), y.Size());
-   MFEM_FORALL(i, height,
+   auto d_I = Read(I, height+1, UseDevice());
+   auto d_J = Read(J, nnz, UseDevice());
+   auto d_x = Read(x.GetMemory(), x.Size(), UseDevice());
+   auto d_y = Write(y.GetMemory(), y.Size(), UseDevice());
+   MFEM_FORALL_SWITCH(UseDevice(), i, height,
    {
       bool d_yi = false;
       const int end = d_I[i+1];
@@ -1020,6 +1027,7 @@ void SparseMatrix::GetBlocks(Array2D<SparseMatrix *> &blocks) const
             bI[k] = 0;
          }
          blocks(i,j) = new SparseMatrix(bI, NULL, NULL, nr, nc);
+         blocks(i,j)->UseDevice(UseDevice());
       }
    }
 
@@ -1387,6 +1395,41 @@ void SparseMatrix::EliminateCols(const Array<int> &cols, const Vector *x,
    }
 }
 
+void SparseMatrix::EliminateCols(const Array<int> &col_marker, SparseMatrix &Ae)
+{
+   if (Rows)
+   {
+      RowNode *nd;
+      for (int row = 0; row < height; row++)
+      {
+         for (nd = Rows[row]; nd != NULL; nd = nd->Prev)
+         {
+            if (col_marker[nd->Column])
+            {
+               Ae.Add(row, nd->Column, nd->Value);
+               nd->Value = 0.0;
+            }
+         }
+      }
+   }
+   else
+   {
+      for (int row = 0; row < height; row++)
+      {
+         for (int j = I[row]; j < I[row+1]; j++)
+         {
+            if (col_marker[J[j]])
+            {
+               Ae.Add(row, J[j], A[j]);
+               A[j] = 0.0;
+            }
+         }
+      }
+   }
+   Ae.UseDevice(UseDevice());
+}
+
+
 void SparseMatrix::EliminateRowCol(int rc, const double sol, Vector &rhs,
                                    DiagonalPolicy dpolicy)
 {
@@ -1841,6 +1884,7 @@ void SparseMatrix::EliminateRowCol(int rc, SparseMatrix &Ae,
          }
       }
    }
+   Ae.UseDevice(UseDevice());
 }
 
 void SparseMatrix::SetDiagIdentity()
@@ -2991,7 +3035,9 @@ SparseMatrix *Transpose (const SparseMatrix &A)
    }
    At_i[0] = 0;
 
-   return  new SparseMatrix(At_i, At_j, At_data, n, m);
+   SparseMatrix * At = new SparseMatrix(At_i, At_j, At_data, n, m);
+   At->UseDevice(A.UseDevice());
+   return At;
 }
 
 SparseMatrix *TransposeAbstractSparseMatrix (const AbstractSparseMatrix &A,
@@ -3195,7 +3241,7 @@ SparseMatrix *Mult (const SparseMatrix &A, const SparseMatrix &B,
       << counter);
 
    delete [] B_marker;
-
+   C->UseDevice( A.UseDevice() || B.UseDevice() );
    return C;
 }
 
@@ -3461,7 +3507,9 @@ SparseMatrix * Add(double a, const SparseMatrix & A, double b,
    }
 
    delete[] marker;
-   return new SparseMatrix(C_i, C_j, C_data, nrows, ncols);
+   SparseMatrix * C = new SparseMatrix(C_i, C_j, C_data, nrows, ncols);
+   C->UseDevice( A.UseDevice() || B.UseDevice() );
+   return C;
 }
 
 SparseMatrix * Add(const SparseMatrix & A, const SparseMatrix & B)
@@ -3475,6 +3523,7 @@ SparseMatrix * Add(Array<SparseMatrix *> & Ai)
 
    SparseMatrix * accumulate = Ai[0];
    SparseMatrix * result = accumulate;
+   bool use_dev = false;
 
    for (int i=1; i < Ai.Size(); ++i)
    {
@@ -3485,8 +3534,9 @@ SparseMatrix * Add(Array<SparseMatrix *> & Ai)
       }
 
       accumulate = result;
+      use_dev = ( use_dev || Ai[i]->UseDevice() );
    }
-
+   result->UseDevice(use_dev);
    return result;
 }
 
@@ -3548,7 +3598,7 @@ SparseMatrix *OuterProduct(const DenseMatrix &A, const SparseMatrix &B)
       }
    }
    C->Finalize();
-
+   C->UseDevice(B.UseDevice());
    return C;
 }
 
@@ -3577,7 +3627,7 @@ SparseMatrix *OuterProduct(const SparseMatrix &A, const DenseMatrix &B)
       }
    }
    C->Finalize();
-
+   C->UseDevice(A.UseDevice());
    return C;
 }
 
@@ -3610,7 +3660,7 @@ SparseMatrix *OuterProduct(const SparseMatrix &A, const SparseMatrix &B)
       }
    }
    C->Finalize();
-
+   C->UseDevice( A.UseDevice() || B.UseDevice() );
    return C;
 }
 
@@ -3632,6 +3682,7 @@ void SparseMatrix::Swap(SparseMatrix &other)
 #endif
 
    mfem::Swap(isSorted, other.isSorted);
+   mfem::Swap(use_dev, other.use_dev);
 }
 
 }

linalg/sparsemat.hpp

@@ -76,6 +76,8 @@ protected:
 
    /// Are the columns sorted already.
    bool isSorted;
+   /// Should the CSR operations be done on the device
+   bool use_dev;
 
    void Destroy();   // Delete all owned data
    void SetEmpty();  // Init all entries with empty values
@@ -305,6 +307,10 @@ public:
    void EliminateCols(const Array<int> &cols, const Vector *x = NULL,
                       Vector *b = NULL);
 
+   /** @brief Similar to EliminateCols + save the eliminated entries into 
+       @a Ae so that (*this) + Ae is equal to the original matrix. */
+   void EliminateCols(const Array<int> &col_marker, SparseMatrix &Ae);
+
    /// Eliminate row @a rc and column @a rc and modify the @a rhs using @a sol.
    /** Eliminates the column @a rc to the @a rhs, deletes the row @a rc and
        replaces the element (rc,rc) with 1.0; assumes that element (i,rc)
@@ -369,9 +375,20 @@ public:
    /// A slightly more general version of the Finalize(int) method.
    void Finalize(int skip_zeros, bool fix_empty_rows);
 
+   /// Returns whether or not CSR format has been finalized.
    bool Finalized() const { return !A.Empty(); }
+   /// Returns whether or not the columns are sorted.
    bool areColumnsSorted() const { return isSorted; }
 
+   /** @brief Specify whether or not to use the device for CSR operations. */
+   /** By default, most CSR operations are done on the device. Calling UseDevice(false)
+       will force these to instead be done on the host. Note that all LIL operations
+       are currently only implemented on the host. */
+   void UseDevice(bool use_dev_) { use_dev = use_dev_; }
+
+   /// Returns whether or not the CSR operations will be done on the device
+   bool UseDevice() const { return use_dev; }
+
    /** @brief Remove entries smaller in absolute value than a given tolerance
        @a tol. If @a fix_empty_rows is true, a zero value is inserted in the
        diagonal entry (for square matrices only) */

linalg/vector.cpp

@@ -817,6 +817,7 @@ double Vector::Max() const
 {
    if (size == 0) { return -infinity(); }
 
+   HostRead();
    double max = data[0];
 
    for (int i = 1; i < size; i++)
@@ -832,13 +833,12 @@ double Vector::Max() const
 
 double Vector::Sum() const
 {
+   HostRead();
    double sum = 0.0;
-
    for (int i = 0; i < size; i++)
    {
       sum += data[i];
    }
-
    return sum;
 }
 

linalg/vector.hpp

@@ -83,7 +83,7 @@ public:
 
        Some derived classes, e.g. GridFunction, enable the use of the
        mfem::Device by default. */
-   void UseDevice(bool use_dev) const { data.UseDevice(use_dev); }
+   virtual void UseDevice(bool use_dev) const { data.UseDevice(use_dev); }
 
    /// Return the device flag of the Memory object used by the Vector
    bool UseDevice() const { return data.UseDevice(); }

mesh/mesh.cpp

@@ -3298,6 +3298,10 @@ Mesh::Mesh(Mesh *orig_mesh, int ref_factor, int ref_type)
    Array<int> rdofs;
    DenseMatrix phys_pts;
    int max_nv = 0;
+
+   DenseMatrix node_coordinates(spaceDim*pow(2, Dim), r_num_elem);
+   H1_FECollection vertex_fec(1, Dim);
+
    for (int el = 0; el < orig_mesh->GetNE(); el++)
    {
       Geometry::Type geom = orig_mesh->GetElementBaseGeometry(el);
@@ -3312,6 +3316,7 @@ Mesh::Mesh(Mesh *orig_mesh, int ref_factor, int ref_type)
       orig_mesh->GetElementTransformation(el)->Transform(rfe->GetNodes(),
                                                          phys_pts);
       const int *c2h_map = rfec.GetDofMap(geom);
+      const int *vertex_map = vertex_fec.GetDofMap(geom);
       for (int i = 0; i < phys_pts.Width(); i++)
       {
          vertices[rdofs[i]].SetCoords(spaceDim, phys_pts.GetColumn(i));
@@ -3326,9 +3331,24 @@ Mesh::Mesh(Mesh *orig_mesh, int ref_factor, int ref_type)
             int cid = RG.RefGeoms[k+nvert*j]; // local Cartesian index
             v[k] = rdofs[c2h_map[cid]];
          }
+         for (int k = 0; k < nvert; k++)
+         {
+            for (int j = 0; j < spaceDim; ++j)
+            {
+               node_coordinates(k*spaceDim + j, NumOfElements)
+                  = vertices[v[vertex_map[k]]](j);
+            }
+         }
          AddElement(elem);
       }
    }
+
+   SetCurvature(1, true, spaceDim);
+   Vector node_coordinates_vec(
+      node_coordinates.Data(),
+      node_coordinates.Width()*node_coordinates.Height());
+   SetNodes(node_coordinates_vec);
+
    // Add refined boundary elements
    for (int el = 0; el < orig_mesh->GetNBE(); el++)
    {
@@ -9580,6 +9600,7 @@ GeometricFactors::GeometricFactors(const Mesh *mesh, const IntegrationRule &ir,
    // For now, we are not using tensor product evaluation
    const Operator *elem_restr = fespace->GetElementRestriction(
                                    ElementDofOrdering::NATIVE);
+
    elem_restr->Mult(*nodes, Enodes);
 
    unsigned eval_flags = 0;

miniapps/CMakeLists.txt

@@ -15,7 +15,8 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 set(MINIAPP_COMMON_HEADERS)
 add_subdirectory(common)
 add_subdirectory(electromagnetics)
+add_subdirectory(fluids)
 add_subdirectory(meshing)
 add_subdirectory(performance)
 add_subdirectory(tools)
-add_subdirectory(nurbs)
+add_subdirectory(nurbs)

miniapps/common/CMakeLists.txt

@@ -27,3 +27,4 @@ endif()
 
 add_library(mfem_miniapps_common ${MFEM_MINIAPPS_COMMON_SOURCES}
             ${MFEM_MINIAPPS_COMMON_HEADERS})
+target_link_libraries(mfem_miniapps_common mfem)

miniapps/fluids/CMakeLists.txt

@@ -0,0 +1,47 @@
+# Copyright (c) 2019, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+if (MFEM_USE_MPI)
+  add_mfem_miniapp(navier_tgv2d
+    MAIN navier_tgv2d.cpp
+    LIBRARIES mfem navier)
+
+  add_mfem_miniapp(navier_tgv3d
+    MAIN navier_tgv3d.cpp
+    LIBRARIES mfem navier)
+
+  add_mfem_miniapp(navier_kovasznay
+    MAIN navier_kovasznay.cpp
+    LIBRARIES mfem navier)
+
+  add_mfem_miniapp(navier_mms
+    MAIN navier_mms.cpp
+    LIBRARIES mfem navier)
+
+  add_library(navier
+    navier_solver.cpp navier_solver.hpp
+    ortho_solver.cpp ortho_solver.hpp)
+
+  target_link_libraries(navier mfem)
+
+  target_include_directories(navier INTERFACE
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<INSTALL_INTERFACE:include/navier>)
+
+  install(TARGETS navier
+    EXPORT ${PROJECT_NAME_UC}Targets
+    DESTINATION ${INSTALL_LIB_DIR})
+
+  install(FILES 
+    ${CMAKE_CURRENT_SOURCE_DIR}/navier_solver.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ortho_solver.hpp
+    DESTINATION include/navier)
+endif()

miniapps/fluids/navier_ethier.cpp

@@ -0,0 +1,132 @@
+#include "flow_solver.hpp"
+#include <fstream>
+
+using namespace mfem;
+using namespace flow;
+
+struct s_FlowContext
+{
+   int order = 7;
+   double kin_vis = 1.0 / 10.0; 
+   double t_final = 1.0;
+   double dt = 1e-4;
+} ctx;
+
+void vel_ethier(const Vector &x, double t, Vector &u)
+{
+   double xi = x(0);
+   double yi = x(1);
+   double zi = x(2);
+   double a = M_PI / 4.0; 
+   double d = M_PI / 2.0;
+
+   double ex = exp(a * xi);
+   double ey = exp(a * yi);
+   double ez = exp(a * zi);
+
+   double e2t = exp(-ctx.kin_vis * d * d * t);
+   
+   double exy = exp(a * (xi + yi));
+   double eyz = exp(a * (yi + zi));
+   double ezx = exp(a * (zi + xi));
+
+   double sxy = sin(a * xi + d * yi);
+   double syz = sin(a * yi + d * zi);
+   double szx = sin(a * zi + d * xi);
+
+   double cxy = cos(a * xi + d * yi);
+   double cyz = cos(a * yi + d * zi);
+   double czx = cos(a * zi + d * xi);
+
+   u(0) = -a * (ex * syz + ez * cxy) * e2t;
+   u(1) = -a * (ey * szx + ex * cyz) * e2t;
+   u(2) = -a * (ez * sxy + ey * czx) * e2t;
+}
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi(argc, argv);
+
+   int serial_refinements = 0;
+
+   Mesh *mesh = new Mesh("../data/inline-hex.mesh");
+
+   mesh->EnsureNodes();
+   GridFunction *nodes = mesh->GetNodes();
+   *nodes *= 2.0;
+   *nodes -= 1.0;
+
+   for (int i = 0; i < serial_refinements; ++i)
+   {
+      mesh->UniformRefinement();
+   }
+
+   if (mpi.Root())
+   {
+      std::cout << "Number of elements: " << mesh->GetNE() << std::endl;
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // Create the flow solver.
+   FlowSolver flowsolver(pmesh, ctx.order, ctx.kin_vis);
+
+   // Set the initial condition.
+   // This is completely user customizeable.
+   ParGridFunction *u_ic = flowsolver.GetCurrentVelocity();
+   VectorFunctionCoefficient u_excoeff(pmesh->Dimension(), vel_ethier);
+   u_ic->ProjectCoefficient(u_excoeff);
+
+   // Add Dirichlet boundary conditions to velocity space restricted to
+   // selected attributes on the mesh.
+   Array<int> attr(pmesh->bdr_attributes.Max());
+   attr = 1;
+   flowsolver.AddVelDirichletBC(vel_ethier, attr);
+
+   double t = 0.0;
+   double dt = ctx.dt;
+   double t_final = ctx.t_final;
+   bool last_step = false;
+
+   flowsolver.Setup(dt);
+
+   ParGridFunction *u_gf = flowsolver.GetCurrentVelocity();
+   ParGridFunction *p_gf = flowsolver.GetCurrentPressure();
+
+   VisItDataCollection visit_dc("ins", pmesh);
+   visit_dc.SetPrefixPath("output");
+   visit_dc.SetCycle(0);
+   visit_dc.SetTime(t);
+   visit_dc.RegisterField("velocity", u_gf);
+   visit_dc.RegisterField("pressure", p_gf);
+   visit_dc.Save();
+
+   for (int step = 0; !last_step; ++step)
+   {
+      if (t + dt >= t_final - dt / 2)
+      {
+         last_step = true;
+      }
+
+      flowsolver.Step(t, dt, step);
+
+      if ((step + 1) % 10 == 0 || last_step)
+      {
+         visit_dc.SetCycle(step);
+         visit_dc.SetTime(t);
+         visit_dc.Save();
+      }
+      if (mpi.Root())
+      {
+         printf("%.5E %.5E\n", t, dt);
+         fflush(stdout);
+      }
+   }
+
+   flowsolver.PrintTimingData();
+
+   delete pmesh;
+
+   return 0;
+}

miniapps/fluids/navier_kovasznay.cpp

@@ -0,0 +1,173 @@
+#include "navier_solver.hpp"
+#include <fstream>
+
+using namespace mfem;
+using namespace navier;
+
+struct s_NavierContext
+{
+   int order = 2;
+   double kin_vis = 1.0 / 40.0;
+   double t_final = 1000e-5;
+   double dt = 1e-5;
+   int ser_ref_levels = 1;
+   bool pa = false;
+   bool ni = false;
+} ctx;
+
+void vel_kovasznay(const Vector &x, double t, Vector &u)
+{
+   double xi = x(0);
+   double yi = x(1);
+
+   double reynolds = 1.0 / ctx.kin_vis;
+   double lam = 0.5 * reynolds
+                - sqrt(0.25 * reynolds * reynolds + 4.0 * M_PI * M_PI);
+
+   u(0) = 1.0 - exp(lam * xi) * cos(2.0 * M_PI * yi);
+   u(1) = lam / (2.0 * M_PI) * exp(lam * xi) * sin(2.0 * M_PI * yi);
+}
+
+double pres_kovasznay(const Vector &x)
+{
+   double xi = x(0);
+   double yi = x(1);
+
+   double reynolds = 1.0 / ctx.kin_vis;
+   double lam = 0.5 * reynolds
+                - sqrt(0.25 * reynolds * reynolds + 4.0 * M_PI * M_PI);
+
+   return -0.5 * exp(2.0 * lam * xi);
+}
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi(argc, argv);
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ctx.ser_ref_levels,
+                  "-rs",
+                  "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&ctx.order,
+                  "-o",
+                  "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ctx.dt, "-dt", "--time-step", "Time step.");
+   args.AddOption(&ctx.t_final, "-tf", "--final-time", "Final time.");
+   args.AddOption(&ctx.pa,
+                  "-pa",
+                  "--enable-pa",
+                  "-no-pi",
+                  "--disable-pi",
+                  "Enable partial assembly.");
+   args.AddOption(&ctx.ni,
+                  "-ni",
+                  "--enable-ni",
+                  "-no-ni",
+                  "--disable-ni",
+                  "Enable numerical integration rules.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (mpi.Root())
+      {
+         args.PrintUsage(std::cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (mpi.Root())
+   {
+      args.PrintOptions(std::cout);
+   }
+
+   Mesh *mesh = new Mesh(2, 4, Element::QUADRILATERAL, false, 1.5, 2.0);
+
+   mesh->EnsureNodes();
+   GridFunction *nodes = mesh->GetNodes();
+   *nodes -= 0.5;
+
+   for (int i = 0; i < ctx.ser_ref_levels; ++i)
+   {
+      mesh->UniformRefinement();
+   }
+
+   if (mpi.Root())
+   {
+      std::cout << "Number of elements: " << mesh->GetNE() << std::endl;
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // Create the flow solver.
+   NavierSolver naviersolver(pmesh, ctx.order, ctx.kin_vis);
+   naviersolver.EnablePA(ctx.pa);
+   naviersolver.EnableNI(ctx.ni);
+
+   // Set the initial condition.
+   // This is completely user customizeable.
+   ParGridFunction *u_ic = naviersolver.GetCurrentVelocity();
+   VectorFunctionCoefficient u_excoeff(pmesh->Dimension(), vel_kovasznay);
+   u_ic->ProjectCoefficient(u_excoeff);
+
+   FunctionCoefficient p_excoeff(pres_kovasznay);
+
+   // Add Dirichlet boundary conditions to velocity space restricted to
+   // selected attributes on the mesh.
+   Array<int> attr(pmesh->bdr_attributes.Max());
+   attr = 1;
+   naviersolver.AddVelDirichletBC(vel_kovasznay, attr);
+
+
+   double t = 0.0;
+   double dt = ctx.dt;
+   double t_final = ctx.t_final;
+   bool last_step = false;
+
+   naviersolver.Setup(dt);
+
+   double err_u = 0.0;
+   double err_p = 0.0;
+   ParGridFunction *u_gf = nullptr;
+   ParGridFunction *p_gf = nullptr;
+
+   for (int step = 0; !last_step; ++step)
+   {
+      if (t + dt >= t_final - dt / 2)
+      {
+         last_step = true;
+      }
+
+      naviersolver.Step(t, dt, step);
+
+      // Compare against exact solution of velocity and pressure.
+      u_gf = naviersolver.GetCurrentVelocity();
+      p_gf = naviersolver.GetCurrentPressure();
+      u_excoeff.SetTime(t);
+      p_excoeff.SetTime(t);
+      err_u = u_gf->ComputeL2Error(u_excoeff);
+      err_p = p_gf->ComputeL2Error(p_excoeff);
+
+      if (mpi.Root())
+      {
+         printf("%10.5E %10.5E %3d %10.5E %10.5E err\n", t, dt, ctx.order, err_u, err_p);
+         fflush(stdout);
+      }
+   }
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock(vishost, visport);
+   sol_sock.precision(8);
+   sol_sock << "parallel " << mpi.WorldSize() << " " << mpi.WorldRank() << "\n";
+   sol_sock << "solution\n" << *pmesh << *u_ic << std::flush;
+
+   naviersolver.PrintTimingData();
+
+   delete pmesh;
+
+   return 0;
+}

miniapps/fluids/navier_mms.cpp

@@ -0,0 +1,200 @@
+#include "navier_solver.hpp"
+#include <fstream>
+
+using namespace mfem;
+using namespace navier;
+
+struct s_NavierContext
+{
+   int order = 5;
+   double kinvis = 1.0;
+   double t_final = 0.5;
+   double dt = 0.25e-1;
+} ctx;
+
+void vel(const Vector &x, double t, Vector &u)
+{
+   double xi = x(0);
+   double yi = x(1);
+
+   u(0) = M_PI * sin(t) * pow(sin(M_PI * xi), 2.0) * sin(2.0 * M_PI * yi);
+   u(1) = -(M_PI * sin(t) * sin(2.0 * M_PI * xi) * pow(sin(M_PI * yi), 2.0));
+}
+
+double p(const Vector &x, double t)
+{
+   double xi = x(0);
+   double yi = x(1);
+
+   return cos(M_PI * xi) * sin(t) * sin(M_PI * yi);
+}
+
+void accel(const Vector &x, double t, Vector &u)
+{
+   double xi = x(0);
+   double yi = x(1);
+
+   u(0) = M_PI * sin(t) * sin(M_PI * xi) * sin(M_PI * yi)
+             * (-1.0
+                + 2.0 * pow(M_PI, 2.0) * sin(t) * sin(M_PI * xi)
+                     * sin(2.0 * M_PI * xi) * sin(M_PI * yi))
+          + M_PI
+               * (2.0 * ctx.kinvis * pow(M_PI, 2.0)
+                     * (1.0 - 2.0 * cos(2.0 * M_PI * xi)) * sin(t)
+                  + cos(t) * pow(sin(M_PI * xi), 2.0))
+               * sin(2.0 * M_PI * yi);
+
+   u(1) = M_PI * cos(M_PI * yi) * sin(t)
+             * (cos(M_PI * xi)
+                + 2.0 * ctx.kinvis * pow(M_PI, 2.0) * cos(M_PI * yi)
+                     * sin(2.0 * M_PI * xi))
+          - M_PI * (cos(t) + 6.0 * ctx.kinvis * pow(M_PI, 2.0) * sin(t))
+               * sin(2.0 * M_PI * xi) * pow(sin(M_PI * yi), 2.0)
+          + 4.0 * pow(M_PI, 3.0) * cos(M_PI * yi) * pow(sin(t), 2.0)
+               * pow(sin(M_PI * xi), 2.0) * pow(sin(M_PI * yi), 3.0);
+}
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi(argc, argv);
+
+   int serial_refinements = 1;
+
+   Mesh *mesh = new Mesh("../data/inline-quad.mesh");
+   mesh->EnsureNodes();
+   GridFunction *nodes = mesh->GetNodes();
+   *nodes *= 2.0;
+   *nodes -= 1.0;
+
+   for (int i = 0; i < serial_refinements; ++i)
+   {
+      mesh->UniformRefinement();
+   }
+
+   if (mpi.Root())
+   {
+      std::cout << "Number of elements: " << mesh->GetNE() << std::endl;
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // Create the flow solver.
+   NavierSolver naviersolver(pmesh, ctx.order, ctx.kinvis);
+   naviersolver.EnablePA(true);
+   naviersolver.EnableNI(false);
+   naviersolver.EnableVerbose(true);
+
+   // Set the initial condition.
+   // This is completely user customizeable.
+   ParGridFunction *u_ic = naviersolver.GetCurrentVelocity();
+   VectorFunctionCoefficient u_excoeff(pmesh->Dimension(), vel);
+   u_ic->ProjectCoefficient(u_excoeff);
+
+   FunctionCoefficient p_excoeff(p);
+
+   // Add Dirichlet boundary conditions to velocity space restricted to
+   // selected attributes on the mesh.
+   Array<int> attr(pmesh->bdr_attributes.Max());
+   attr = 1;
+   naviersolver.AddVelDirichletBC(vel, attr);
+
+   Array<int> domain_attr(pmesh->attributes.Max());
+   domain_attr = 1.0;
+   naviersolver.AddAccelTerm(accel, domain_attr);
+
+   double t = 0.0;
+   double dt = ctx.dt;
+   double t_final = ctx.t_final;
+   bool last_step = false;
+
+   naviersolver.Setup(dt);
+
+   double err_u = 0.0;
+   double err_p = 0.0;
+   ParGridFunction *u_gf = nullptr;
+   ParGridFunction *p_gf = nullptr;
+   u_gf = naviersolver.GetCurrentVelocity();
+   p_gf = naviersolver.GetCurrentPressure();
+
+   char vishost[] = "localhost";
+   int visport = 19916;
+   socketstream sol_sock(vishost, visport);
+   sol_sock.precision(8);
+   sol_sock << "parallel " << mpi.WorldSize() << " " << mpi.WorldRank() << "\n";
+   sol_sock << "solution\n" << *pmesh << *u_ic << "keys rRlj\n" << std::flush;
+
+   double cfl = 0.0;
+   double cfl_max = 0.8;
+   double cfl_atol = 1e-4;
+
+   for (int step = 0; !last_step; ++step)
+   {
+      if (t + dt >= t_final - dt / 2)
+      {
+         last_step = true;
+      }
+
+      if (true)
+      {
+         naviersolver.ProvisionalStep(t, dt, step);
+
+         cfl = naviersolver.ComputeCFL(*naviersolver.GetProvisionalVelocity(),
+                                       dt);
+         if (mpi.Root())
+         {
+            printf("CFL = %.5E\n", cfl);
+         }
+
+         double errest = cfl / (cfl_max + cfl_atol);
+         if (errest >= 1.0)
+         {
+            std::cout << "RETRY" << std::endl;
+            // Decline time step and retry
+            dt *= 0.5;
+            step -= 1;
+         }
+         else
+         {
+            // Accept the time step
+            t += dt;
+
+            // Make decision on new time step
+            double fac_safety = 2.0;
+            double eta = pow(1.0 / (fac_safety * errest), 1.0 / (1.0 + 3.0));
+            double fac_min = 0.1;
+            double fac_max = 10.0;
+            dt = dt * std::min(fac_max, std::max(fac_min, eta));
+
+            // Queue new time step in the history array
+            naviersolver.UpdateTimestepHistory(dt);
+         }
+      }
+      else
+      {
+         naviersolver.Step(t, dt, step);
+         t += dt;
+      }
+
+      // Compare against exact solution of velocity and pressure.
+      u_excoeff.SetTime(t);
+      p_excoeff.SetTime(t);
+      err_u = u_gf->ComputeL2Error(u_excoeff);
+      err_p = p_gf->ComputeL2Error(p_excoeff);
+
+      if (mpi.Root())
+      {
+        printf("%.5E %.5E %.5E %.5E err\n", t, dt, err_u, err_p);
+        fflush(stdout);
+      }
+   }
+
+   sol_sock << "parallel " << mpi.WorldSize() << " " << mpi.WorldRank() << "\n";
+   sol_sock << "solution\n" << *pmesh << *u_ic << std::flush;
+
+   naviersolver.PrintTimingData();
+
+   delete pmesh;
+
+   return 0;
+}

miniapps/fluids/navier_solver.hpp

@@ -0,0 +1,283 @@
+#pragma once
+
+#include "mfem.hpp"
+#include "ortho_solver.hpp"
+
+namespace mfem
+{
+namespace navier
+{
+typedef void(VecFuncT)(const Vector &x, double t, Vector &u);
+typedef double(ScalarFuncT)(const Vector &x, double t);
+
+/// Container for a Dirichlet boundary condition of the velocity field.
+class VelDirichletBC_T
+{
+public:
+   VelDirichletBC_T(VecFuncT *f,
+                    Array<int> attr,
+                    VectorFunctionCoefficient coeff)
+      : f(f), attr(attr), coeff(coeff)
+   {}
+
+   VecFuncT *f;
+   Array<int> attr;
+   VectorFunctionCoefficient coeff;
+};
+
+/// Container for a Dirichlet boundary condition of the pressure field.
+class PresDirichletBC_T
+{
+public:
+   PresDirichletBC_T(ScalarFuncT *f,
+                     Array<int> attr,
+                     FunctionCoefficient coeff)
+      : f(f), attr(attr), coeff(coeff)
+   {}
+
+   ScalarFuncT *f;
+   Array<int> attr;
+   FunctionCoefficient coeff;
+};
+
+/// Container for an acceleration term.
+class AccelTerm_T
+{
+public:
+   AccelTerm_T(VecFuncT *f,
+               Array<int> attr,
+               VectorFunctionCoefficient coeff)
+      : f(f), attr(attr), coeff(coeff)
+   {}
+
+   VecFuncT *f;
+   Array<int> attr;
+   VectorFunctionCoefficient coeff;
+};
+
+/// Navier Stokes solver.
+/**
+ * Transient Navier Stokes solver in a split scheme formulation.
+ */
+class NavierSolver
+{
+public:
+   NavierSolver(ParMesh *mesh, int order, double kin_vis);
+
+   void Setup(double dt);
+   
+   /// Compute provisional solution at the next time step t+dt.
+   /**
+    * Compute provisional solution at the next time step t+dt without
+    * automatically accepting the solution. The method should be used
+    * in combination with UpdateTimestepHistory if the user decides that
+    * the solution fulfills all a posteriori requirements.
+    */
+   void ProvisionalStep(double time, double dt, int cur_step);
+
+   /// Compute solution at the next time step t+dt.
+   /**
+    * Compute solution at the next time step t+dt and automatically
+    * accept the solution. This method should be used when using constant
+    * time steps.
+    */
+   void Step(double time, double dt, int cur_step);
+
+   /// Return a pointer to the current velocity ParGridFunction.
+   ParGridFunction *GetCurrentVelocity() { return &un_gf; }
+
+   /// Return a pointer to the provisional velocity ParGridFunction.
+   ParGridFunction *GetProvisionalVelocity() { return &un_next_gf; }
+
+   /// Return a pointer to the current pressure ParGridFunction.
+   ParGridFunction *GetCurrentPressure() { return &pn_gf; }
+
+   /// Add a Dirichlet boundary condition to the velocity field.
+   void AddVelDirichletBC(VecFuncT *f, Array<int> &attr);
+
+   /// Add a Dirichlet boundary condition to the pressure field.
+   void AddPresDirichletBC(ScalarFuncT *f, Array<int> &attr);
+
+   /// Add an accelaration term to the RHS of the equation.
+   /**
+    * The VectorFunction \p f is evaluated at the current time t
+    * and extrapolated with the nonlinear parts of the Navier Stokes
+    * equation.
+    */
+   void AddAccelTerm(VecFuncT *f, Array<int> &attr);
+
+   /// Enable partial assembly for every operator.
+   void EnablePA(bool pa = true) { partial_assembly = pa; }
+
+   /// Enable numerical integration rules.
+   void EnableNI(bool ni = true) { numerical_integ = ni; }
+
+   void EnableDebug(bool d = true) { debug = d; }
+
+   void EnableVerbose(bool v = true) { verbose = v; }
+
+   /// Rotate entries in the time step and solution history arrays.
+   void UpdateTimestepHistory(double dt);
+
+   /// Set the maximum order to use for the BDF method.
+   void SetMaxBDFOrder(int maxbdforder) { max_bdf_order = maxbdforder; };
+
+   /// Compute $\nabla times \nabla times u$ for $u \in (H^1)^2$
+   void ComputeCurl2D(ParGridFunction &u,
+                      ParGridFunction &cu,
+                      bool assume_scalar = false);
+
+   /// Compute $\nabla times \nabla times u$ for $u \in (H^1)^3$
+   void ComputeCurl3D(ParGridFunction &u, ParGridFunction &cu);
+
+   /// Compute the global maximum cell wise CFL number.
+   double ComputeCFL(ParGridFunction &u, double &dt);
+
+   void PrintTimingData();
+
+   ~NavierSolver();
+
+protected:
+   void PrintInfo();
+
+   // Set time integration coefficient based on the time step
+   // history. This works with variable and constant step size.
+   // For details of computation of the coefficient
+   // see [Wang and Ruuth, JSTOR, 2008].
+   void SetTimeIntegrationCoefficients(int step);
+
+   void Orthogonalize(Vector &v);
+
+   void MeanZero(ParGridFunction &v);
+
+   void EliminateRHS(Operator &A,
+                     ConstrainedOperator &constrainedA,
+                     const Array<int> &ess_tdof_list,
+                     Vector &x,
+                     Vector &b,
+                     Vector &X,
+                     Vector &B,
+                     int copy_interior = 0);
+
+   bool debug = false;
+   bool verbose = false;
+   bool partial_assembly = false;
+   bool numerical_integ = false;
+
+   ParMesh *pmesh;
+
+   double order;
+   double kin_vis;
+
+   IntegrationRules rules_ni;
+
+   FiniteElementCollection *vfec;
+   FiniteElementCollection *pfec;
+   ParFiniteElementSpace *vfes;
+   ParFiniteElementSpace *pfes;
+
+   ParNonlinearForm *N;
+   ParBilinearForm *Mv_form;
+   ParBilinearForm *Sp_form;
+   ParMixedBilinearForm *D_form;
+   ParMixedBilinearForm *G_form;
+   ParBilinearForm *H_form;
+
+   VectorGridFunctionCoefficient *FText_gfcoeff;
+   ParLinearForm *FText_bdr_form;
+   ParLinearForm *g_bdr_form;
+   ParLinearForm *f_form;
+   ParLinearForm *mass_lf = nullptr;
+
+   ConstantCoefficient onecoeff;
+   double volume = 0.0;
+
+   ConstantCoefficient nlcoeff;
+   ConstantCoefficient Sp_coeff;
+   ConstantCoefficient H_lincoeff;
+   ConstantCoefficient H_bdfcoeff;
+
+   OperatorHandle Mv;
+   OperatorHandle Sp;
+   OperatorHandle D;
+   OperatorHandle G;
+   OperatorHandle H;
+
+   Solver *MvInvPC;
+   CGSolver *MvInv;
+
+   Solver *SpInvPC;
+   OrthoSolver *SpInvOrthoPC;
+   CGSolver *SpInv;
+
+   Solver *HInvPC;
+   CGSolver *HInv;
+
+   Vector fn, un, un_next, unm1, unm2, Nun, Nunm1, Nunm2, Fext, FText, Lext,
+      resu;
+   Vector tmp1;
+
+   Vector pn, resp, FText_bdr, g_bdr;
+
+   ParGridFunction un_gf, un_next_gf, curlu_gf, curlcurlu_gf, Lext_gf, FText_gf,
+      resu_gf;
+
+   ParGridFunction pn_gf, resp_gf;
+
+   // All essential attributes
+   Array<int> vel_ess_attr;
+   Array<int> pres_ess_attr;
+
+   // All essential true dofs
+   Array<int> vel_ess_tdof;
+   Array<int> pres_ess_tdof;
+
+   // Bookkeeping for velocity dirichlet bcs
+   std::vector<VelDirichletBC_T> vel_dbcs;
+
+   // Bookkeeping for pressure dirichlet bcs
+   std::vector<PresDirichletBC_T> pres_dbcs;
+
+   // Bookkeeping for acceleration (forcing) terms
+   std::vector<AccelTerm_T> accel_terms;
+
+   int max_bdf_order = 3;
+   int cur_step = 0;
+   std::vector<double> dthist = {0.0, 0.0, 0.0};
+
+   // BDFk/EXTk coefficients
+   double bd0;
+   double bd1;
+   double bd2;
+   double bd3;
+   double ab1;
+   double ab2;
+   double ab3;
+
+   // Timers
+   StopWatch sw_setup, sw_step, sw_single_step, sw_extrap, sw_curlcurl,
+      sw_spsolve, sw_hsolve;
+
+   // Printlevels
+   int pl_mvsolve = 0;
+   int pl_spsolve = 0;
+   int pl_hsolve = 0;
+   int pl_amg = 0;
+
+   // Tolerances
+   double rtol_spsolve = 1e-12;
+   double rtol_hsolve = 1e-12;
+
+   // Iteration counts
+   int iter_mvsolve, iter_spsolve, iter_hsolve;
+
+   // Residuals
+   double res_mvsolve, res_spsolve, res_hsolve;
+
+   // LOR PC related
+   ParSpaceHierarchy *spaceHierarchy;
+   Array<H1_FECollection*>* collections;
+   ParMultigridBilinearForm* mgOperator;
+};
+} // namespace navier
+} // namespace mfem

miniapps/fluids/navier_tgv2d.cpp

@@ -0,0 +1,141 @@
+#include "navier_solver.hpp"
+#include <fstream>
+
+using namespace mfem;
+using namespace navier;
+
+struct s_NavierContext
+{
+   int order = 5;
+   double kin_vis = 1.0;
+   double t_final = 1.0;
+   double dt = 1e-2;
+} ctx;
+
+void vel_tgv(const Vector &x, double t, Vector &u)
+{
+   double xi = x(0);
+   double yi = x(1);
+
+   double F = exp(-2.0 * ctx.kin_vis * t);
+
+   u(0) = cos(xi) * sin(yi) * F;
+   u(1) = -sin(xi) * cos(yi) * F;
+}
+
+double p_tgv(const Vector &x, double t)
+{
+   double xi = x(0);
+   double yi = x(1);
+
+   double F = exp(-2.0 * ctx.kin_vis * t);
+
+   return -0.25 * (cos(2.0 * xi) + cos(2.0 * yi)) * pow(F, 2.0);
+}
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi(argc, argv);
+
+   int serial_refinements = 2;
+
+   Mesh *mesh = new Mesh("../data/inline-quad.mesh");
+   // Mesh *mesh = new Mesh("../data/periodic-square.mesh");
+
+   mesh->EnsureNodes();
+   GridFunction *nodes = mesh->GetNodes();
+   *nodes *= 2.0;
+   *nodes -= 1.0;
+   *nodes *= M_PI;
+
+   for (int i = 0; i < serial_refinements; ++i)
+   {
+      mesh->UniformRefinement();
+   }
+
+   if (mpi.Root())
+   {
+      std::cout << "Number of elements: " << mesh->GetNE() << std::endl;
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // Create the flow solver.
+   NavierSolver naviersolver(pmesh, ctx.order, ctx.kin_vis);
+   naviersolver.EnablePA(true);
+   naviersolver.EnableNI(false);
+
+   // Set the initial condition.
+   // This is completely user customizeable.
+   ParGridFunction *u_ic = naviersolver.GetCurrentVelocity();
+   VectorFunctionCoefficient u_excoeff(pmesh->Dimension(), vel_tgv);
+   u_ic->ProjectCoefficient(u_excoeff);
+
+   FunctionCoefficient p_excoeff(p_tgv);
+
+   // Add Dirichlet boundary conditions to velocity space restricted to
+   // selected attributes on the mesh.
+   Array<int> attr(pmesh->bdr_attributes.Max());
+   attr = 1;
+   naviersolver.AddVelDirichletBC(vel_tgv, attr);
+
+
+   double t = 0.0;
+   double dt = ctx.dt;
+   double t_final = ctx.t_final;
+   bool last_step = false;
+
+   naviersolver.Setup(dt);
+
+   double err_u = 0.0;
+   double err_p = 0.0;
+   ParGridFunction *u_gf = nullptr;
+   ParGridFunction *p_gf = nullptr;
+   u_gf = naviersolver.GetCurrentVelocity();
+   p_gf = naviersolver.GetCurrentPressure();
+
+   for (int step = 0; !last_step; ++step)
+   {
+      if (t + dt >= t_final - dt / 2)
+      {
+         last_step = true;
+      }
+
+      naviersolver.Step(t, dt, step);
+
+      // if (step > 2)
+      // {
+      //    double cfl = naviersolver.ComputeCFL(*u_gf, dt);
+      //    if (mpi.Root())
+      //    {
+      //       printf("CFL = %.5E\n", cfl);
+      //    }
+      // }
+
+      // Compare against exact solution of velocity and pressure.
+      u_excoeff.SetTime(t);
+      p_excoeff.SetTime(t);
+      err_u = u_gf->ComputeL2Error(u_excoeff);
+      err_p = p_gf->ComputeL2Error(p_excoeff);
+
+      if (mpi.Root())
+      {
+         printf("%.5E %.5E %.5E %.5E err\n", t, dt, err_u, err_p);
+         fflush(stdout);
+      }
+   }
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock(vishost, visport);
+   sol_sock.precision(8);
+   sol_sock << "parallel " << mpi.WorldSize() << " " << mpi.WorldRank() << "\n";
+   sol_sock << "solution\n" << *pmesh << *u_ic << std::flush;
+
+   naviersolver.PrintTimingData();
+
+   delete pmesh;
+
+   return 0;
+}

miniapps/fluids/navier_tgv3d.cpp

@@ -0,0 +1,363 @@
+#include "navier_solver.hpp"
+#include <fstream>
+
+using namespace mfem;
+using namespace navier;
+
+struct s_NavierContext
+{
+   int order = 7;
+   double kin_vis = 1.0 / 1600.0;
+   double t_final = 10e-3;
+   double dt = 1e-3;
+   bool pa = false;
+   bool ni = false;
+} ctx;
+
+void vel_tgv(const Vector &x, double t, Vector &u)
+{
+   double xi = x(0);
+   double yi = x(1);
+   double zi = x(2);
+
+   u(0) = sin(xi) * cos(yi) * cos(zi);
+   u(1) = -cos(xi) * sin(yi) * cos(zi);
+   u(2) = 0.0;
+}
+
+class QOI
+{
+public:
+   QOI(ParMesh *pmesh)
+   {
+      H1_FECollection h1fec(1);
+      ParFiniteElementSpace h1fes(pmesh, &h1fec);
+
+      onecoeff.constant = 1.0;
+      mass_lf = new ParLinearForm(&h1fes);
+      mass_lf->AddDomainIntegrator(new DomainLFIntegrator(onecoeff));
+      mass_lf->Assemble();
+
+      ParGridFunction one_gf(&h1fes);
+      one_gf.ProjectCoefficient(onecoeff);
+
+      volume = mass_lf->operator()(one_gf);
+   };
+
+   double ComputeKineticEnergy(ParGridFunction &v)
+   {
+      Vector velx, vely, velz;
+      double integ = 0.0;
+      const FiniteElement *fe;
+      ElementTransformation *T;
+      FiniteElementSpace *fes = v.FESpace();
+
+      for (int i = 0; i < fes->GetNE(); i++)
+      {
+         fe = fes->GetFE(i);
+         double intorder = 2 * fe->GetOrder();
+         const IntegrationRule *ir = &(
+            IntRules.Get(fe->GetGeomType(), intorder));
+
+         v.GetValues(i, *ir, velx, 1);
+         v.GetValues(i, *ir, vely, 2);
+         v.GetValues(i, *ir, velz, 3);
+
+         T = fes->GetElementTransformation(i);
+         for (int j = 0; j < ir->GetNPoints(); j++)
+         {
+            const IntegrationPoint &ip = ir->IntPoint(j);
+            T->SetIntPoint(&ip);
+
+            double vel2 = velx(j) * velx(j) + vely(j) * vely(j)
+                          + velz(j) * velz(j);
+
+            integ += ip.weight * T->Weight() * vel2;
+         }
+      }
+
+      double global_integral = 0.0;
+      MPI_Allreduce(&integ,
+                    &global_integral,
+                    1,
+                    MPI_DOUBLE,
+                    MPI_SUM,
+                    MPI_COMM_WORLD);
+
+      return 0.5 * global_integral / volume;
+   };
+
+   ~QOI() { delete mass_lf; };
+
+private:
+   ConstantCoefficient onecoeff;
+   ParLinearForm *mass_lf;
+   double volume;
+};
+
+template<typename T>
+T sq(T x)
+{
+   return x * x;
+}
+
+void ComputeQCriterion(ParGridFunction &u, ParGridFunction &q)
+{
+   FiniteElementSpace *v_fes = u.FESpace();
+   FiniteElementSpace *fes = q.FESpace();
+
+   // AccumulateAndCountZones
+   Array<int> zones_per_vdof;
+   zones_per_vdof.SetSize(fes->GetVSize());
+   zones_per_vdof = 0;
+
+   q = 0.0;
+
+   // Local interpolation
+   int elndofs;
+   Array<int> v_dofs, dofs;
+   Vector vals;
+   Vector loc_data;
+   int vdim = v_fes->GetVDim();
+   DenseMatrix grad_hat;
+   DenseMatrix dshape;
+   DenseMatrix grad;
+
+   for (int e = 0; e < fes->GetNE(); ++e)
+   {
+      fes->GetElementVDofs(e, dofs);
+      v_fes->GetElementVDofs(e, v_dofs);
+      u.GetSubVector(v_dofs, loc_data);
+      vals.SetSize(dofs.Size());
+      ElementTransformation *tr = fes->GetElementTransformation(e);
+      const FiniteElement *el = fes->GetFE(e);
+      elndofs = el->GetDof();
+      int dim = el->GetDim();
+      dshape.SetSize(elndofs, dim);
+
+      for (int dof = 0; dof < elndofs; ++dof)
+      {
+         // Project
+         const IntegrationPoint &ip = el->GetNodes().IntPoint(dof);
+         tr->SetIntPoint(&ip);
+
+         // Eval
+         // GetVectorGradientHat
+         el->CalcDShape(tr->GetIntPoint(), dshape);
+         grad_hat.SetSize(vdim, dim);
+         DenseMatrix loc_data_mat(loc_data.GetData(), elndofs, vdim);
+         MultAtB(loc_data_mat, dshape, grad_hat);
+
+         const DenseMatrix &Jinv = tr->InverseJacobian();
+         grad.SetSize(grad_hat.Height(), Jinv.Width());
+         Mult(grad_hat, Jinv, grad);
+
+         double q_val = 0.5 * (sq(grad(0, 0)) + sq(grad(1, 1)) + sq(grad(2, 2)))
+                        + grad(0, 1) * grad(1, 0) + grad(0, 2) * grad(2, 0)
+                        + grad(1, 2) * grad(2, 1);
+
+         vals(dof) = q_val;
+      }
+
+      // Accumulate values in all dofs, count the zones.
+      for (int j = 0; j < dofs.Size(); j++)
+      {
+         int ldof = dofs[j];
+         q(ldof) += vals[j];
+         zones_per_vdof[ldof]++;
+      }
+   }
+
+   // Communication
+
+   // Count the zones globally.
+   GroupCommunicator &gcomm = q.ParFESpace()->GroupComm();
+   gcomm.Reduce<int>(zones_per_vdof, GroupCommunicator::Sum);
+   gcomm.Bcast(zones_per_vdof);
+
+   // Accumulate for all vdofs.
+   gcomm.Reduce<double>(q.GetData(), GroupCommunicator::Sum);
+   gcomm.Bcast<double>(q.GetData());
+
+   // Compute means
+   for (int i = 0; i < q.Size(); i++)
+   {
+      const int nz = zones_per_vdof[i];
+      if (nz)
+      {
+         q(i) /= nz;
+      }
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi(argc, argv);
+
+   int ser_ref_levels = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ser_ref_levels,
+                  "-rs",
+                  "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&ctx.order,
+                  "-o",
+                  "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ctx.dt, "-dt", "--time-step", "Time step.");
+   args.AddOption(&ctx.t_final, "-tf", "--final-time", "Final time.");
+   args.AddOption(&ctx.pa,
+                  "-pa",
+                  "--enable-pa",
+                  "-no-pi",
+                  "--disable-pi",
+                  "Enable partial assembly.");
+   args.AddOption(&ctx.ni,
+                  "-ni",
+                  "--enable-ni",
+                  "-no-ni",
+                  "--disable-ni",
+                  "Enable numerical integration rules.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (mpi.Root())
+      {
+         args.PrintUsage(std::cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (mpi.Root())
+   {
+      args.PrintOptions(std::cout);
+   }
+
+   Mesh *orig_mesh = new Mesh("../../data/periodic-cube.mesh");
+   Mesh *mesh = new Mesh(orig_mesh, ser_ref_levels, BasisType::ClosedUniform);
+   delete orig_mesh;
+
+   mesh->EnsureNodes();
+   GridFunction *nodes = mesh->GetNodes();
+   *nodes *= M_PI;
+
+   int nel = mesh->GetNE();
+   if (mpi.Root())
+   {
+      std::cout << "Number of elements: " << nel << std::endl;
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // Create the flow solver.
+   NavierSolver naviersolver(pmesh, ctx.order, ctx.kin_vis);
+   naviersolver.EnablePA(ctx.pa);
+   naviersolver.EnableNI(ctx.ni);
+
+   // Set the initial condition.
+   // This is completely user customizeable.
+   ParGridFunction *u_ic = naviersolver.GetCurrentVelocity();
+   VectorFunctionCoefficient u_excoeff(pmesh->Dimension(), vel_tgv);
+   u_ic->ProjectCoefficient(u_excoeff);
+
+   double t = 0.0;
+   double dt = ctx.dt;
+   double t_final = ctx.t_final;
+   bool last_step = false;
+
+   naviersolver.Setup(dt);
+
+   ParGridFunction *u_gf = naviersolver.GetCurrentVelocity();
+   ParGridFunction *p_gf = naviersolver.GetCurrentPressure();
+
+   ParGridFunction w_gf(*u_gf);
+   ParGridFunction q_gf(*p_gf);
+   naviersolver.ComputeCurl3D(*u_gf, w_gf);
+   ComputeQCriterion(*u_gf, q_gf);
+
+   QOI kin_energy(pmesh);
+
+   VisItDataCollection visit_dc("ins", pmesh);
+   visit_dc.SetPrefixPath("output");
+   visit_dc.SetCycle(0);
+   visit_dc.SetTime(t);
+   visit_dc.RegisterField("velocity", u_gf);
+   visit_dc.RegisterField("pressure", p_gf);
+   visit_dc.RegisterField("vorticity", &w_gf);
+   visit_dc.RegisterField("qcriterion", &q_gf);
+   visit_dc.Save();
+
+   std::ofstream ofs0("output/qcrit.gf");
+   q_gf.Save(ofs0);
+   ofs0.close();
+   std::ofstream ofs1("output/mesh");
+   pmesh->Print(ofs1);
+   ofs1.close();
+
+   double u_inf_loc = u_gf->Normlinf();
+   double p_inf_loc = p_gf->Normlinf();
+   double u_inf = GlobalLpNorm(infinity(), u_inf_loc, MPI_COMM_WORLD);
+   double p_inf = GlobalLpNorm(infinity(), p_inf_loc, MPI_COMM_WORLD);
+   double ke = kin_energy.ComputeKineticEnergy(*u_gf);
+
+   std::string fname = "tgv_out_p_" + std::to_string(ctx.order) + ".txt";
+   FILE *f;
+
+   if (mpi.Root())
+   {
+      int nel1d = std::round(pow(nel, 1.0 / 3.0));
+      int ngridpts = p_gf->ParFESpace()->GlobalVSize();
+      printf("%.5E %.5E %.5E %.5E %.5E\n", t, dt, u_inf, p_inf, ke);
+
+      f = fopen(fname.c_str(), "w");
+      fprintf(f, "3D Taylor Green Vortex\n");
+      fprintf(f, "order = %d\n", ctx.order);
+      fprintf(f, "grid = %d x %d x %d\n", nel1d, nel1d, nel1d);
+      fprintf(f, "dofs per component = %d\n", ngridpts);
+      fprintf(f, "=================================================\n");
+      fprintf(f, "        time                   kinetic energy\n");
+      fprintf(f, "%20.16e     %20.16e\n", t, ke);
+      fflush(f);
+      fflush(stdout);
+   }
+
+   for (int step = 0; !last_step; ++step)
+   {
+      if (t + dt >= t_final - dt / 2)
+      {
+         last_step = true;
+      }
+
+      naviersolver.Step(t, dt, step);
+
+      if ((step + 1) % 100 == 0 || last_step)
+      {
+         naviersolver.ComputeCurl3D(*u_gf, w_gf);
+         ComputeQCriterion(*u_gf, q_gf);
+         visit_dc.SetCycle(step);
+         visit_dc.SetTime(t);
+         visit_dc.Save();
+      }
+
+      double u_inf_loc = u_gf->Normlinf();
+      double p_inf_loc = p_gf->Normlinf();
+      double u_inf = GlobalLpNorm(infinity(), u_inf_loc, MPI_COMM_WORLD);
+      double p_inf = GlobalLpNorm(infinity(), p_inf_loc, MPI_COMM_WORLD);
+      double ke = kin_energy.ComputeKineticEnergy(*u_gf);
+      if (mpi.Root())
+      {
+         printf("%.5E %.5E %.5E %.5E %.5E\n", t, dt, u_inf, p_inf, ke);
+         fprintf(f, "%20.16e     %20.16e\n", t, ke);
+         fflush(f);
+         fflush(stdout);
+      }
+   }
+
+   naviersolver.PrintTimingData();
+
+   delete pmesh;
+
+   return 0;
+}

miniapps/fluids/ortho_solver.cpp

@@ -0,0 +1,41 @@
+#include "ortho_solver.hpp"
+
+using namespace mfem;
+using namespace navier;
+
+OrthoSolver::OrthoSolver() : Solver(0, true) {}
+
+void OrthoSolver::SetOperator(const Operator &op)
+{
+   oper = &op;
+}
+
+void OrthoSolver::Mult(const Vector &b, Vector &x) const
+{
+   // Orthoganlize input.
+   Orthoganalize(b, b_ortho);
+
+   // Apply operator.
+   oper->Mult(b_ortho, x);
+
+   // Orthoganlize output.
+   Orthoganalize(x, x);
+}
+
+void OrthoSolver::Orthoganalize(const Vector &v, Vector &v_ortho) const
+{
+   double loc_sum = v.Sum();
+   double global_sum = 0.0;
+   int loc_size = v.Size();
+   int global_size = 0;
+
+   MPI_Allreduce(&loc_sum, &global_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+   MPI_Allreduce(&loc_size, &global_size, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+   double ratio = global_sum / static_cast<double>(global_size);
+   v_ortho.SetSize(v.Size());
+   for (int i = 0; i < v_ortho.Size(); ++i)
+   {
+      v_ortho(i) = v(i) - ratio;
+   }
+}

miniapps/fluids/ortho_solver.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "mfem.hpp"
+
+namespace mfem
+{
+namespace navier
+{
+class OrthoSolver : public Solver
+{
+public:
+   OrthoSolver();
+
+   virtual void SetOperator(const Operator &op);
+
+   void Mult(const Vector &b, Vector &x) const;
+
+private:
+   const Operator *oper;
+
+   mutable Vector b_ortho;
+
+   void Orthoganalize(const Vector &v, Vector &v_ortho) const;
+};
+} // namespace flow
+} // namespace mfem

miniapps/fluids/poisson.cpp

@@ -0,0 +1,368 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <cmath>
+
+using namespace std;
+using namespace mfem;
+
+class OrthoSolver : public Solver
+{
+public:
+   OrthoSolver();
+
+   virtual void SetOperator(const Operator &op);
+
+   void Mult(const Vector &b, Vector &x) const;
+
+private:
+   const Operator *oper;
+
+   mutable Vector b_ortho;
+
+   void Orthoganalize(const Vector &v, Vector &v_ortho) const;
+};
+
+OrthoSolver::OrthoSolver() : Solver(0, true) {}
+
+void OrthoSolver::SetOperator(const Operator &op)
+{
+   width = op.Width();
+   oper = &op;
+}
+
+void OrthoSolver::Mult(const Vector &b, Vector &x) const
+{
+   // Orthoganlize input.
+   Orthoganalize(b, b_ortho);
+
+   // Apply operator.
+   oper->Mult(b_ortho, x);
+
+   // Orthoganlize output.
+   Orthoganalize(x, x);
+}
+
+void OrthoSolver::Orthoganalize(const Vector &v, Vector &v_ortho) const
+{
+   double loc_sum = v.Sum();
+   double global_sum = 0.0;
+   int loc_size = v.Size();
+   int global_size = 0;
+
+   MPI_Allreduce(&loc_sum, &global_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+   MPI_Allreduce(&loc_size, &global_size, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+   double ratio = global_sum / static_cast<double>(global_size);
+   v_ortho.SetSize(v.Size());
+   for (int i = 0; i < v_ortho.Size(); ++i)
+   {
+      v_ortho(i) = v(i) - ratio;
+   }
+}
+
+double ComputeResidual(Operator &A, Vector &x, Vector &b)
+{
+   Vector r(x.Size());
+   A.Mult(x, r);
+   r -= b;
+   r.HostRead();
+   return GlobalLpNorm(infinity(), r.Normlinf(), MPI_COMM_WORLD);
+}
+
+void OrthoRHS(Vector &v)
+{
+   double loc_sum = v.Sum();
+   double global_sum = 0.0;
+   int loc_size = v.Size();
+   int global_size = 0;
+
+   MPI_Allreduce(&loc_sum, &global_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+   MPI_Allreduce(&loc_size, &global_size, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+   v -= global_sum / static_cast<double>(global_size);
+}
+
+void MkMeanZero(ParGridFunction &v)
+{
+   ConstantCoefficient one{1.0};
+   ParLinearForm mass_lf{v.ParFESpace()};
+   mass_lf.AddDomainIntegrator(new DomainLFIntegrator(one));
+   mass_lf.Assemble();
+
+   ParGridFunction one_gf(v.ParFESpace());
+   one_gf.ProjectCoefficient(one);
+
+   double volume = mass_lf(one_gf);
+   double integ = mass_lf(v);
+
+   v -= integ / volume;
+}
+
+double rhs(const Vector &xpt)
+{
+   int dim = xpt.Size();
+   double x = xpt[0];
+   double y = (dim >= 2) ? xpt[1] : 0.0;
+   double z1 = ((dim >= 3) ? xpt[2] : 0.0) + 1.0;
+   return sin(x)*cos(y)*z1*z1;
+}
+
+int driver(int argc, char *argv[])
+{
+   int num_procs, myid;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   const char *device_config = "cpu";
+   int order = 2;
+   int npatches = 1;
+   int ref_levels = 0;
+   bool visualization = false;
+   bool uniform_ref = false;
+   bool run_amg = true;
+   bool run_as = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Polynomial degree");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&uniform_ref, "-u", "--uniform-refinement", "-no-u",
+                  "--no-uniform-refinement", "Enable uniform refinement");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&npatches, "-n", "--npatches",
+                  "Number of patches to use in additive Schwarz method");
+   args.AddOption(&run_amg, "-amg", "--run-amg", "-no-amg", "--no-run-amg",
+                  "Solve system using hypre AMG");
+   args.AddOption(&run_as, "-as", "--run-as", "-no-as", "--no-run-as",
+                  "Solve system using additive Schwarz");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Device device(device_config);
+   device.Print();
+
+   // Create the serial mesh, and do some uniform refinement if requested.
+   std::unique_ptr<Mesh> mesh_ho(new Mesh(mesh_file, 1, 1));
+   mesh_ho->SetCurvature(order, true, -1, Ordering::byNODES);
+
+   int dim = mesh_ho->Dimension();
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh_ho->UniformRefinement();
+   }
+
+   // Define the parallel mesh
+   ParMesh pmesh_ho(MPI_COMM_WORLD, *mesh_ho);
+   int basis_lor = uniform_ref ? BasisType::ClosedUniform
+                   : BasisType::GaussLobatto;
+   ParMesh pmesh_lor(&pmesh_ho, order, basis_lor);
+   // Output the meshes to files for visualization.
+   // Delete the serial mesh
+   mesh_ho.reset();
+
+   H1_FECollection fec_ho(order, dim);
+   H1_FECollection fec_lor(1, dim);
+   ParFiniteElementSpace fespace_ho(&pmesh_ho, &fec_ho);
+   ParFiniteElementSpace fespace_lor(&pmesh_lor, &fec_lor);
+   ParFiniteElementSpace fespace_coarse(&pmesh_ho, &fec_lor);
+
+   int nel_total = pmesh_ho.ReduceInt(pmesh_ho.GetNE());
+   HYPRE_Int size_ho = fespace_ho.GlobalTrueVSize();
+   HYPRE_Int size_lor = fespace_lor.GlobalTrueVSize();
+   HYPRE_Int size_coarse = fespace_coarse.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of HO elements in mesh: " << nel_total << endl;
+      cout << "Number of HO finite element unknowns:     " << size_ho << endl;
+      cout << "Number of LOR finite element unknowns:    " << size_lor << endl;
+      cout << "Number of coarse finite element unknowns: "
+           << size_coarse << endl;
+   }
+
+   // Determine the list of true (i.e. parallel conforming) essential
+   // boundary dofs. In this example, the boundary conditions are defined
+   // by marking all the boundary attributes from the mesh as essential
+   // (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list, ess_tdof_list_coarse;
+
+   int nbdr;
+   if (pmesh_ho.bdr_attributes.Size() > 0)
+   {
+      nbdr = pmesh_ho.bdr_attributes.Max();
+   }
+   else
+   {
+      nbdr = 0;
+   }
+
+   Array<int> ess_bdr(nbdr);
+
+   // Pure Neumann...
+   // ess_bdr = 0;
+   // ess_bdr = 1;
+   if (nbdr >= 2)
+   {
+      ess_bdr[1] = 0;
+   }
+   //ess_bdr = 1;
+   ess_bdr = 0;
+
+   if (pmesh_ho.bdr_attributes.Size())
+   {
+      fespace_coarse.GetEssentialTrueDofs(ess_bdr, ess_tdof_list_coarse);
+      fespace_ho.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ConstantCoefficient one(1.0);
+   FunctionCoefficient rhs_coeff(rhs);
+
+   HypreParMatrix A0;
+   ParBilinearForm a_coarse(&fespace_coarse);
+   a_coarse.AddDomainIntegrator(new DiffusionIntegrator);
+   a_coarse.Assemble();
+   a_coarse.FormSystemMatrix(ess_tdof_list_coarse, A0);
+
+   ParBilinearForm a_lor(&fespace_lor);
+   HypreParMatrix A_lor;
+   a_lor.AddDomainIntegrator(new DiffusionIntegrator);
+   a_lor.Assemble();
+   a_lor.FormSystemMatrix(ess_tdof_list, A_lor);
+
+   ParLinearForm b(&fespace_ho);
+   b.AddDomainIntegrator(new DomainLFIntegrator(rhs_coeff));
+   b.Assemble();
+
+   b.Randomize(3);
+
+   OrthoRHS(b);
+
+   ParBilinearForm a(&fespace_ho);
+   OperatorHandle A;
+   a.AddDomainIntegrator(new DiffusionIntegrator);
+   a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   a.Assemble();
+
+   ParGridFunction x(&fespace_ho);
+   // Test out inhomogeneous (g=1) Dirichlet conditions
+   // x.ProjectBdrCoefficient(one, ess_bdr);
+   x = 0.0;
+
+   Vector X, B;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+
+   // std::ofstream matout("a_poisson.txt");
+   // A->PrintMatlab(matout);
+   // matout.close();
+   // return(0);
+
+   CGSolver itsolv(MPI_COMM_WORLD);
+
+   itsolv.SetPrintLevel(1);
+   itsolv.SetMaxIter(500);
+   itsolv.SetRelTol(1e-6);
+   itsolv.SetAbsTol(0.0);
+   itsolv.SetOperator(*A);
+
+   // Solve 1. AMG:
+   double amg_elapsed_solv = -1, amg_elapsed_setup = -1, amg_resnorm = -1;
+   // The AMG preconditioner is defined in terms of the LOR matrix
+   // since the goal is to avoid ever forming the high-order system matrix.
+   if (run_amg)
+   {
+      tic_toc.Clear();
+      tic_toc.Start();
+
+      HypreBoomerAMG amg(A_lor);;
+      // HYPRE_BoomerAMGSetAggNumLevels(amg, 0);
+      // HYPRE_BoomerAMGSetRelaxType(amg, 3);
+      // HYPRE_BoomerAMGSetRelaxType(amg, 18);
+
+      // amg.SetPrintLevel(0); // 1
+
+      OrthoSolver orth_amg;
+      orth_amg.SetOperator(amg);
+
+      itsolv.SetPreconditioner(orth_amg);
+      // Force setup of AMG preconditioner. This is a stupid hack but necessary
+      // because MFEM doesn't expose the setup_called member data.
+      amg.Mult(B, X);
+      tic_toc.Stop();
+      amg_elapsed_setup = tic_toc.RealTime();
+      tic_toc.Clear();
+      X = 0.0;
+      tic_toc.Start();
+      itsolv.Mult(B, X);
+      tic_toc.Stop();
+      amg_elapsed_solv = tic_toc.RealTime();
+      amg_resnorm = ComputeResidual(*A, X, B);
+      if (myid == 0) { std::cout << std::endl; }
+   }
+
+   // Solve 2. AS:
+   double as_elapsed_solv = -1, as_elapsed_setup = -1, as_resnorm = -1;
+   if (myid == 0)
+   {
+      std::cout << "AS residual:            " << as_resnorm << '\n';
+      std::cout << "AMG residual:           " << amg_resnorm << '\n';
+      std::cout << '\n';
+      std::cout << "AS elapsed setup time:  " << as_elapsed_setup << '\n';
+      std::cout << "AS elapsed solve time:  " << as_elapsed_solv << '\n';
+      std::cout << '\n';
+      std::cout << "AMG elapsed setup time: " << amg_elapsed_setup << '\n';
+      std::cout << "AMG elapsed solve time: " << amg_elapsed_solv << '\n';
+      std::cout << '\n';
+      std::cout << "AS elapsed total time:  "
+                << as_elapsed_solv + as_elapsed_setup << '\n';
+      std::cout << "AMG elapsed total time: "
+                << amg_elapsed_solv + amg_elapsed_setup << '\n';
+      std::cout << std::endl;
+   }
+
+   // Recover the parallel grid function corresponding to X. This is the local
+   // finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   MkMeanZero(x);
+
+   // Then send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh_ho << x << flush;
+   }
+
+   return 0;
+}
+
+int main(int argc, char **argv)
+{
+   MPI_Init(&argc, &argv);
+   int res = driver(argc, argv);
+   MPI_Finalize();
+   return res;
+}