Adding BlockGSPreconditioner.

This now matches results from old 2D templated code, used for the NLA paper.

config/defaults.cmake

@@ -83,6 +83,11 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 

config/defaults.mk

@@ -155,6 +155,10 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
+ifeq (YES,$(MFEM_USE_CUDA))
+   # This is only necessary when hypre is built with cuda:
+   HYPRE_LIB += -lcusparse -lcurand
+endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK),NONO)

examples/ex1p_helmholtz.cpp

@@ -0,0 +1,503 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double u_exact(const Vector &x);
+double f_exact(const Vector &x);
+
+// #define FORM_DEFINITE
+#define USE_GMRES
+
+#define USE_CSL
+
+#define K2 250.0
+
+int dim;
+double kappa;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/beam-tet.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+
+   kappa = 2.0 * M_PI;
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+
+   {
+      double minsize = pmesh->GetElementSize(0);
+      double maxsize = minsize;
+      for (int i=1; i<pmesh->GetNE(); ++i)
+      {
+         const double size_i = pmesh->GetElementSize(i);
+         minsize = std::min(minsize, size_i);
+         maxsize = std::max(maxsize, size_i);
+      }
+
+      cout << myid << ": Element size range: (" << minsize << ", " << maxsize << ")"
+           << endl;
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+
+   //ConstantCoefficient bcoef(1.0);
+   FunctionCoefficient bcoef(f_exact);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient neg(-K2);
+   ConstantCoefficient pos(K2);
+   b->AddDomainIntegrator(new DomainLFIntegrator(bcoef));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   a->AddDomainIntegrator(new MassIntegrator(neg));
+
+#ifdef FORM_DEFINITE
+   ParBilinearForm *adef = new ParBilinearForm(fespace);
+   adef->AddDomainIntegrator(new DiffusionIntegrator(one));
+   adef->AddDomainIntegrator(new MassIntegrator(pos));
+
+   if (static_cond) { adef->EnableStaticCondensation(); }
+   adef->Assemble();
+
+   ParGridFunction xdef(fespace);
+   xdef = 0.0;
+
+   ParLinearForm *bdef = new ParLinearForm(fespace);
+   bdef->AddDomainIntegrator(new DomainLFIntegrator(bcoef));
+   bdef->Assemble();
+
+   HypreParMatrix Adef;
+   Vector Bdef, Xdef;
+   adef->FormLinearSystem(ess_tdof_list, xdef, *bdef, Adef, Xdef, Bdef);
+#endif
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   //A.Print("helmholtz");
+
+   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   //     preconditioner from hypre.
+#ifdef FORM_DEFINITE
+   HypreSolver *amg = new HypreBoomerAMG(Adef);
+#else
+   HypreSolver *amg = new HypreBoomerAMG(A);
+#endif
+
+   const bool fullDirect = true;
+
+   if (fullDirect)
+   {
+#ifdef USE_CSL
+      Vector Bdef, Xdef;
+
+      ParBilinearForm *Mform =  new ParBilinearForm(fespace);
+      Mform->AddDomainIntegrator(new MassIntegrator(pos));
+      Mform->Assemble();
+
+      HypreParMatrix Mmat, Smat, Mcopy;
+      Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
+      Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
+                              Bdef);  // There must be a better way than creating two identical matrices.
+
+      ParBilinearForm *Sform =  new ParBilinearForm(fespace);
+      Sform->AddDomainIntegrator(new DiffusionIntegrator(one));
+      Sform->Assemble();
+
+      Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
+
+      const double beta1 = 1.0;
+      const double beta2 = 1.0;
+
+      Mmat *= -beta1;
+
+      HypreParMatrix * cslRe = ParAdd(&Smat, &Mmat);
+
+      Mcopy *= beta2;
+
+      ComplexHypreParMatrix chpm(cslRe, &Mcopy, false, false);
+
+      HypreParMatrix *cSysMat = chpm.GetSystemMatrix();
+
+      Array<int> block_trueOffsets(3); // number of variables + 1
+      block_trueOffsets[0] = 0;
+      block_trueOffsets[1] = fespace->TrueVSize();
+      block_trueOffsets[2] = fespace->TrueVSize();
+      block_trueOffsets.PartialSum();
+
+      // Note that B is of true size.
+      BlockVector trueY(block_trueOffsets), trueX(block_trueOffsets),
+                  trueRhs(block_trueOffsets);
+
+      trueRhs.GetBlock(0) = B;
+      trueRhs.GetBlock(1) = 0.0;
+
+      Operator * Arow = new STRUMPACKRowLocMatrix(*cSysMat);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      //Solver * precond = strumpack;
+
+      // strumpack->Mult(B, X);
+
+      BlockOperator blockDiagA(block_trueOffsets);
+      for (int i=0; i<2; ++i)
+      {
+         blockDiagA.SetDiagonalBlock(i, &A);
+      }
+
+      ProductOperator prod(&blockDiagA, strumpack, false, false);
+      //GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
+      BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
+
+      gmres->SetOperator(prod);
+      gmres->SetRelTol(1e-8);
+      gmres->SetMaxIter(10000);
+      gmres->SetPrintLevel(1);
+
+      gmres->Mult(trueRhs, trueY);
+      strumpack->Mult(trueY, trueX);
+
+      X = trueX.GetBlock(0);
+      double xim2 = trueX.GetBlock(1).Norml2();
+      xim2 *= xim2;
+      double sumxim2 = 0.0;
+
+      MPI_Allreduce(&xim2, &sumxim2, 1, MPI_DOUBLE, MPI_SUM, fespace->GetComm());
+
+      if (myid == 0)
+      {
+         cout << myid << ": norm of Xim " << trueX.GetBlock(1).Norml2() << ", global " <<
+              sqrt(sumxim2) << endl;
+      }
+
+      delete gmres;
+      delete strumpack;
+      delete Arow;
+#else
+      Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      //Solver * precond = strumpack;
+
+      strumpack->Mult(B, X);
+
+      delete strumpack;
+      delete Arow;
+#endif
+   }
+   else
+   {
+#ifdef USE_GMRES
+      HypreGMRES *gmres = new HypreGMRES(A);
+      gmres->SetTol(1e-12);
+      gmres->SetMaxIter(1000);
+      gmres->SetPrintLevel(10);
+      gmres->SetPreconditioner(*amg);
+      gmres->Mult(B, X);
+      delete gmres;
+#else
+      HyprePCG *pcg = new HyprePCG(A);
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(100);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*amg);
+      pcg->Mult(B, X);
+#endif
+   }
+
+   /*
+   HYPRE_ParCSRMatrix* amgP = amg->Get_Restriction();
+   HypreParMatrix P0(amgP[0], false);
+   HypreParMatrix P1(amgP[1], false);
+   HypreParMatrix P2(amgP[2], false);
+   //HypreParMatrix P3(amgP[3], false);
+
+   P0.Print("P0");
+   */
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // Compute and print the L^2 norm of the error.
+   {
+      FunctionCoefficient uex(u_exact);
+
+      double err = x.ComputeL2Error(uex);
+      double xnrm = x.ComputeL2Error(zero);
+      ParGridFunction zerogf(fespace);
+      zerogf = 0.0;
+      double normE = zerogf.ComputeL2Error(uex);
+      if (myid == 0)
+      {
+         cout << "|| E_h - E ||_{L^2} = " << err << endl;
+         cout << "|| E_h ||_{L^2} = " << xnrm << endl;
+         cout << "|| E ||_{L^2} = " << normE << endl;
+      }
+   }
+
+   // 14. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 16. Free the used memory.
+   //delete pcg;
+   delete amg;
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+double u_exact(const Vector & x)
+{
+   double xi(x(0));
+   double yi(x(1));
+   double zi(1.0);
+
+   if (x.Size() == 3)
+   {
+      zi = x(2);
+   }
+
+   return sin(kappa*xi)*sin(kappa*yi)*sin(kappa*zi);
+}
+
+double f_exact(const Vector &x)
+{
+   double xi(x(0));
+   double yi(x(1));
+   double zi(1.0);
+
+   if (x.Size() == 3)
+   {
+      zi = x(2);
+   }
+
+   const double s = 1.0;
+
+   return ((3.0*kappa*kappa) - (s*K2)) * sin(kappa*xi)*sin(kappa*yi)*sin(
+             kappa*zi) / s;
+}

examples/ex3p.cpp

@@ -158,9 +158,9 @@ int main(int argc, char *argv[])
    //    by marking all the boundary attributes from the mesh as essential
    //    (Dirichlet) and converting them to a list of true dofs.
    Array<int> ess_tdof_list;
+   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
    if (pmesh->bdr_attributes.Size())
    {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
       ess_bdr = 1;
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
@@ -207,18 +207,36 @@ int main(int argc, char *argv[])
    // 13. Solve the system AX=B using PCG with the AMS preconditioner from hypre
    //     (in the full assembly case) or CG with Jacobi preconditioner (in the
    //     partial assembly case).
-
-   if (pa) // Jacobi preconditioning in partial assembly mode
+   if (pa) // matrix-free auxiliary space solver with PA
    {
-      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);
+      StopWatch sw;
+      sw.Clear();
+      sw.Start();
+
+      MatrixFreeAMS *ams = new MatrixFreeAMS(*a, *A, *fespace, muinv, sigma, NULL,
+                                             ess_bdr); //, 2, 2);
+
+      //OperatorJacobiSmoother jacobi(*a, ess_tdof_list);
 
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(1000);
       cg.SetPrintLevel(1);
       cg.SetOperator(*A);
-      cg.SetPreconditioner(Jacobi);
+      cg.SetPreconditioner(*ams);
+      //cg.SetPreconditioner(jacobi);
+      //cg.iterative_mode = false;
+      sw.Stop();
+      cout << myid << ": MatrixFreeAMS-CG setup time " << sw.RealTime() << endl;
+
+      sw.Clear();
+      sw.Start();
+
       cg.Mult(B, X);
+
+      sw.Stop();
+      cout << myid << ": MatrixFreeAMS-CG solve time " << sw.RealTime() << endl;
+      ams->PrintTimings(myid);
    }
    else
    {

examples/ex3p_airy.cpp

@@ -0,0 +1,425 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+#define TEST_AIRY
+
+#ifdef TEST_AIRY
+#include "gsl_sf_airy.h"
+
+#define XSHIFT 0.0 // 0.25
+#endif
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+int dim;
+
+//#define K2_VALUE 11.9
+//#define K2_VALUE 2.1
+//#define K2_VALUE 1500.9
+#define K2_VALUE 10981.41589009910  // 104.792251097584^2 or 5 GHz
+
+void test_Airy_epsilon(const Vector &x, Vector &e)
+{
+   e(0) = 1.0;
+   e(1) = 1.0;
+   e(2) = (4.0 * (x(0) + XSHIFT)) - 1.0;
+
+   e *= -K2_VALUE;
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   //const char *mesh_file = "../data/beam-tet.mesh";
+   const char *mesh_file = "../data/inline-tetHalf.mesh";
+   //const char *mesh_file = "inline-tetSlab.mesh";
+   //const char *mesh_file = "../data/inline-hexHalf.mesh";
+   //const char *mesh_file = "../data/inline-tet.mesh";
+   int order = 2;
+   bool static_cond = false;
+   bool visualization = 1;
+#ifdef MFEM_USE_STRUMPACK
+   bool use_strumpack = false;
+#endif
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use STRUMPACK's double complex linear solver.");
+#endif
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+      cout << "Using k2 " << K2_VALUE << endl;
+   }
+   kappa = freq * M_PI;
+
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
+      ref_levels = 0;
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   long globalNE = pmesh->GetGlobalNE();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+      cout << "Number of mesh elements: " << globalNE << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   VectorFunctionCoefficient f(sdim, f_exact);
+   ParLinearForm *b = new ParLinearForm(fespace);
+   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   ParGridFunction solproj(fespace);
+   VectorFunctionCoefficient E(sdim, E_exact);
+   x.ProjectCoefficient(E);
+
+   solproj = x;
+
+   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+   //     mass domain integrators.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(-K2_VALUE);
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+#ifdef TEST_AIRY
+   VectorFunctionCoefficient epsilon(3, test_Airy_epsilon);
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(epsilon));
+#else
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+#endif
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+
+#ifdef MFEM_USE_STRUMPACK
+   if (use_strumpack)
+   {
+      Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      //Solver * precond = strumpack;
+
+      strumpack->Mult(B, X);
+      //X = 0.0;
+
+      {
+         // Check residual
+         Vector res(X.Size());
+         Vector ssol(X.Size());
+         ssol = X;
+
+         const double Bnrm = B.Norml2();
+         const double Bnrm2 = Bnrm*Bnrm;
+
+         A.Mult(ssol, res);
+         res -= B;
+
+         const double Rnrm = res.Norml2();
+         const double Rnrm2 = Rnrm*Rnrm;
+
+         double sumBnrm2 = 0.0;
+         double sumRnrm2 = 0.0;
+         MPI_Allreduce(&Bnrm2, &sumBnrm2, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+         MPI_Allreduce(&Rnrm2, &sumRnrm2, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+         if (myid == 0)
+         {
+            cout << myid << ": STRUMPACK residual norm " << sqrt(sumRnrm2) << ", B norm " <<
+                 sqrt(sumBnrm2) << endl;
+         }
+      }
+
+      delete strumpack;
+      delete Arow;
+   }
+   else
+#endif
+   {
+      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+      //     preconditioner from hypre.
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      HypreSolver *ams = new HypreAMS(A, prec_fespace);
+      HyprePCG *pcg = new HyprePCG(A);
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*ams);
+      pcg->Mult(B, X);
+
+      delete pcg;
+      delete ams;
+   }
+
+   chrono.Stop();
+   if (myid == 0)
+   {
+      cout << "Solver time " << chrono.RealTime() << endl;
+   }
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 14. Compute and print the L^2 norm of the error.
+   {
+      double err = x.ComputeL2Error(E);
+      ParGridFunction x0(fespace);
+      x0 = 0.0;
+
+      double solnrm = x0.ComputeL2Error(E);
+
+      if (myid == 0)
+      {
+         cout << "\n|| E_h - E ||_{L^2} = " << err << ", relative error " << err / solnrm
+              << endl;
+      }
+   }
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+      //solproj.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete a;
+   delete sigma;
+   delete muinv;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim == 3)
+   {
+#ifdef TEST_AIRY
+      const double y = (4.0 * (x(0) + XSHIFT)) - 1.0;
+      const double k = sqrt(K2_VALUE);
+      const double beta = pow(0.25 * k, 2.0/3.0);
+
+      E(0) = 0.0;
+      E(1) = 0.0;
+      E(2) = gsl_sf_airy_Ai(-beta * y, GSL_PREC_DOUBLE);
+#else
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+#endif
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+#ifdef TEST_AIRY
+      f = 0.0;
+#else
+      f(0) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(0));
+#endif
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/ex3p_cfosls.cpp

@@ -0,0 +1,848 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "gsl_sf_airy.h"
+
+#include "multigrid.hpp"
+
+using namespace std;
+using namespace mfem;
+
+
+//#define DIRECT_SOLVER
+
+// Define exact solution
+void E_exact(const Vector & x, Vector & E);
+void H_exact(const Vector & x, Vector & H);
+void f_exact_H(const Vector & x, Vector & f_H);
+void get_maxwell_solution(const Vector & x, double E[], double curlE[],
+                          double curl2E[]);
+void epsilon_func(const Vector &x, Vector &M);
+void epsilon2_func(const Vector &x, Vector &M);
+void epsilon_func_mat(const Vector &x, DenseMatrix &M);
+
+int dim;
+double omega;
+int sol = 3;
+
+
+class FOSLSSolver : public Solver
+{
+public:
+   FOSLSSolver(ParFiniteElementSpace *fespace_, std::vector<HypreParMatrix*>& P,
+               const int myid_)
+      : Solver(2 * fespace_->GetTrueVSize()), M_inv(MPI_COMM_WORLD),
+        fespace(fespace_),
+        n(fespace_->GetTrueVSize()), nfull(fespace_->GetVSize()), LSpcg(MPI_COMM_WORLD),
+        myid(myid_)
+   {
+      z.SetSize(n);
+      Minv_x.SetSize(n);
+
+      ParMesh *pmesh = fespace->GetParMesh();
+      int dim = pmesh->Dimension();
+      int sdim = pmesh->SpaceDimension();
+
+      VectorFunctionCoefficient epsilon(dim, epsilon_func);
+      VectorFunctionCoefficient epsilonT(epsilon);  // transpose of epsilon
+      VectorFunctionCoefficient epsilon2(dim,epsilon2_func);
+      ConstantCoefficient pos(omega);
+      ConstantCoefficient sigma(omega*omega);
+      ScalarVectorProductCoefficient coeff(pos,epsilon);
+      ScalarVectorProductCoefficient coeffT(pos,epsilonT);
+      ScalarVectorProductCoefficient coeff2(sigma,epsilon2);
+
+      bM = new ParBilinearForm(fespace);
+      bM->AddDomainIntegrator(new VectorFEMassIntegrator());
+      bM->Assemble();
+      bM->Finalize();
+
+      bM_eps = new ParBilinearForm(fespace);
+      bM_eps->AddDomainIntegrator(new VectorFEMassIntegrator(epsilonT));
+      bM_eps->Assemble();
+      bM_eps->Finalize();
+
+      bM_curl = new ParMixedBilinearForm(fespace,fespace);
+      bM_curl->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator());
+      bM_curl->Assemble();
+      bM_curl->Finalize();
+
+      Array<int> ess_tdof_list;  // empty
+
+      bM->FormSystemMatrix(ess_tdof_list, M);
+      bM_eps->FormSystemMatrix(ess_tdof_list, M_eps);
+      //bM_curl->FormColSystemMatrix(ess_tdof_list, M_curl);
+      {
+         OperatorPtr M_curl_ptr;
+         bM_curl->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, M_curl_ptr);
+         M_curl = M_curl_ptr.As<HypreParMatrix>();
+      }
+
+      M_inv.SetAbsTol(1.0e-12);
+      M_inv.SetRelTol(1.0e-12);
+      M_inv.SetMaxIter(100);
+      M_inv.SetOperator(M);
+      M_inv.SetPrintLevel(0);
+
+      block_trueOffsets.SetSize(5);
+
+      block_trueOffsets[0] = 0;
+      block_trueOffsets[1] = n;
+      block_trueOffsets[2] = n;
+      block_trueOffsets[3] = n;
+      block_trueOffsets[4] = n;
+      block_trueOffsets.PartialSum();
+
+      trueRhs = new BlockVector(block_trueOffsets);
+      trueSol = new BlockVector(block_trueOffsets);
+
+      //    _           _    _  _       _  _
+      //   |             |  |    |     |    |
+      //   |  A00   A01  |  | E  |     |F_E |
+      //   |             |  |    |  =  |    |
+      //   |  A10   A11  |  | H  |     |F_G |
+      //   |_           _|  |_  _|     |_  _|
+      //
+      // A00 = (curl E, curl F) + \omega^2 (E,F)
+      // A01 = - \omega *( (curl E, F) + (E,curl F)
+      // A10 = - \omega *( (curl H, G) + (H,curl G)
+      // A11 = (curl H, curl G) + \omega^2 (H,G)
+
+      ParBilinearForm *a_EE = new ParBilinearForm(fespace);
+      a_EE->AddDomainIntegrator(new CurlCurlIntegrator());
+      a_EE->AddDomainIntegrator(new VectorFEMassIntegrator(coeff2));
+      a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator());
+      a_EE->Assemble();
+      a_EE->Finalize();
+      HypreParMatrix *A_EE = new HypreParMatrix;
+      a_EE->FormSystemMatrix(ess_tdof_list, *A_EE);
+
+      ParBilinearForm *a_HH = new ParBilinearForm(fespace);
+      a_HH->AddDomainIntegrator(new CurlCurlIntegrator());
+      a_HH->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
+      a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator());
+      a_HH->Assemble();
+      a_HH->Finalize();
+      HypreParMatrix *A_HH = new HypreParMatrix;
+      a_HH->FormSystemMatrix(ess_tdof_list, *A_HH);
+
+      ParBilinearForm *a_tang = new ParBilinearForm(fespace);
+      a_tang->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(1.0));
+      a_tang->Assemble();
+      a_tang->Finalize();
+      OperatorHandle A_tang_ptr;
+      a_tang->FormSystemMatrix(ess_tdof_list, A_tang_ptr);
+      HypreParMatrix *A_tang = A_tang_ptr.As<HypreParMatrix>();
+
+      // (k curl u, eps v) + (k u, curl v)
+      ParMixedBilinearForm *a_mix1 = new ParMixedBilinearForm(fespace,fespace);
+      a_mix1->AddDomainIntegrator(new MixedVectorCurlIntegrator(coeffT));
+      a_mix1->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(pos));
+      a_mix1->Assemble();
+      a_mix1->Finalize();
+      HypreParMatrix *A_mix1 = NULL; // new HypreParMatrix;
+      //a_mix1->FormColSystemMatrix(ess_tdof_list, *A_mix1);
+      {
+         OperatorPtr A_mix1_ptr;
+         a_mix1->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, A_mix1_ptr);
+         A_mix1 = A_mix1_ptr.As<HypreParMatrix>();
+      }
+
+      // (k curl u, v) + (k eps u, curl v)
+      ParMixedBilinearForm *a_mix2 = new ParMixedBilinearForm(fespace,fespace);
+      a_mix2->AddDomainIntegrator(new MixedVectorCurlIntegrator(pos));
+      a_mix2->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(coeff));
+      a_mix2->Assemble();
+      a_mix2->Finalize();
+      HypreParMatrix *A_mix2 = NULL; // new HypreParMatrix;
+      //a_mix2->FormColSystemMatrix(ess_tdof_list, *A_mix2);
+      {
+         OperatorPtr A_mix2_ptr;
+         a_mix2->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, A_mix2_ptr);
+         A_mix2 = A_mix2_ptr.As<HypreParMatrix>();
+      }
+
+      BlockOperator *LS_Maxwellop = new BlockOperator(block_trueOffsets);
+      const int numBlocks = 4;
+
+      LS_Maxwellop->SetBlock(0, 0, A_EE);
+      LS_Maxwellop->SetBlock(1, 0, A_mix2, -1.0); // no bc
+      LS_Maxwellop->SetBlock(3, 0, A_tang, -1.0);
+      LS_Maxwellop->SetBlock(0, 1, A_mix1, -1.0);  // no bc
+      LS_Maxwellop->SetBlock(1, 1, A_HH);
+      LS_Maxwellop->SetBlock(2, 1, A_tang, -1.0);  // other rotation
+      LS_Maxwellop->SetBlock(1, 2, A_tang);
+      LS_Maxwellop->SetBlock(2, 2, A_EE);
+      LS_Maxwellop->SetBlock(3, 2, A_mix2, -1.0);  // no bc
+      LS_Maxwellop->SetBlock(0, 3, A_tang);  // other rotation
+      LS_Maxwellop->SetBlock(2, 3, A_mix1, -1.0);  // no bc
+      LS_Maxwellop->SetBlock(3, 3, A_HH);
+
+      // Set up the preconditioner
+      Array2D<HypreParMatrix*> blockA(numBlocks, numBlocks);
+      Array2D<double> blockAcoef(numBlocks, numBlocks);
+
+      for (int i=0; i<numBlocks; ++i)
+      {
+         for (int j=0; j<numBlocks; ++j)
+         {
+            if (LS_Maxwellop->IsZeroBlock(i,j) == 0)
+            {
+               blockA(i,j) = static_cast<HypreParMatrix *>(&LS_Maxwellop->GetBlock(i,j));
+               blockAcoef(i,j) = LS_Maxwellop->GetBlockCoef(i,j);
+            }
+            else
+            {
+               blockA(i,j) = NULL;
+               blockAcoef(i,j) = 1.0;
+            }
+         }
+      }
+
+      LSpcg.SetAbsTol(1.0e-12);
+      LSpcg.SetRelTol(1.0e-8);
+      LSpcg.SetMaxIter(2000);
+      LSpcg.SetOperator(*LS_Maxwellop);
+      LSpcg.SetPrintLevel(1);
+
+      BlockMGSolver * precMG = NULL;
+
+#ifdef DIRECT_SOLVER
+      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
+      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
+      Array2D<SparseMatrix*> Asp;
+      Asp.SetSize(numBlocks,numBlocks);
+
+      {
+         int nprocs, rank;
+         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+         MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+
+         std::vector<int> allnumrows(nprocs);
+         const int blockNumRows = n;
+         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT,
+                       MPI_COMM_WORLD);
+
+         for (int b=0; b<numBlocks; ++b)
+         {
+            blockProcOffsets[b].resize(nprocs);
+            all_block_num_loc_rows[b].resize(nprocs);
+
+            for (int j=0; j<numBlocks; ++j)
+            {
+               Asp(b,j) = NULL;
+            }
+         }
+
+         blockProcOffsets[0][0] = 0;
+         for (int i=0; i<nprocs-1; ++i)
+         {
+            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
+         }
+
+         for (int i=0; i<nprocs; ++i)
+         {
+            for (int b=0; b<numBlocks; ++b)
+            {
+               all_block_num_loc_rows[b][i] = allnumrows[i];
+            }
+
+            for (int b=1; b<numBlocks; ++b)
+            {
+               blockProcOffsets[b][i] = blockProcOffsets[0][i];
+            }
+         }
+      }
+
+      LSH = CreateHypreParMatrixFromBlocks(MPI_COMM_WORLD, block_trueOffsets, blockA,
+                                           Asp,
+                                           blockAcoef, blockProcOffsets, all_block_num_loc_rows);
+
+      invLSH = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*LSH), MPI_COMM_WORLD);
+#else
+      precMG = new BlockMGSolver(LS_Maxwellop->Height(), LS_Maxwellop->Width(),
+                                 blockA, blockAcoef, P);
+      precMG->SetTheta(0.5);
+      LSpcg.SetPreconditioner(*precMG);
+#endif
+   }
+
+   void SetOperator(const Operator &op) { }
+
+   void Mult(const Vector &x, Vector &y) const
+   {
+      // Solve (curl E, curl v) - k^2 (eps E, v) + ik <pi(u), pi(v)> = (x, v), with no BC,
+      // where x is complex, using FOSLS. This is the Galerkin discretization of
+      // curl curl u - k^2 eps u = x, with ik n x u x n - n x curl u = 0 on the boundary.
+
+      MFEM_VERIFY(x.Size() == 2*n, "");
+
+      (*trueRhs) = 0.0;
+
+      for (int i=0; i<n; ++i)
+      {
+         z[i] = x[i];   // Set z = x_Re
+      }
+
+      M_inv.Mult(z, Minv_x);
+      M_eps.Mult(Minv_x, z);
+
+      trueRhs->GetBlock(0) -= z;
+
+      M_curl->Mult(Minv_x, z);
+      z *= 1.0 / omega;
+
+      trueRhs->GetBlock(1) = z;
+
+      for (int i=0; i<n; ++i)
+      {
+         z[i] = x[n + i];   // Set z = x_Im
+      }
+
+      M_inv.Mult(z, Minv_x);
+      M_eps.Mult(Minv_x, z);
+
+      trueRhs->GetBlock(2) -= z;
+
+      M_curl->Mult(Minv_x, z);
+      z *= 1.0 / omega;
+
+      trueRhs->GetBlock(3) += z;
+
+#ifdef DIRECT_SOLVER
+      invLSH->Mult(*trueRhs, *trueSol);
+#else
+      LSpcg.Mult(*trueRhs, *trueSol);
+#endif
+
+      for (int i=0; i<n; ++i)
+      {
+         y[i] = trueSol->GetBlock(0)[i];   // Set y_Re = E_Re
+      }
+
+      for (int i=0; i<n; ++i)
+      {
+         y[n + i] = trueSol->GetBlock(2)[i];   // Set y_Im = E_Im
+      }
+   }
+
+   void ComplexFOSLSTest()
+   {
+      Array<int> block_offsets;
+
+      block_offsets.SetSize(5);
+
+      block_offsets[0] = 0;
+      block_offsets[1] = nfull;
+      block_offsets[2] = nfull;
+      block_offsets[3] = nfull;
+      block_offsets[4] = nfull;
+      block_offsets.PartialSum();
+
+      BlockVector rhs(block_offsets);
+      BlockVector rhsIm(block_offsets);
+
+      rhs = 0.0;
+      rhsIm = 0.0;
+
+      const double ci = 0.0; // 3.3;
+
+      // Exact complex solution: E = Er + i Ei = Epw + ci i Epw, where Epw is E_exact.
+
+      // Set up the linear form with the real part Fr only.
+      const int sdim = 3;
+      VectorFunctionCoefficient Eex(sdim, E_exact);
+      VectorFunctionCoefficient Hex(sdim, H_exact);
+      ConstantCoefficient negOne(-1.0);
+      VectorFunctionCoefficient mEex(sdim, E_exact, &negOne);
+      VectorFunctionCoefficient mHex(sdim, H_exact, &negOne);
+
+      ConstantCoefficient neg(-omega);
+      ConstantCoefficient pos(omega);
+      VectorFunctionCoefficient f_H(3,f_exact_H);  // f / omega
+      ScalarVectorProductCoefficient sf_H(neg,f_H);
+      ScalarVectorProductCoefficient spf_H(pos,f_H);
+      ScalarVectorProductCoefficient mf_H(negOne,f_H);
+
+      VectorFunctionCoefficient epsilon(3, epsilon_func);
+      VectorFunctionCoefficient epsilonT(3, epsilon_func);  // transpose of epsilon
+      MatrixFunctionCoefficient epsilonTmat(3,
+                                            epsilon_func_mat);  // transpose of epsilon
+
+      MatVecCoefficient epsT_spf_H(epsilonTmat, spf_H);
+      MatVecCoefficient epsT_sf_H(epsilonTmat, sf_H);
+
+      ParLinearForm *b_E = new ParLinearForm;
+      b_E->Update(fespace, rhs.GetBlock(0), 0);
+      b_E->AddDomainIntegrator(new VectorFEDomainLFIntegrator(
+                                  epsT_sf_H)); // (k^{-1} Fr, -k eps Qr)
+      b_E->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                    Eex));  // <n x E_Re x n, n x Q_Re x n>
+      b_E->Assemble();
+
+      ParLinearForm *b_H = new ParLinearForm;
+      b_H->Update(fespace, rhs.GetBlock(1), 0);
+      b_H->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(
+                                  f_H));  // (k^{-1} Fr, curl Rr)
+      b_H->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                    Hex));  // <n x H_Re, n x R_Re>
+      b_H->Assemble();
+
+      ParLinearForm *b_E_Im = new ParLinearForm;
+      b_E_Im->Update(fespace, rhs.GetBlock(2), 0);
+      b_E_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                       mHex));  // <n x H_Re, n x Q_Im x n>
+      b_E_Im->Assemble();
+
+      ParLinearForm *b_H_Im = new ParLinearForm;
+      b_H_Im->Update(fespace, rhs.GetBlock(3), 0);
+      b_H_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                       mEex));  // -<n x E_Re x n, n x R_Im>
+      b_H_Im->Assemble();
+
+      // Add the imaginary part Fi.
+
+      ParLinearForm *b_Ei = new ParLinearForm;
+      b_Ei->Update(fespace, rhsIm.GetBlock(0), 0);
+      b_Ei->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                     Hex));  // -<n x H_Im, n x Q_Re x n>
+      b_Ei->Assemble();
+
+      ParLinearForm *b_Hi = new ParLinearForm;
+      b_Hi->Update(fespace, rhsIm.GetBlock(1), 0);
+      b_Hi->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                     Eex));  // <n x E_Im x n, n x R_Re>
+      b_Hi->Assemble();
+
+      ParLinearForm *b_Ei_Im = new ParLinearForm;
+      b_Ei_Im->Update(fespace, rhsIm.GetBlock(2), 0);
+      b_Ei_Im->AddDomainIntegrator(new VectorFEDomainLFIntegrator(
+                                      epsT_sf_H)); // -(k^{-1} Fi, k eps Qi)
+      b_Ei_Im->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                        Eex));  // <n x E_Im x n, n x Q_Im x n>
+      b_Ei_Im->Assemble();
+
+      ParLinearForm *b_Hi_Im = new ParLinearForm;
+      b_Hi_Im->Update(fespace, rhsIm.GetBlock(3), 0);
+      b_Hi_Im->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(
+                                      f_H)); // (k^{-1} Fi, curl Ri)
+      b_Hi_Im->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                        Hex));  // <n x H_Im, n x R_Im>
+      b_Hi_Im->Assemble();
+
+      rhsIm *= ci;
+      rhs += rhsIm;
+
+      for (int i=0; i<4; ++i)
+      {
+         fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(i),
+                                                         trueRhs->GetBlock(i));
+      }
+
+#ifdef DIRECT_SOLVER
+      invLSH->Mult(*trueRhs, *trueSol);
+#else
+      LSpcg.Mult(*trueRhs, *trueSol);
+#endif
+
+      // Check error
+      ParGridFunction E_gf(fespace);
+      int order = 2;
+      int order_quad = std::max(2, 2*order+1);
+      const IntegrationRule *irs[Geometry::NumGeom];
+      for (int i=0; i < Geometry::NumGeom; ++i)
+      {
+         irs[i] = &(IntRules.Get(i, order_quad));
+      }
+
+      ParMesh *pmesh = fespace->GetParMesh();
+
+      // Check error of real part
+
+      E_gf.SetFromTrueDofs(trueSol->GetBlock(0));
+      double Error_E = E_gf.ComputeL2Error(Eex, irs);
+      double norm_E = ComputeGlobalLpNorm(2, Eex, *pmesh, irs);
+
+      cout << myid << ": real error " << Error_E << " relative to " << norm_E << endl;
+
+      {
+         ostringstream mesh_name, sol_name;
+         mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+         sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+         ofstream mesh_ofs(mesh_name.str().c_str());
+         mesh_ofs.precision(8);
+         pmesh->Print(mesh_ofs);
+
+         ofstream sol_ofs(sol_name.str().c_str());
+         sol_ofs.precision(8);
+         E_gf.Save(sol_ofs);
+      }
+
+      // Check error of imaginary part
+      E_gf.SetFromTrueDofs(trueSol->GetBlock(2));
+      const double iml2 = E_gf.Norml2();
+      E_gf *= ci == 0.0 ? 0.0 : (1.0 / ci);
+      Error_E = E_gf.ComputeL2Error(Eex, irs);
+
+      cout << myid << ": imag error " << Error_E << " relative to " << norm_E <<
+           ", l2 norm " << iml2 << endl;
+   }
+
+private:
+
+   BlockVector *trueRhs, *trueSol;
+
+   Array<int> block_trueOffsets;
+
+   ParBilinearForm *bM, *bM_eps;
+   ParMixedBilinearForm *bM_curl;
+
+   HypreParMatrix M, M_eps;
+   HypreParMatrix *M_curl;
+
+   CGSolver M_inv;
+
+   const int n;
+   const int nfull;
+   const int myid;
+
+   mutable Vector z, Minv_x;
+
+   CGSolver LSpcg;
+
+   STRUMPACKSolver *invLSH;
+   HypreParMatrix *LSH;
+
+   ParFiniteElementSpace *fespace;
+};
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Initialize MPI
+   int num_procs, myid;
+   MPI_Init(&argc, &argv); // Initialize MPI
+   MPI_Comm_size(MPI_COMM_WORLD,
+                 &num_procs); //total number of processors available
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid); // Determine process identifier
+   // 1. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../data/inline-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   // visualization flag
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 1.0;
+   // number of mg levels
+   int ref_levels = 1;
+   // number of initial ref
+   int initref = 1;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&ref_levels, "-ref", "--ref_levels",
+                  "Number of Refinements.");
+   args.AddOption(&initref, "-initref", "--initref",
+                  "Number of initial refinements.");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - "
+                  " 1:sinusoidal, 2: point source, 3: plane wave");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Angular frequency
+   //omega = 2.0*k*M_PI;
+   omega = k;
+
+   // 2. Read the mesh from the given mesh file.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   if (sol == 4)
+   {
+      mesh->EnsureNodes();
+      GridFunction *nodes = mesh->GetNodes();
+      (*nodes) *= 0.5;
+   }
+
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < initref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 4. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   std::vector<ParFiniteElementSpace * > fespaces(ref_levels+1);
+   std::vector<ParMesh * > ParMeshes(ref_levels+1);
+   std::vector<HypreParMatrix*> P(ref_levels);
+
+   for (int i = 0; i < ref_levels; i++)
+   {
+      ParMeshes[i] =new ParMesh(*pmesh);
+      fespaces[i] = new ParFiniteElementSpace(*fespace, *ParMeshes[i]);
+      pmesh->UniformRefinement();
+      // Update fespace
+      fespace->Update();
+      OperatorHandle Tr(Operator::Hypre_ParCSR);
+      fespace->GetTrueTransferOperator(*fespaces[i], Tr);
+      Tr.SetOperatorOwner(false);
+      Tr.Get(P[i]);
+   }
+   fespaces[ref_levels] = new ParFiniteElementSpace(*fespace);
+
+   FOSLSSolver fosls(fespace, P, myid);
+
+   fosls.ComplexFOSLSTest();
+
+   for (auto p: ParMeshes) { delete p; }
+   for (auto p: fespaces) { delete p; }
+   for (auto p: P) { delete p; }
+   ParMeshes.clear();
+   fespaces.clear();
+   P.clear();
+   delete fec;
+   delete fespace;
+   delete pmesh;
+
+   MPI_Finalize();
+   return 0;
+}
+
+//define exact solution
+void E_exact(const Vector &x, Vector &E)
+{
+   double curlE[3], curl2E[3];
+   get_maxwell_solution(x, E, curlE, curl2E);
+}
+
+void H_exact(const Vector &x, Vector &H)
+{
+   double E[3], curlE[3], curl2E[3];
+   get_maxwell_solution(x, E, curlE, curl2E);
+   for (int i = 0; i<3; i++) { H(i) = curlE[i]/omega; }
+}
+
+
+void f_exact_H(const Vector &x, Vector &f)
+{
+   // curl H - omega E = f
+   // = curl (curl E / omega) - omega E
+   f = 0.0;
+   if (sol !=4)
+   {
+      double E[3], curlE[3], curl2E[3];
+      get_maxwell_solution(x, E, curlE, curl2E);
+      f(0) = curl2E[0] / omega - omega * E[0];
+      f(1) = curl2E[1] / omega - omega * E[1];
+      f(2) = curl2E[2] / omega - omega * E[2];
+   }
+}
+
+void get_maxwell_solution(const Vector &X, double E[], double curlE[],
+                          double curl2E[])
+{
+   double x = X[0];
+   double y = X[1];
+   double z = X[2];
+
+   if (sol ==-1)
+   {
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
+      E[2] = x * y * (1.0 - x) * (1.0 - y);
+
+      curlE[0] = -(x-1.0) * x * (y*(2.0*z-3.0)+1.0);
+      curlE[1] = -2.0*(y-1.0)*y*(x-z);
+      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
+
+      curl2E[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
+      curl2E[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
+      curl2E[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
+   }
+   else if (sol == 0) // polynomial
+   {
+      // Polynomial vanishing on the boundary
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = (1.0 - x) * x * y * (1.0 - z) * z;
+      E[2] = (1.0 - x) * x * (1.0 - y) * y;
+      //
+      curlE[0] = -(-1.0 + x) * x * (1.0 + y * (-3.0 + 2.0 * z));
+      curlE[1] = -2.0 * (-1.0 + y) * y * (x - z);
+      curlE[2] = (1.0 + (-3.0 + 2.0 * x) * y) * (-1.0 + z) * z;
+
+      curl2E[0] = -2.0 * (-1.0 + y) * y + (-3.0 + 2.0 * x) * (-1.0 + z) * z;
+      curl2E[1] = -2.0 * y * (-x + x * x + (-1.0 + z) * z);
+      curl2E[2] = -2.0 * (-1.0 + y) * y + (-1.0 + x) * x * (-3.0 + 2.0 * z);
+   }
+   else if (sol == 1) // sinusoidal
+   {
+      E[0] = sin(omega * y);
+      E[1] = sin(omega * z);
+      E[2] = sin(omega * x);
+
+      curlE[0] = -omega * cos(omega * z);
+      curlE[1] = -omega * cos(omega * x);
+      curlE[2] = -omega * cos(omega * y);
+
+      curl2E[0] = omega * omega * E[0];
+      curl2E[1] = omega * omega * E[1];
+      curl2E[2] = omega * omega * E[2];
+   }
+   else if (sol == 2) // point source
+   {
+      // shift to avoid singularity
+      double x0 = x + 0.1;
+      double x1 = y + 0.1;
+      double x2 = z + 0.1;
+      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
+
+      E[0] = cos(omega * r);
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      double r_x = x0 / r;
+      double r_y = x1 / r;
+      double r_z = x2 / r;
+      double r_xy = -(r_x / r) * r_y;
+      double r_xz = -(r_x / r) * r_z;
+      double r_yx = r_xy;
+      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
+      double r_zx = r_xz;
+      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
+
+      curlE[0] = 0.0;
+      curlE[1] = -omega * r_z * sin(omega * r);
+      curlE[2] =  omega * r_y * sin(omega * r);
+
+      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
+                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
+      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(
+                               omega * r));
+      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(
+                               omega * r));
+   }
+   else if (sol == 3) // plane wave
+   {
+      double coeff = omega / sqrt(3.0);
+      E[0] = cos(coeff * (x + y + z));
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      curlE[0] = 0.0;
+      curlE[1] = -coeff * sin(coeff * (x + y + z));
+      curlE[2] = coeff * sin(coeff * (x + y + z));
+
+      curl2E[0] = 2.0 * coeff * coeff * E[0];
+      curl2E[1] = -coeff * coeff * E[0];
+      curl2E[2] = -coeff * coeff * E[0];
+   }
+   else if (sol == -1)
+   {
+      E[0] = cos(omega * y);
+      E[1] = 0.0;
+
+      curlE[0] = 0.0;
+      curlE[1] = 0.0;
+      curlE[2] = -omega * sin(omega * y);
+
+      curl2E[0] = omega*omega * cos(omega*y);
+      curl2E[1] = 0.0;
+      curl2E[2] = 0.0;
+   }
+   else if (sol == 4) // Airy function
+   {
+      E[0] = 0;
+      E[1] = 0;
+      // double b = -pow(omega/4.0,2.0/3.0)*(4.0*x(0)-1.0);
+      double b = -pow(omega/4.0,2.0/3.0)*(4.0*x-1.0);
+      //E[2] = boost::math::airy_ai(b);
+      E[2] = gsl_sf_airy_Ai(b, GSL_PREC_DOUBLE);
+
+      curlE[0] = 0.0;
+      curlE[1] = 4.0 * pow(omega/4.0,2.0/3.0) * gsl_sf_airy_Ai_deriv(b,
+                                                                     GSL_PREC_DOUBLE);
+      curlE[2] = 0.0;
+
+      // not used
+      curl2E[0] = 0.0;
+      curl2E[1] = 0.0;
+      curl2E[2] = 0.0;
+   }
+}
+
+void epsilon_func(const Vector &x, Vector &M)
+{
+   M.SetSize(3);
+
+   M = 1.0;
+   if (sol == 4)
+   {
+      M[2] = 4.0*x(0)-1.0;
+   }
+}
+
+void epsilon2_func(const Vector &x, Vector &M)
+{
+   M.SetSize(3);
+
+   M = 1.0;
+   if (sol == 4)
+   {
+      M[2] = (4.0*x(0)-1.0) * (4.0*x(0)-1.0);
+   }
+}
+
+void epsilon_func_mat(const Vector &x, DenseMatrix &M)
+{
+   M.SetSize(3);
+
+   M = 0.0;
+   M(0,0) = 1.0;
+   M(1,1) = 1.0;
+   if (sol != 4)
+   {
+      M(2,2) = 1.0;
+   }
+   else
+   {
+      M(2,2) = 4.0*x(0)-1.0;
+   }
+}

examples/ex3p_fosls.cpp

@@ -0,0 +1,764 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#include "gsl_sf_airy.h"
+
+#include "multigrid.hpp"
+
+using namespace std;
+using namespace mfem;
+
+#define COMPLEX_IMPEDANCE  // Whether to use impedance BC and solve a complex system
+
+// Define exact solution
+void E_exact(const Vector & x, Vector & E);
+void H_exact(const Vector & x, Vector & H);
+void f_exact_H(const Vector & x, Vector & f_H);
+void get_maxwell_solution(const Vector & x, double E[], double curlE[],
+                          double curl2E[]);
+void epsilon_func(const Vector &x, DenseMatrix &M);
+void epsilon2_func(const Vector &x, DenseMatrix &M);
+
+int dim;
+double omega;
+int sol = 4;
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Initialize MPI
+   int num_procs, myid;
+   MPI_Init(&argc, &argv); // Initialize MPI
+   MPI_Comm_size(MPI_COMM_WORLD,
+                 &num_procs); //total number of processors available
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid); // Determine process identifier
+   // 1. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../data/inline-hex.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // static condensation flag
+   bool static_cond = false;
+   // visualization flag
+   bool visualization = 1;
+   // number of wavelengths
+   double k = 1.0;
+   // number of mg levels
+   int ref_levels = 1;
+   // number of initial ref
+   int initref = 1;
+
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&ref_levels, "-ref", "--ref_levels",
+                  "Number of Refinements.");
+   args.AddOption(&initref, "-initref", "--initref",
+                  "Number of initial refinements.");
+   args.AddOption(&sol, "-sol", "--exact",
+                  "Exact solution flag - "
+                  " 1:sinusoidal, 2: point source, 3: plane wave");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   // check if the inputs are correct
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Angular frequency
+   //omega = 2.0*k*M_PI;
+   omega = k;
+
+   // 2. Read the mesh from the given mesh file.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+
+   if (sol == 4)
+   {
+      mesh->EnsureNodes();
+      GridFunction *nodes = mesh->GetNodes();
+      (*nodes) *= 0.5;
+   }
+
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 3. Executing uniform h-refinement
+   for (int i = 0; i < initref; i++ )
+   {
+      mesh->UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+
+   // 4. Define a finite element space on the mesh.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   std::vector<ParFiniteElementSpace * > fespaces(ref_levels+1);
+   std::vector<ParMesh * > ParMeshes(ref_levels+1);
+   std::vector<HypreParMatrix*> P(ref_levels);
+
+   for (int i = 0; i < ref_levels; i++)
+   {
+      ParMeshes[i] =new ParMesh(*pmesh);
+      fespaces[i] = new ParFiniteElementSpace(*fespace, *ParMeshes[i]);
+      pmesh->UniformRefinement();
+      // Update fespace
+      fespace->Update();
+      OperatorHandle Tr(Operator::Hypre_ParCSR);
+      fespace->GetTrueTransferOperator(*fespaces[i], Tr);
+      Tr.SetOperatorOwner(false);
+      Tr.Get(P[i]);
+   }
+   fespaces[ref_levels] = new ParFiniteElementSpace(*fespace);
+
+   Array<int> ess_tdof_listE;
+   Array<int> ess_tdof_listH;
+   Array<int> ess_bdrE(pmesh->bdr_attributes.Max());
+   Array<int> ess_bdrH(pmesh->bdr_attributes.Max());
+#ifdef COMPLEX_IMPEDANCE
+   ess_bdrE = 0;
+   ess_bdrH = 0;  // Neumann
+#else
+   ess_bdrE = 1;
+   ess_bdrH = 0;  // Neumann
+#endif
+   fespace->GetEssentialTrueDofs(ess_bdrE, ess_tdof_listE);
+   fespace->GetEssentialTrueDofs(ess_bdrH, ess_tdof_listH);
+
+#ifdef COMPLEX_IMPEDANCE
+   Array<int> bdr_attr(pmesh->bdr_attributes.Max());
+   bdr_attr = 1;
+   ConstantCoefficient impedance(omega);
+   RestrictedCoefficient imp_rest(impedance, bdr_attr);
+   Array<int> block_offsets(5);
+   Array<int> block_trueOffsets(5);
+#else
+   Array<int> block_offsets(3);
+   Array<int> block_trueOffsets(3);
+#endif
+   block_offsets[0] = 0;
+   block_offsets[1] = fespace->GetVSize();
+   block_offsets[2] = fespace->GetVSize();
+#ifdef COMPLEX_IMPEDANCE
+   block_offsets[3] = fespace->GetVSize();
+   block_offsets[4] = fespace->GetVSize();
+#endif
+   block_offsets.PartialSum();
+
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = fespace->TrueVSize();
+   block_trueOffsets[2] = fespace->TrueVSize();
+#ifdef COMPLEX_IMPEDANCE
+   block_trueOffsets[3] = fespace->TrueVSize();
+   block_trueOffsets[4] = fespace->TrueVSize();
+#endif
+   block_trueOffsets.PartialSum();
+
+   //    _           _    _  _       _  _
+   //   |             |  |    |     |    |
+   //   |  A00   A01  |  | E  |     |F_E |
+   //   |             |  |    |  =  |    |
+   //   |  A10   A11  |  | H  |     |F_G |
+   //   |_           _|  |_  _|     |_  _|
+   //
+   // A00 = (curl E, curl F) + \omega^2 (E,F)
+   // A01 = - \omega *( (curl E, F) + (E,curl F)
+   // A10 = - \omega *( (curl H, G) + (H,curl G)
+   // A11 = (curl H, curl G) + \omega^2 (H,G)
+
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
+
+   x = 0.0;
+   rhs = 0.0;
+   trueX = 0.0;
+   trueRhs = 0.0;
+
+   VectorFunctionCoefficient Eex(sdim, E_exact);
+   ConstantCoefficient negOne(-1.0);
+   VectorFunctionCoefficient mEex(sdim, E_exact, &negOne);
+
+   ParGridFunction * E_gf = new ParGridFunction;
+   ParGridFunction * Exact_gf = new ParGridFunction(fespace);
+   E_gf->MakeRef(fespace, x.GetBlock(0));
+   E_gf->ProjectCoefficient(Eex);
+   Exact_gf->ProjectCoefficient(Eex);
+
+   VectorFunctionCoefficient Hex(sdim, H_exact);
+   ParGridFunction * H_gf = new ParGridFunction;
+   H_gf->MakeRef(fespace, x.GetBlock(1));
+   H_gf->ProjectCoefficient(Hex);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient sigma(pow(omega, 2));
+   ConstantCoefficient neg(-abs(omega));
+   ConstantCoefficient pos(abs(omega));
+
+   MatrixFunctionCoefficient epsilon(dim,epsilon_func);
+   TransposeMatrixCoefficient epsilonT(epsilon);  // transpose of epsilon
+   MatrixFunctionCoefficient epsilon2(dim,epsilon2_func);
+   ScalarMatrixProductCoefficient coeff(neg,epsilon);
+   ScalarMatrixProductCoefficient coeff2(sigma,epsilon2);
+
+   // 6. Set up the linear form
+   VectorFunctionCoefficient f_H(sdim,f_exact_H);
+   ScalarVectorProductCoefficient sf_H(neg,f_H);
+
+   MatVecCoefficient epsT_sf_H(epsilonT, sf_H);
+
+   ParLinearForm *b_E = new ParLinearForm;
+   b_E->Update(fespace, rhs.GetBlock(0), 0);
+   b_E->AddDomainIntegrator(new VectorFEDomainLFIntegrator(epsT_sf_H));
+#ifdef COMPLEX_IMPEDANCE
+   b_E->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                 Eex));  // <g_Im, n x Q x n> = <n x E_Re x n, n x Q x n>
+
+   ParLinearForm *b_E_Im = new ParLinearForm;
+   b_E_Im->Update(fespace, rhs.GetBlock(2), 0);
+   b_E_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                    Hex));  // -<g_Re, n x Q x n> = <n x H_Re, n x Q x n>
+   b_E_Im->Assemble();
+#endif
+   b_E->Assemble();
+
+   ParLinearForm *b_H = new ParLinearForm;
+   b_H->Update(fespace, rhs.GetBlock(1), 0);
+   b_H->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
+#ifdef COMPLEX_IMPEDANCE
+   b_H->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
+                                 Hex));  // -<g_Re, n x R> = <n x H_Re, n x R>
+
+   ParLinearForm *b_H_Im = new ParLinearForm;
+   b_H_Im->Update(fespace, rhs.GetBlock(3), 0);
+   b_H_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
+                                    mEex));  // -<g_Im, n x R> = -<n x E_Re x n, n x R>
+   b_H_Im->Assemble();
+#endif
+   b_H->Assemble();
+
+#ifdef COMPLEX_IMPEDANCE
+   fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(2),
+                                                   trueRhs.GetBlock(2));
+   fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(3),
+                                                   trueRhs.GetBlock(3));
+#endif
+
+   // 7. Bilinear form a(.,.) on the finite element space
+   ParBilinearForm *a_EE = new ParBilinearForm(fespace);
+   a_EE->AddDomainIntegrator(new CurlCurlIntegrator(one));
+   a_EE->AddDomainIntegrator(new VectorFEMassIntegrator(coeff2));
+
+#ifdef COMPLEX_IMPEDANCE
+   a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator());
+   //a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator(imp_rest));
+   //a_EE->AddBoundaryIntegrator(new BoundaryMassIntegrator(imp_rest));
+   //a_EE->AddBdrFaceIntegrator(new BoundaryMassIntegrator(imp_rest));
+#endif
+
+   a_EE->Assemble();
+   a_EE->Finalize();
+   HypreParMatrix *A_EE = new HypreParMatrix;
+   a_EE->FormLinearSystem(ess_tdof_listE, x.GetBlock(0), rhs.GetBlock(0), *A_EE,
+                          trueX.GetBlock(0), trueRhs.GetBlock(0));
+
+   ParBilinearForm *a_HH = new ParBilinearForm(fespace);
+   a_HH->AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
+   a_HH->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
+
+#ifdef COMPLEX_IMPEDANCE
+   a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator());
+   //a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator(imp_rest));
+   //a_HH->AddBoundaryIntegrator(new BoundaryMassIntegrator(imp_rest));
+#endif
+
+   a_HH->Assemble();
+   a_HH->Finalize();
+   HypreParMatrix *A_HH = new HypreParMatrix;
+   a_HH->FormLinearSystem(ess_tdof_listH, x.GetBlock(1), rhs.GetBlock(1), *A_HH,
+                          trueX.GetBlock(1), trueRhs.GetBlock(1));
+
+   ParMixedBilinearForm *a_HE = new ParMixedBilinearForm(fespace,fespace);
+   a_HE->AddDomainIntegrator(new MixedVectorCurlIntegrator(neg));
+   a_HE->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(coeff));
+   a_HE->Assemble();
+   a_HE->Finalize();
+   HypreParMatrix *A_HE = new HypreParMatrix;
+   a_HE->FormColLinearSystem(ess_tdof_listE,x.GetBlock(0),rhs.GetBlock(1),*A_HE,
+                             trueX.GetBlock(0),trueRhs.GetBlock(1));
+
+   HypreParMatrix *A_EH = A_HE->Transpose();
+
+#ifdef COMPLEX_IMPEDANCE
+   ParBilinearForm *a_EH_Im = new ParBilinearForm(fespace);
+   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(imp_rest));
+   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(impedance));
+   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(omega));
+   a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator());
+   a_EH_Im->Assemble();
+   a_EH_Im->Finalize();
+
+   OperatorHandle A_EH_Im_ptr;
+   a_EH_Im->FormSystemMatrix(ess_tdof_listE,
+                             A_EH_Im_ptr);  // empty ess_tdof_list for impedance
+
+   HypreParMatrix *A_EH_Im = A_EH_Im_ptr.As<HypreParMatrix>();
+
+   HypreParMatrix *A_HE_Im = A_EH_Im->Transpose();
+#endif
+
+   BlockOperator *LS_Maxwellop = new BlockOperator(block_trueOffsets);
+#ifdef COMPLEX_IMPEDANCE
+   const int numBlocks = 4;
+#else
+   const int numBlocks = 2;
+#endif
+
+   LS_Maxwellop->SetBlock(0, 0, A_EE);
+   LS_Maxwellop->SetBlock(0, 1, A_EH);
+   LS_Maxwellop->SetBlock(1, 0, A_HE);
+   LS_Maxwellop->SetBlock(1, 1, A_HH);
+
+#ifdef COMPLEX_IMPEDANCE
+   LS_Maxwellop->SetBlock(0, 3, A_EH_Im);
+   LS_Maxwellop->SetBlock(1, 2, A_HE_Im, -1.0);
+
+   LS_Maxwellop->SetBlock(2, 1, A_EH_Im, -1.0);
+   LS_Maxwellop->SetBlock(3, 0, A_HE_Im);
+
+   LS_Maxwellop->SetBlock(2, 2, A_EE);
+   LS_Maxwellop->SetBlock(2, 3, A_EH);
+   LS_Maxwellop->SetBlock(3, 2, A_HE);
+   LS_Maxwellop->SetBlock(3, 3, A_HH);
+#endif
+
+   if (myid == 0)
+   {
+      cout << "Size of fine grid system: "
+           << 2.0 * A_EE->GetGlobalNumRows() << " x " << 2.0* A_EE->GetGlobalNumCols() <<
+           endl;
+   }
+
+   // Set up the preconditioner
+   Array2D<HypreParMatrix*> blockA(numBlocks, numBlocks);
+   Array2D<double> blockAcoef(numBlocks, numBlocks);
+
+   for (int i=0; i<numBlocks; ++i)
+   {
+      for (int j=0; j<numBlocks; ++j)
+      {
+         if (LS_Maxwellop->IsZeroBlock(i,j) == 0)
+         {
+            blockA(i,j) = static_cast<HypreParMatrix *>(&LS_Maxwellop->GetBlock(i,j));
+            blockAcoef(i,j) = LS_Maxwellop->GetBlockCoef(i,j);
+         }
+         else
+         {
+            blockA(i,j) = NULL;
+            blockAcoef(i,j) = 1.0;
+         }
+      }
+   }
+
+   // // double nnz = A_HH->NNZ();
+   // // double ndof = A_HH->GetGlobalNumRows();
+   // // double est_mem_b = nnz*12.0 + (ndof+1.0)*4;
+   // // double gb = est_mem_b*4.0/pow(1024.0,3);
+
+   // // mfem::out << "Estimated memory taken by the global matrix: " <<  gb << endl;
+
+   int maxit(2000);
+   double rtol(1.e-8);
+   double atol(1.e-12);
+
+   // trueX = 0.0;
+   CGSolver pcg(MPI_COMM_WORLD);
+   pcg.SetAbsTol(atol);
+   pcg.SetRelTol(rtol);
+   pcg.SetMaxIter(maxit);
+   pcg.SetOperator(*LS_Maxwellop);
+   pcg.SetPrintLevel(1);
+   chrono.Clear();
+   chrono.Start();
+   BlockMGSolver * precMG = new BlockMGSolver(LS_Maxwellop->Height(),
+                                              LS_Maxwellop->Width(), blockA, blockAcoef, P);
+   //precMG->SetTheta(0.5);
+   // // int lv_coarse = min(ref_levels,ref_levels-1);
+   // // int levels = ref_levels - lv_coarse;
+   // // BlkParSchwarzSmoother * precAS = new BlkParSchwarzSmoother(fespaces[lv_coarse]->GetParMesh(),levels,fespaces[ref_levels],LS_Maxwellop);
+   chrono.Stop();
+   if (myid == 0)
+   {
+      cout << "MG Setup time: " << chrono.RealTime() << endl;
+   }
+
+   // Randomize H RHS
+   //trueRhs = 1.0;
+   /*
+   for (int i=block_trueOffsets[1]; i<block_trueOffsets[2]; ++i)
+     trueRhs[i] = i % 53;
+   */
+
+   chrono.Clear();
+   chrono.Start();
+   pcg.SetPreconditioner(*precMG);
+   // // pcg.SetPreconditioner(*precAS);
+   pcg.Mult(trueRhs, trueX);
+   chrono.Stop();
+   delete precMG;
+   // // delete precAS;
+
+   // // trueX = 0.0;
+   // // invA->Mult(trueRhs,trueX);
+
+   if (myid == 0)
+   {
+      cout << "MG Solution time time: " << chrono.RealTime() << endl;
+   }
+   // // cin.get();
+   // //  if(myid == 0)
+   // //    cout << "MG prec Solution time: " << chrono.RealTime() << endl;
+
+
+   // // chrono.Clear();
+   // // chrono.Start();
+   // // Block_AMSSolver * precAMS = new Block_AMSSolver(block_trueOffsets,fespaces);
+   // // precAMS->SetSmootherType(Block_AMS::BlkSmootherType::SCHWARZ);
+   // // precAMS->SetSmootherType(Block_AMS::BlkSmootherType::HYPRE);
+   // // precAMS->SetOperator(LS_Maxwellop);
+   // // precAMS->SetTheta(1.0/5.0);
+   // // // 0-Smoother, 1-Grad, 2,3,4-Pix,Piy,Piz
+   // // precAMS->SetCycleType("023414320");
+   // // precAMS->SetNumberofCycles(1);
+   // // chrono.Stop();
+   // // if(myid == 0)
+   // //    cout << "BlkAMS Setup time: " << chrono.RealTime() << endl;
+
+   // // // resolve with block AMS
+   // // trueX = 0;
+   // // chrono.Clear();
+   // // chrono.Start();
+   // // pcg.SetPreconditioner(*precAMS);
+   // // pcg.Mult(trueRhs, trueX);
+   // // chrono.Stop();
+   // // delete precAMS;
+
+   // // if(myid == 0)
+   // //    cout << "BlockAMS Solution time: " << chrono.RealTime() << endl;
+
+
+   a_EE->RecoverFEMSolution(trueX.GetBlock(0), *b_E, *E_gf);
+   a_HH->RecoverFEMSolution(trueX.GetBlock(1), *b_H, *H_gf);
+
+
+   int order_quad = max(2, 2*order+1);
+   const IntegrationRule *irs[Geometry::NumGeom];
+   for (int i=0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+
+   double Error_E = E_gf->ComputeL2Error(Eex, irs);
+   double norm_E = ComputeGlobalLpNorm(2, Eex, *pmesh, irs);
+
+   double Error_H = H_gf->ComputeL2Error(Hex, irs);
+   double norm_H = ComputeGlobalLpNorm(2, Hex , *pmesh, irs);
+
+   if (myid == 0)
+   {
+      cout << "|| E_h - E || = " << Error_E  << "\n";
+      cout << "|| E_h - E ||/||E|| = " << Error_E/norm_E  << "\n";
+      cout << "|| H_h - H || = " << Error_H  << "\n";
+      cout << "|| H_h - H ||/||H|| = " << Error_H/norm_H  << "\n";
+      cout << "Total error = " << setprecision(15) << sqrt(Error_H*Error_H
+                                                           +Error_E*Error_E) << "\n";
+   }
+
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      E_gf->Save(sol_ofs);
+   }
+
+   //ParGridFunction ExactE(fespace);
+   /*
+   if (visualization)
+     {
+       // 8. Connect to GLVis.
+       char vishost[] = "localhost";
+       int  visport   = 19916;
+       // socketstream E_sock(vishost, visport);
+       // E_sock << "parallel " << num_procs << " " << myid << "\n";
+       // E_sock.precision(8);
+       // E_sock << "solution\n" << *pmesh << *E_gf << "window_title 'Electric field'" << endl;
+       socketstream Exact_sock(vishost, visport);
+       Exact_sock << "parallel " << num_procs << " " << myid << "\n";
+       Exact_sock.precision(8);
+       Exact_sock << "solution\n" << *pmesh << *Exact_gf << "window_title 'Electric field'" << endl;
+
+       // MPI_Barrier(pmesh->GetComm());
+       // socketstream Eex_sock(vishost, visport);
+       // Eex_sock << "parallel " << num_procs << " " << myid << "\n";
+       // Eex_sock.precision(8);
+       // Eex_sock << "solution\n" << *pmesh << *Exact_gf << "window_title 'Exact Electric field'" << endl;
+     }
+   */
+   // delete A_EE;
+   // delete A_HE;
+   // delete A_EH;
+   // delete A_HH;
+   // delete LS_Maxwellop;
+   // delete a_EE;
+   // delete a_HE;
+   // delete a_HH;
+   // delete b_E;
+   // delete b_H;
+   delete E_gf;
+   delete Exact_gf;
+   for (auto p: ParMeshes) { delete p; }
+   for (auto p: fespaces) { delete p; }
+   for (auto p: P) { delete p; }
+   ParMeshes.clear();
+   fespaces.clear();
+   P.clear();
+   delete fec;
+   delete fespace;
+   delete pmesh;
+
+   // cout << "Freed memory: " << endl;
+   // cin.get();
+   MPI_Finalize();
+   return 0;
+}
+
+//define exact solution
+void E_exact(const Vector &x, Vector &E)
+{
+   double curlE[3], curl2E[3];
+   get_maxwell_solution(x, E, curlE, curl2E);
+}
+
+void H_exact(const Vector &x, Vector &H)
+{
+   double E[3], curlE[3], curl2E[3];
+   get_maxwell_solution(x, E, curlE, curl2E);
+   for (int i = 0; i<3; i++) { H(i) = curlE[i]/omega; }
+}
+
+
+void f_exact_H(const Vector &x, Vector &f)
+{
+   // curl H - omega E = f
+   // = curl (curl E / omega) - omega E
+   f = 0.0;
+   if (sol !=4)
+   {
+      double E[3], curlE[3], curl2E[3];
+      get_maxwell_solution(x, E, curlE, curl2E);
+      f(0) = curl2E[0] / omega - omega * E[0];
+      f(1) = curl2E[1] / omega - omega * E[1];
+      f(2) = curl2E[2] / omega - omega * E[2];
+   }
+}
+
+void get_maxwell_solution(const Vector &X, double E[], double curlE[],
+                          double curl2E[])
+{
+   double x = X[0];
+   double y = X[1];
+   double z = X[2];
+
+
+
+   if (sol ==-1)
+   {
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
+      E[2] = x * y * (1.0 - x) * (1.0 - y);
+
+      curlE[0] = -(x-1.0) * x * (y*(2.0*z-3.0)+1.0);
+      curlE[1] = -2.0*(y-1.0)*y*(x-z);
+      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
+
+      curl2E[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
+      curl2E[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
+      curl2E[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
+   }
+   else if (sol == 0) // polynomial
+   {
+      // Polynomial vanishing on the boundary
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = (1.0 - x) * x * y * (1.0 - z) * z;
+      E[2] = (1.0 - x) * x * (1.0 - y) * y;
+      //
+      curlE[0] = -(-1.0 + x) * x * (1.0 + y * (-3.0 + 2.0 * z));
+      curlE[1] = -2.0 * (-1.0 + y) * y * (x - z);
+      curlE[2] = (1.0 + (-3.0 + 2.0 * x) * y) * (-1.0 + z) * z;
+
+      curl2E[0] = -2.0 * (-1.0 + y) * y + (-3.0 + 2.0 * x) * (-1.0 + z) * z;
+      curl2E[1] = -2.0 * y * (-x + x * x + (-1.0 + z) * z);
+      curl2E[2] = -2.0 * (-1.0 + y) * y + (-1.0 + x) * x * (-3.0 + 2.0 * z);
+   }
+   else if (sol == 1) // sinusoidal
+   {
+      E[0] = sin(omega * y);
+      E[1] = sin(omega * z);
+      E[2] = sin(omega * x);
+
+      curlE[0] = -omega * cos(omega * z);
+      curlE[1] = -omega * cos(omega * x);
+      curlE[2] = -omega * cos(omega * y);
+
+      curl2E[0] = omega * omega * E[0];
+      curl2E[1] = omega * omega * E[1];
+      curl2E[2] = omega * omega * E[2];
+   }
+   else if (sol == 2) // point source
+   {
+      // shift to avoid singularity
+      double x0 = x + 0.1;
+      double x1 = y + 0.1;
+      double x2 = z + 0.1;
+      //
+      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
+
+      E[0] = cos(omega * r);
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      double r_x = x0 / r;
+      double r_y = x1 / r;
+      double r_z = x2 / r;
+      double r_xy = -(r_x / r) * r_y;
+      double r_xz = -(r_x / r) * r_z;
+      double r_yx = r_xy;
+      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
+      double r_zx = r_xz;
+      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
+
+      curlE[0] = 0.0;
+      curlE[1] = -omega * r_z * sin(omega * r);
+      curlE[2] =  omega * r_y * sin(omega * r);
+
+      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
+                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
+      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(
+                               omega * r));
+      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(
+                               omega * r));
+   }
+   else if (sol == 3) // plane wave
+   {
+      double coeff = omega / sqrt(3.0);
+      E[0] = cos(coeff * (x + y + z));
+      E[1] = 0.0;
+      E[2] = 0.0;
+
+      curlE[0] = 0.0;
+      curlE[1] = -coeff * sin(coeff * (x + y + z));
+      curlE[2] = coeff * sin(coeff * (x + y + z));
+
+      curl2E[0] = 2.0 * coeff * coeff * E[0];
+      curl2E[1] = -coeff * coeff * E[0];
+      curl2E[2] = -coeff * coeff * E[0];
+   }
+   else if (sol == -1)
+   {
+      E[0] = cos(omega * y);
+      E[1] = 0.0;
+
+      curlE[0] = 0.0;
+      curlE[1] = 0.0;
+      curlE[2] = -omega * sin(omega * y);
+
+      curl2E[0] = omega*omega * cos(omega*y);
+      curl2E[1] = 0.0;
+      curl2E[2] = 0.0;
+   }
+   else if (sol == 4) // Airy function
+   {
+      E[0] = 0;
+      E[1] = 0;
+      // double b = -pow(omega/4.0,2.0/3.0)*(4.0*x(0)-1.0);
+      double b = -pow(omega/4.0,2.0/3.0)*(4.0*x-1.0);
+      //E[2] = boost::math::airy_ai(b);
+      E[2] = gsl_sf_airy_Ai(b, GSL_PREC_DOUBLE);
+
+      curlE[0] = 0.0;
+      curlE[1] = 4.0 * pow(omega/4.0,2.0/3.0) * gsl_sf_airy_Ai_deriv(b,
+                                                                     GSL_PREC_DOUBLE);
+      curlE[2] = 0.0;
+
+      //  not used
+      curl2E[0] = 0.0;
+      curl2E[1] = 0.0;
+      curl2E[2] = 0.0;
+   }
+}
+
+
+void epsilon_func(const Vector &x, DenseMatrix &M)
+{
+   M.SetSize(3);
+
+   M = 0.0;
+   M(0,0) = 1.0;
+   M(1,1) = 1.0;
+   if (sol != 4)
+   {
+      M(2,2) = 1.0;
+   }
+   else
+   {
+      M(2,2) = 4.0*x(0)-1.0;
+      // M(2,2) = 2.0;
+   }
+}
+
+void epsilon2_func(const Vector &x, DenseMatrix &M)
+{
+   M.SetSize(3);
+
+   M = 0.0;
+   M(0,0) = 1.0;
+   M(1,1) = 1.0;
+   if (sol != 4)
+   {
+      M(2,2) = 1.0;
+   }
+   else
+   {
+      M(2,2) = (4.0*x(0)-1.0) * (4.0*x(0)-1.0);
+      // M(2,2) = 4.0;
+   }
+}

examples/ex3p_sp.cpp

@@ -0,0 +1,915 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+int dim;
+
+#define SIGMAVAL -250.0
+//#define FORM_DEFINITE
+//#define SOLVE_A2
+//#define ITER_A2
+
+//#define USE_CSL
+
+//#define USE_HELMHOLTZ
+
+//#define TEST_MULTIPLE_SP
+
+#ifdef USE_HELMHOLTZ
+void GetHelmholtzMatrix(ParMesh *pmesh, const int dir, HypreParMatrix *A)
+{
+   const int order = 1;
+   FiniteElementCollection *fec;
+   fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   Array<int> ess_tdof_list;
+
+   const bool homogeneousBCeverywhere = false;
+   if (homogeneousBCeverywhere)
+   {
+      if (pmesh->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+   }
+   else
+   {
+      // Set boundary conditions, depending on dir.
+      MFEM_VERIFY(dim == 3, "");
+      for (int i=0; i<pmesh->GetNBE(); ++i)
+      {
+         Element *elem = pmesh->GetBdrElement(i);
+         MFEM_VERIFY(elem->GetNVertices() >= 3, "");
+         const int *vertices = elem->GetVertices();
+         double *v[3];
+         for (int j=0; j<3; ++j)
+         {
+            v[j] = pmesh->GetVertex(vertices[j]);
+         }
+
+         double u[3];
+         double w[3];
+         for (int j=0; j<3; ++j)
+         {
+            u[j] = v[1][j] - v[0][j];  // An edge tangent
+            w[j] = v[2][j] - v[1][j];  // Another edge tangent, not parallel to u.
+         }
+
+         double n[3];  // normal vector, taken as the cross product u x v
+         n[0] = (u[1]*w[2]) - (u[2]*w[1]);
+         n[1] = (u[2]*w[0]) - (u[0]*w[2]);
+         n[2] = (u[0]*w[1]) - (u[1]*w[0]);
+
+         double t = sqrt((n[0]*n[0]) + (n[1]*n[1]) + (n[2]*n[2]));
+
+         int d = -1;
+         for (int j=0; j<3; ++j)
+         {
+            n[j] /= t;  // normalize
+            if (fabs(fabs(n[j]) - 1.0) < 1.0e-8)
+            {
+               d = j;
+            }
+         }
+
+         MFEM_VERIFY(d >= 0, "");
+
+         if (d != dir)  // face has essential BC at all DOF's.
+         {
+            elem->SetAttribute(1);
+         }
+         else
+         {
+            elem->SetAttribute(0);
+         }
+      }
+
+      Array<int> ess_bdr(2);
+      ess_bdr = 0;
+      ess_bdr[1] = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient neg(SIGMAVAL);
+
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   a->AddDomainIntegrator(new MassIntegrator(neg));
+
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient zero(0.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(zero));
+   b->Assemble();
+
+   bool static_cond = false;
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, *A, X, B);
+}
+#endif
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/beam-tet.mesh";
+   int order = 2;
+   bool static_cond = false;
+   bool visualization = 1;
+#ifdef MFEM_USE_STRUMPACK
+   bool use_strumpack = false;
+#endif
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use STRUMPACK's double complex linear solver.");
+#endif
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   kappa = freq * M_PI;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      //(int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   {
+      double minsize = pmesh->GetElementSize(0);
+      double maxsize = minsize;
+      for (int i=1; i<pmesh->GetNE(); ++i)
+      {
+         const double size_i = pmesh->GetElementSize(i);
+         minsize = std::min(minsize, size_i);
+         maxsize = std::max(maxsize, size_i);
+      }
+
+      cout << myid << ": Element size range: (" << minsize << ", " << maxsize << ")"
+           << endl;
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   long globalNE = pmesh->GetGlobalNE();
+   if (myid == 0)
+   {
+      cout << "Number of mesh elements: " << globalNE << endl;
+      cout << "Number of finite element unknowns: " << size << endl;
+      cout << "Root local number of finite element unknowns: " << fespace->TrueVSize()
+           << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   VectorFunctionCoefficient f(sdim, f_exact);
+   ParLinearForm *b = new ParLinearForm(fespace);
+   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient E(sdim, E_exact);
+   x.ProjectCoefficient(E);
+
+   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+   //     mass domain integrators.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(SIGMAVAL);
+   Coefficient *sigmaAbs = new ConstantCoefficient(fabs(SIGMAVAL));
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+
+   //cout << myid << ": NBE " << pmesh->GetNBE() << endl;
+
+#ifdef FORM_DEFINITE
+   ParBilinearForm *adef = new ParBilinearForm(fespace);
+   adef->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   adef->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
+
+   if (static_cond) { adef->EnableStaticCondensation(); }
+   adef->Assemble();
+
+   HypreParMatrix Adef;
+   Vector Bdef, Xdef;
+   adef->FormLinearSystem(ess_tdof_list, x, *b, Adef, Xdef, Bdef);
+#endif
+
+#ifdef USE_CSL
+   Vector Bdef, Xdef;
+
+   ParBilinearForm *Mform =  new ParBilinearForm(fespace);
+   Mform->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
+   Mform->Assemble();
+
+   // Mform->Finalize();
+
+   HypreParMatrix Mmat, Smat, Mcopy;
+   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
+   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
+                           Bdef);  // There must be a better way than creating two identical matrices.
+
+   ParBilinearForm *Sform =  new ParBilinearForm(fespace);
+   Sform->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   Sform->Assemble();
+
+   Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
+
+   ParBilinearForm *agrad = new ParBilinearForm(fespace);
+   //agrad->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   agrad->AddDomainIntegrator(new VectorFEMassIntegrator(*muinv));
+
+   if (static_cond) { agrad->EnableStaticCondensation(); }
+   agrad->Assemble();
+   HypreParMatrix Agrad;
+   agrad->FormLinearSystem(ess_tdof_list, x, *b, Agrad, Xdef, Bdef);
+#endif
+
+#ifdef ITER_A2
+   Vector Bdef, Xdef;
+
+   ParBilinearForm *Mform =  new ParBilinearForm(fespace);
+   Mform->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
+   Mform->Assemble();
+
+   Mform->Finalize();
+
+   HypreParMatrix Mmat, Mcopy;
+   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
+   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
+                           Bdef);  // There must be a better way to implement M^2.
+
+   /*
+   HypreParMatrix *Mmat = Mform->ParallelAssemble();
+   HypreParMatrix *Mcopy = Mform->ParallelAssemble();  // There must be a better way to implement M^2.
+   */
+
+   ParBilinearForm *Sform =  new ParBilinearForm(fespace);
+   Sform->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   Sform->Assemble();
+
+   HypreParMatrix Smat, Scopy;
+   Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
+   Sform->FormLinearSystem(ess_tdof_list, x, *b, Scopy, Xdef, Bdef);
+#endif
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+#ifdef SOLVE_A2
+   HypreParMatrix Acopy;
+   {
+      Vector Bdum, Xdum;
+      a->FormLinearSystem(ess_tdof_list, x, *b, Acopy, Xdum, Bdum);
+   }
+#endif
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+
+   //A.Print("maxwell1000_2");
+
+#ifdef MFEM_USE_STRUMPACK
+   if (use_strumpack)
+   {
+      const bool fullDirect = true;
+
+#ifdef USE_CSL
+      const double beta1 = 1.0;
+      const double beta2 = 0.5;
+
+      Mmat *= -beta1;
+
+      // HypreParMatrix *cslRe = Add(1.0, Smat, -beta1, Mmat);
+      HypreParMatrix * cslRe = ParAdd(&Smat, &Mmat);
+
+      Mcopy *= beta2;
+
+      //ComplexHypreParMatrix chpm(cslRe, &Mcopy, false, false);
+      ComplexHypreParMatrix chpm(&A, &Mcopy, false,
+                                 false);  // For the case beta1 = 1.
+
+      HypreParMatrix *cSysMat = chpm.GetSystemMatrix();
+
+      Array<int> block_offsets(3); // number of variables + 1
+      block_offsets[0] = 0;
+      block_offsets[1] = fespace->GetVSize();
+      block_offsets[2] = fespace->GetVSize();
+      block_offsets.PartialSum();
+
+      Array<int> block_trueOffsets(3); // number of variables + 1
+      block_trueOffsets[0] = 0;
+      block_trueOffsets[1] = fespace->TrueVSize();
+      block_trueOffsets[2] = fespace->TrueVSize();
+      block_trueOffsets.PartialSum();
+
+      //cout << myid << ": V size " << fespace->GetVSize() << ", true " << fespace->TrueVSize() << ", global true " << size << ", B size "
+      //<< B.Size() << ", X size " << X.Size() << endl;
+
+      // Note that B is of true size.
+      BlockVector trueY(block_trueOffsets), trueX(block_trueOffsets),
+                  trueRhs(block_trueOffsets);
+
+      trueRhs.GetBlock(0) = B;
+      trueRhs.GetBlock(1) = 0.0;
+
+      Operator * Arow = new STRUMPACKRowLocMatrix(*cSysMat);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+#endif
+
+      if (fullDirect)
+      {
+#ifdef USE_CSL
+         //Solver * precond = strumpack;
+
+         // strumpack->Mult(B, X);
+
+         BlockOperator blockDiagA(block_trueOffsets);
+         for (int i=0; i<2; ++i)
+         {
+            blockDiagA.SetDiagonalBlock(i, &A);
+         }
+
+         ParFiniteElementSpace *prec_fespace =
+            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+         HypreSolver *amsgrad = new HypreAMS(Agrad, prec_fespace);
+
+#ifdef HYPRE_DYLAN
+         {
+            Vector Xtmp(X);
+            amsgrad->Mult(B,
+                          Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
+         }
+
+         HypreAMSG *amsg = new HypreAMSG((HypreAMS*) amsgrad, argc, argv);
+         BlockOperator blockDiagP(block_trueOffsets);
+         for (int i=0; i<2; ++i)
+         {
+            blockDiagP.SetDiagonalBlock(i, amsg);
+         }
+
+         TripleProductOperator strumpackProj(&blockDiagP, strumpack, &blockDiagP, false,
+                                             false, false);
+         ProductOperator prod(&blockDiagA, &strumpackProj, false, false);
+#else
+         ProductOperator prod(&blockDiagA, strumpack, false, false);
+#endif
+
+         GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
+         //BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
+
+         gmres->SetOperator(prod);
+         gmres->SetRelTol(1e-12);
+         gmres->SetMaxIter(1000);
+         gmres->SetPrintLevel(1);
+
+         gmres->Mult(trueRhs, trueY);
+         strumpack->Mult(trueY, trueX);
+
+         X = trueX.GetBlock(0);
+         double xim2 = trueX.GetBlock(1).Norml2();
+         xim2 *= xim2;
+         double sumxim2 = 0.0;
+
+         MPI_Allreduce(&xim2, &sumxim2, 1, MPI_DOUBLE, MPI_SUM, fespace->GetComm());
+
+         if (myid == 0)
+         {
+            cout << myid << ": norm of Xim " << trueX.GetBlock(1).Norml2() << ", global " <<
+                 sqrt(sumxim2) << endl;
+         }
+
+         delete gmres;
+         delete strumpack;
+         delete Arow;
+#else
+         cout << "Solving with STRUMPACK" << endl;
+
+#ifdef TEST_MULTIPLE_SP
+         const int Ns = 2;
+         std::vector<Operator*> Arows(Ns);
+         std::vector<STRUMPACKSolver*> strumpacks(Ns);
+
+         //Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+         for (int m=0; m<Ns; ++m)
+         {
+            Arows[m] = new STRUMPACKRowLocMatrix(A);
+
+            //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpacks[m] = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpacks[m]->SetPrintFactorStatistics(true);
+            strumpacks[m]->SetPrintSolveStatistics(false);
+            strumpacks[m]->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+            strumpacks[m]->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+            // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+            // strumpack->SetSymmetricPattern(true);
+            strumpacks[m]->SetOperator(*Arows[m]);
+            strumpacks[m]->SetFromCommandLine();
+            //Solver * precond = strumpack;
+
+            strumpacks[m]->Mult(B, X);
+
+            //delete strumpack;
+            //delete Arow;
+         }
+#else
+         Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+         STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+         strumpack->SetPrintFactorStatistics(true);
+         strumpack->SetPrintSolveStatistics(false);
+         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+         // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+         // strumpack->SetSymmetricPattern(true);
+         strumpack->SetOperator(*Arow);
+         strumpack->SetFromCommandLine();
+         //Solver * precond = strumpack;
+
+         cout << "Solving with strumpack one time" << endl;
+
+         strumpack->Mult(B, X);
+
+         delete strumpack;
+         delete Arow;
+#endif
+#endif
+      }
+      else
+      {
+         ParFiniteElementSpace *prec_fespace =
+            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+         HypreSolver *ams = new HypreAMS(A, prec_fespace);
+
+#ifdef HYPRE_DYLAN
+         {
+            Vector Xtmp(X);
+            ams->Mult(B,
+                      Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
+         }
+
+         HypreParMatrix H[3];
+#ifdef USE_HELMHOLTZ
+         for (int i=0; i<3; ++i)
+         {
+            GetHelmholtzMatrix(pmesh, i, &(H[i]));
+         }
+#endif
+
+#ifdef USE_CSL
+         HypreIAMS *iams = new HypreIAMS(A, H, strumpack, &trueX, &trueY,
+                                         (HypreAMS*) ams, argc, argv);
+#else
+         HypreIAMS *iams = new HypreIAMS(A, H, strumpack, NULL, NULL, (HypreAMS*) ams,
+                                         argc, argv);
+#endif
+
+         GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
+         //FGMRESSolver *gmres = new FGMRESSolver(fespace->GetComm());
+         //BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
+         //MINRESSolver *gmres = new MINRESSolver(fespace->GetComm());
+
+         gmres->SetOperator(A);
+         gmres->SetRelTol(1e-16);
+         gmres->SetMaxIter(1000);
+         gmres->SetPrintLevel(1);
+
+#ifdef SOLVE_A2
+         {
+            StopWatch chronoA2;
+            chronoA2.Clear();
+            chronoA2.Start();
+
+            HypreParMatrix * A2 = ParMult(&A, &Acopy);
+
+            chronoA2.Stop();
+            cout << "A2 setup time " << chronoA2.RealTime() << endl;
+
+            Vector AB(B);
+            A.Mult(B, AB);
+            gmres->SetOperator(*A2);
+
+            HypreSolver *ams2 = new HypreAMS(A, prec_fespace);
+            {
+               Vector Xtmp(X);
+               ams2->Mult(B,
+                          Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
+            }
+
+#ifdef ITER_A2
+            // Iteratively solve 0.5 (A^2 + S^2 + M^2) u^{k+1} = 0.5 (SM + MS) u^k + Ab
+
+            StopWatch chronoIterA2;
+            chronoIterA2.Clear();
+            chronoIterA2.Start();
+
+            HypreParMatrix * M2 = ParMult(&Mmat, &Mcopy);
+            HypreParMatrix * S2 = ParMult(&Smat, &Scopy);
+
+            HypreParMatrix * MS = ParMult(&Mmat, &Scopy);
+            HypreParMatrix * SM = ParMult(&Smat, &Mcopy);
+
+            HypreParMatrix * Bmat = ParAdd(SM, MS);
+            (*Bmat) *= 0.5;
+
+            // TODO: there must be a better way to form a sum of three matrices. Of course, we could define an operator that does 3 mat-vecs.
+            //HypreParMatrix * S2M2 = ParAdd(S2, M2);
+            //HypreParMatrix * iterMat = ParAdd(A2, S2M2);
+            HypreParMatrix * iterMat = ParAdd(A2, Bmat);
+
+            chronoIterA2.Stop();
+            cout << "Iter A2 setup time " << chronoIterA2.RealTime() << endl;
+
+            /*
+            HypreSolver *ams3 = new HypreAMS(*iterMat, prec_fespace);
+            {
+              Vector Xtmp(X);
+              ams3->Mult(B, Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
+            }
+            */
+
+            /*
+            // GMRES
+            gmres->SetOperator(*iterMat);
+            gmres->SetPreconditioner(*ams2);
+            */
+
+
+            //HypreBoomerAMG *amg = new HypreBoomerAMG(*iterMat);
+            HypreBoomerAMG *amg = new HypreBoomerAMG(*A2);
+
+            // PCG
+            HyprePCG *pcg = new HyprePCG(*iterMat);
+            //HyprePCG *pcg = new HyprePCG(*A2);
+            pcg->SetTol(1e-12);
+            pcg->SetMaxIter(10);
+            pcg->SetPrintLevel(2);
+            pcg->SetPreconditioner(*amg);
+
+            /*
+            // Strumpack linear solver
+            Operator * Arow = new STRUMPACKRowLocMatrix(*iterMat);
+
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpack->SetPrintFactorStatistics(true);
+            strumpack->SetPrintSolveStatistics(false);
+            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+            strumpack->SetOperator(*Arow);
+            strumpack->SetFromCommandLine();
+            */
+
+            Vector iterRHS(AB);
+            Vector iterU(AB);
+            Vector iterU0(AB);
+
+            iterU = 0.0;
+            iterU0 = 0.0;
+
+            bool iterate = true;
+            int numIter = 0;
+            while (iterate)
+            {
+               iterRHS = iterU;
+               iterRHS.Add(-1.0, iterU0);
+
+               cout << "Iteration " << numIter + 1 << ": diff norm " << iterRHS.Norml2() <<
+                    endl;
+
+               iterU0 = iterU;
+
+               Bmat->Mult(iterU0, iterRHS);
+               //iterRHS.Add(2.0, AB);
+               iterRHS.Add(1.0, AB);
+
+               //gmres->Mult(iterRHS, iterU);
+               pcg->Mult(iterRHS, iterU);
+               //strumpack->Mult(iterRHS, iterU);
+
+               numIter++;
+
+               if (numIter > 100)
+               {
+                  iterate = false;
+               }
+            }
+
+            //delete strumpack;
+            //delete Arow;
+
+            delete pcg;
+
+            X = iterU;
+#else
+            //HypreIAMS *iams2 = new HypreIAMS(*A2, (HypreAMS*) ams2, argc, argv);
+            //gmres->SetPreconditioner(*iams2);
+            cout << myid << ": Solving" << endl;
+            gmres->SetPreconditioner(*ams2);
+            gmres->Mult(AB, X);
+            cout << myid << ": Solved" << endl;
+            return 3;
+#endif
+         }
+#else
+         gmres->SetPreconditioner(*iams);
+         gmres->Mult(B, X);
+#endif
+#else
+         HypreGMRES *gmres = new HypreGMRES(A);
+         gmres->SetTol(1e-12);
+         gmres->SetMaxIter(100);
+         gmres->SetPrintLevel(10);
+
+#ifdef FORM_DEFINITE
+         HypreSolver *amsdef = new HypreAMS(Adef, prec_fespace);
+         gmres->SetPreconditioner(*amsdef);
+#else
+         gmres->SetPreconditioner(*ams);
+#endif
+         gmres->Mult(B, X);
+#endif
+
+         delete gmres;
+         //delete iams;
+         //delete ams;
+      }
+   }
+   else
+#endif
+   {
+      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+      //     preconditioner from hypre.
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      HypreSolver *ams = new HypreAMS(A, prec_fespace);
+      HyprePCG *pcg = new HyprePCG(A);
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*ams);
+      pcg->Mult(B, X);
+
+      delete pcg;
+      delete ams;
+   }
+
+   chrono.Stop();
+   cout << myid << ": Solver time " << chrono.RealTime() << endl;
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 14. Compute and print the L^2 norm of the error.
+   {
+      double err = x.ComputeL2Error(E);
+      Vector zeroVec(3);
+      zeroVec = 0.0;
+      VectorConstantCoefficient vzero(zeroVec);
+      ParGridFunction zerogf(fespace);
+      zerogf = 0.0;
+      double normE = zerogf.ComputeL2Error(E);
+      double normX = x.ComputeL2Error(vzero);
+      if (myid == 0)
+      {
+         cout << "|| E_h - E ||_{L^2} = " << err << endl;
+         cout << "|| E_h ||_{L^2} = " << normX << endl;
+         cout << "|| E ||_{L^2} = " << normE << endl;
+      }
+   }
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete a;
+   delete sigma;
+   delete muinv;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+      f(0) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/multigrid.hpp

@@ -0,0 +1,517 @@
+#pragma once
+
+#include "mfem.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
+{
+   //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
+   STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
+   strumpack->SetPrintFactorStatistics(true);
+   strumpack->SetPrintSolveStatistics(false);
+   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+   strumpack->SetOperator(*Arow);
+   strumpack->SetFromCommandLine();
+   return strumpack;
+}
+
+hypre_CSRMatrix* GetHypreParMatrixData(const HypreParMatrix & hypParMat)
+{
+   // First cast the parameter to a hypre_ParCSRMatrix
+   hypre_ParCSRMatrix * parcsr_op =
+      (hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(hypParMat);
+
+   MFEM_ASSERT(parcsr_op != NULL,"STRUMPACK: const_cast failed in SetOperator");
+
+   // Create the CSRMatrixMPI A_ by borrowing the internal data from a hypre_CSRMatrix.
+   return hypre_MergeDiagAndOffd(parcsr_op);
+}
+
+// Row and column offsets are assumed to be the same, for each process.
+// Array offsets stores process-local offsets with respect to the blocks. Process offsets are not included.
+HypreParMatrix* CreateHypreParMatrixFromBlocks(MPI_Comm comm,
+                                               Array<int> const& offsets, Array2D<HypreParMatrix*> const& blocks,
+                                               Array2D<SparseMatrix*> const& blocksSp,
+                                               Array2D<double> const& coefficient,
+                                               std::vector<std::vector<int> > const& blockProcOffsets,
+                                               std::vector<std::vector<int> > const& all_block_num_loc_rows)
+{
+   const int numBlocks = offsets.Size() - 1;
+   const int num_loc_rows = offsets[numBlocks];
+
+   int nprocs, rank;
+   MPI_Comm_rank(comm, &rank);
+   MPI_Comm_size(comm, &nprocs);
+
+   std::vector<int> all_num_loc_rows(nprocs);
+   std::vector<int> procOffsets(nprocs);
+   std::vector<std::vector<int> > procBlockOffsets(nprocs);
+
+   MPI_Allgather(&num_loc_rows, 1, MPI_INT, all_num_loc_rows.data(), 1, MPI_INT,
+                 comm);
+
+   int first_loc_row = 0;
+   int glob_nrows = 0;
+   procOffsets[0] = 0;
+   for (int i=0; i<nprocs; ++i)
+   {
+      glob_nrows += all_num_loc_rows[i];
+      if (i < rank)
+      {
+         first_loc_row += all_num_loc_rows[i];
+      }
+
+      if (i < nprocs-1)
+      {
+         procOffsets[i+1] = procOffsets[i] + all_num_loc_rows[i];
+      }
+
+      if (numBlocks > 0)
+      {
+         procBlockOffsets[i].resize(numBlocks);
+         procBlockOffsets[i][0] = 0;
+      }
+
+      for (int j=1; j<numBlocks; ++j)
+      {
+         procBlockOffsets[i][j] = procBlockOffsets[i][j-1] + all_block_num_loc_rows[j
+                                                                                    -1][i];
+      }
+   }
+
+   const int glob_ncols = glob_nrows;
+
+   std::vector<int> opI(num_loc_rows+1);
+   std::vector<int> cnt(num_loc_rows);
+
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      opI[i] = 0;
+      cnt[i] = 0;
+   }
+
+   opI[num_loc_rows] = 0;
+
+   Array2D<hypre_CSRMatrix*> csr_blocks(numBlocks, numBlocks);
+
+   // Loop over all blocks, to determine nnz for each row.
+   for (int i=0; i<numBlocks; ++i)
+   {
+      for (int j=0; j<numBlocks; ++j)
+      {
+         if (blocks(i, j) == NULL)
+         {
+            csr_blocks(i, j) = NULL;
+
+            if (blocksSp(i, j) != NULL)
+            {
+               const int nrows = blocksSp(i, j)->Height();
+               for (int k=0; k<nrows; ++k)
+               {
+                  const int rowg = offsets[i] + k;
+                  opI[rowg + 1] += blocksSp(i, j)->GetI()[k+1] - blocksSp(i, j)->GetI()[k];
+               }
+            }
+         }
+         else
+         {
+            MFEM_VERIFY(blocksSp(i, j) == NULL, "");
+
+            csr_blocks(i, j) = GetHypreParMatrixData(*(blocks(i, j)));
+
+            const int nrows = csr_blocks(i, j)->num_rows;
+
+            for (int k=0; k<nrows; ++k)
+            {
+               const int rowg = offsets[i] + k;
+               //(*(leftInjection(i, j)))[k]
+               opI[rowg + 1] += csr_blocks(i, j)->i[k+1] - csr_blocks(i, j)->i[k];
+            }
+         }
+      }
+   }
+
+   // Now opI[i] is nnz for row i-1. Do a partial sum to get offsets.
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      opI[i+1] += opI[i];
+   }
+
+   const int nnz = opI[num_loc_rows];
+
+   std::vector<HYPRE_Int> opJ(nnz);
+   std::vector<double> data(nnz);
+
+   // Loop over all blocks, to set matrix data.
+   for (int i=0; i<numBlocks; ++i)
+   {
+      for (int j=0; j<numBlocks; ++j)
+      {
+         if (csr_blocks(i, j) != NULL || blocksSp(i, j) != NULL)
+         {
+            const bool useCSR = (csr_blocks(i, j) != NULL);
+
+            const int nrows = useCSR ? csr_blocks(i, j)->num_rows : blocksSp(i,
+                                                                             j)->Height();
+            const double coef = coefficient(i, j);
+
+            int *Iarray = useCSR ? csr_blocks(i, j)->i : blocksSp(i, j)->GetI();
+
+            //const bool failure = (nrows != offsets[i+1] - offsets[i]);
+
+            MFEM_VERIFY(nrows == offsets[i+1] - offsets[i], "");
+
+            for (int k=0; k<nrows; ++k)
+            {
+               const int rowg = offsets[i] + k;  // process-local row
+               const int nnz_k = Iarray[k+1] - Iarray[k];
+               const int osk = Iarray[k];
+
+               for (int l=0; l<nnz_k; ++l)
+               {
+                  // Find the column process offset for the block.
+                  const int bcol = useCSR ? csr_blocks(i, j)->j[osk + l] : blocksSp(i,
+                                                                                    j)->GetJ()[osk + l];
+                  int bcolproc = 0;
+
+                  for (int p=1; p<nprocs; ++p)
+                  {
+                     if (blockProcOffsets[j][p] > bcol)
+                     {
+                        bcolproc = p-1;
+                        break;
+                     }
+                  }
+
+                  if (blockProcOffsets[j][nprocs - 1] <= bcol)
+                  {
+                     bcolproc = nprocs - 1;
+                  }
+
+                  const int colg = procOffsets[bcolproc] + procBlockOffsets[bcolproc][j] +
+                                   (bcol - blockProcOffsets[j][bcolproc]);
+
+                  if (colg < 0)
+                  {
+                     cout << "BUG, negative global column index" << endl;
+                  }
+
+                  opJ[opI[rowg] + cnt[rowg]] = colg;
+                  data[opI[rowg] + cnt[rowg]] = useCSR ? coef * csr_blocks(i,
+                                                                           j)->data[osk + l] : coef * blocksSp(i, j)->GetData()[osk + l];
+                  cnt[rowg]++;
+               }
+            }
+         }
+      }
+   }
+
+   bool cntCheck = true;
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      if (cnt[i] != opI[i+1] - opI[i])
+      {
+         cntCheck = false;
+      }
+   }
+
+   MFEM_VERIFY(cntCheck, "");
+
+   for (int i=0; i<numBlocks; ++i)
+   {
+      for (int j=0; j<numBlocks; ++j)
+      {
+         if (csr_blocks(i, j) != NULL)
+         {
+            hypre_CSRMatrixDestroy(csr_blocks(i, j));
+         }
+      }
+   }
+
+   std::vector<HYPRE_Int> rowStarts2(2);
+   rowStarts2[0] = first_loc_row;
+   rowStarts2[1] = first_loc_row + all_num_loc_rows[rank];
+
+   if (nnz > 0)
+   {
+      HYPRE_Int minJ = opJ[0];
+      HYPRE_Int maxJ = opJ[0];
+      for (int i=0; i<nnz; ++i)
+      {
+         minJ = std::min(minJ, opJ[i]);
+         maxJ = std::max(maxJ, opJ[i]);
+
+         if (opJ[i] >= glob_ncols)
+         {
+            cout << "Column indices out of range" << endl;
+         }
+      }
+   }
+
+   HypreParMatrix *hmat = new HypreParMatrix(comm, num_loc_rows, glob_nrows,
+                                             glob_ncols, (int*) opI.data(), (HYPRE_Int*) opJ.data(), (double*) data.data(),
+                                             (HYPRE_Int*) rowStarts2.data(), (HYPRE_Int*) rowStarts2.data());
+
+   return hmat;
+}
+
+class BlockMGSolver : public Solver
+{
+private:
+   /// The linear system matrix
+   Array2D<HypreParMatrix *>&
+   Af;  // TODO: remove this, as it is used only in the constructor
+   Array2D<double>&
+   Acoef;  // TODO: remove this, as it is used only in the constructor
+   vector<Array<int>> Aoffsets;
+   vector<Array<int>> Poffsets_i;
+   vector<Array<int>> Poffsets_j;
+   std::vector<Array2D<HypreParMatrix *>> A;
+   std::vector<HypreParMatrix *>& P;
+   std::vector<BlockOperator *> BlkP;
+   std::vector<BlockOperator *> BlkA;
+   std::vector<BlockOperator *> S;
+   HypreParMatrix * Ac;
+   int numGrids, numBlocks;
+   STRUMPACKSolver *invAc = nullptr;
+   double theta = 0.5;
+
+public:
+   BlockMGSolver(const int height, const int width, Array2D<HypreParMatrix *>& Af_,
+                 Array2D<double>& Acoef_, std::vector<HypreParMatrix *>& P_);
+
+   virtual void SetOperator(const Operator &op) {}
+
+   virtual void SetTheta(const double a) { theta = a; }
+
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~BlockMGSolver();
+};
+
+BlockMGSolver::BlockMGSolver(const int height, const int width,
+                             Array2D<HypreParMatrix *>& Af_, Array2D<double>& Acoef_,
+                             std::vector<HypreParMatrix *>& P_)
+   : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_)
+{
+   numBlocks = Af.NumRows();
+   MFEM_VERIFY(Af.NumCols() == numBlocks, "");
+   numGrids = P.size();
+   BlkP.resize(numGrids);
+   BlkA.resize(numGrids+1);
+   S.resize(numGrids);
+   A.resize(numGrids + 1);
+   A[numGrids] = Af;
+   Aoffsets.resize(numGrids+1);
+   Poffsets_i.resize(numGrids);
+   Poffsets_j.resize(numGrids);
+   // Construct Bilinear form Matrices on each level
+   for (int k = numGrids ; k > 0; k--)
+   {
+      A[k - 1].SetSize(numBlocks,numBlocks);
+      Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
+      for (int i=0; i<numBlocks; i++)
+      {
+         Aoffsets[k][i+1] = A[k](i,i)->Height();
+      }
+
+      Aoffsets[k].PartialSum();
+      BlkA[k] = new BlockOperator(Aoffsets[k]);
+      S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
+
+      for (int i=0; i<numBlocks; i++)
+      {
+         for (int j=0; j<numBlocks; j++)
+         {
+            if (A[k](i,j) == NULL)
+            {
+               A[k - 1](i,j) = NULL;
+            }
+            else
+            {
+               A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
+               BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
+            }
+         }
+
+         HypreSmoother *S_i = new HypreSmoother;
+         S_i->SetType(HypreSmoother::Jacobi);
+         S_i->SetOperator(*(A[k](i,i)));
+
+         S[k - 1]->SetBlock(i,i,S_i);
+      }
+
+      Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
+      Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
+      for (int i=0; i<numBlocks; i++)
+      {
+         Poffsets_i[k-1][i+1] = P[k-1]->Height();
+         Poffsets_j[k-1][i+1] = P[k-1]->Width();
+      }
+      Poffsets_i[k-1].PartialSum();
+      Poffsets_j[k-1].PartialSum();
+
+      BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
+      for (int i=0; i<numBlocks; i++)
+      {
+         BlkP[k-1]->SetBlock(i,i,P[k-1]);
+      }
+   }
+   // Set up coarse solve operator
+   // Convert the coarse grid blockmatrix to a HypreParMatrix
+   Array<int> offsets(numBlocks+1);
+   offsets[0]=0;
+   for (int i=0; i<numBlocks; i++)
+   {
+      offsets[i+1]=A[0](i,i)->Height();
+   }
+
+   offsets.PartialSum();
+
+   BlkA[0] = new BlockOperator(offsets);
+
+   Array2D<SparseMatrix*> Asp;
+   //Array2D<double> Acoef;
+   Asp.SetSize(numBlocks,numBlocks);
+   //Acoef.SetSize(numBlocks,numBlocks);
+   for (int i=0; i<numBlocks; i++)
+   {
+      for (int j=0; j<numBlocks; j++)
+      {
+         if (A[0](i,j) != NULL)
+         {
+            BlkA[0]->SetBlock(i, j, A[0](i,j), Acoef(i,j));
+         }
+
+         Asp(i,j) = NULL;
+         //Acoef(i,j) = 1.0;
+      }
+   }
+
+   // Convert to HypreParMatrix
+   HypreParMatrix * Ac;
+
+   std::vector<std::vector<int> > blockProcOffsets(numBlocks);
+   std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
+
+   {
+      int nprocs, rank;
+      MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+      MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+
+      std::vector<int> allnumrows(nprocs);
+      const int blockNumRows = A[0](0,0)->Height();
+      MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT,
+                    MPI_COMM_WORLD);
+
+      for (int b=0; b<numBlocks; ++b)
+      {
+         blockProcOffsets[b].resize(nprocs);
+         all_block_num_loc_rows[b].resize(nprocs);
+      }
+
+      blockProcOffsets[0][0] = 0;
+      for (int i=0; i<nprocs-1; ++i)
+      {
+         blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
+      }
+
+      for (int i=0; i<nprocs; ++i)
+      {
+         for (int b=0; b<numBlocks; ++b)
+         {
+            all_block_num_loc_rows[b][i] = allnumrows[i];
+         }
+
+         for (int b=1; b<numBlocks; ++b)
+         {
+            blockProcOffsets[b][i] = blockProcOffsets[0][i];
+         }
+      }
+   }
+
+   Ac = CreateHypreParMatrixFromBlocks(MPI_COMM_WORLD, offsets, A[0], Asp,
+                                       Acoef, blockProcOffsets, all_block_num_loc_rows);
+
+   invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), MPI_COMM_WORLD);
+
+   delete Ac;
+}
+
+void BlockMGSolver::Mult(const Vector &r, Vector &z) const
+{
+   // Residual vectors
+   std::vector<Vector> rv(numGrids + 1);
+   // correction vectors
+   std::vector<Vector> zv(numGrids + 1);
+   // allocation
+   for (int i = 0; i <= numGrids ; i++)
+   {
+      int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
+      rv[i].SetSize(n);
+      zv[i].SetSize(n);
+   }
+   // Initial residual
+   rv[numGrids] = r;
+
+   // smooth and update residuals down to the coarsest level
+   for (int i = numGrids; i > 0 ; i--)
+   {
+      // Pre smooth
+      S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+      // compute residual
+      int n = BlkA[i]->Width();
+      Vector w(n);
+      BlkA[i]->Mult(zv[i], w);
+      rv[i] -= w;
+      // Restrict
+      BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
+   }
+
+   // Coarse grid Solve
+   invAc->Mult(rv[0], zv[0]);
+   //
+   for (int i = 1; i <= numGrids ; i++)
+   {
+      // Prolong correction
+      Vector u(BlkP[i - 1]->Height());
+      BlkP[i - 1]->Mult(zv[i - 1], u);
+      // Update correction
+      zv[i] += u;
+      // Update residual
+      Vector v(BlkA[i]->Height());
+      BlkA[i]->Mult(u, v); rv[i] -= v;
+      // Post smooth
+      S[i - 1]->Mult(rv[i], v); v *= theta;
+      // Update correction
+      zv[i] += v;
+   }
+   z = zv[numGrids];
+}
+
+BlockMGSolver::~BlockMGSolver()
+{
+   for (int i = numGrids - 1; i >= 0 ; i--)
+   {
+      delete S[i];
+      delete BlkP[i];
+      delete BlkA[i];
+      for (int j=0; j<numBlocks; j++)
+      {
+         for (int k=0; k<numBlocks; k++)
+         {
+            delete A[i](j,k);
+         }
+      }
+      A[i].DeleteAll();
+   }
+   delete BlkA[numGrids];
+   delete invAc;
+   A.clear();
+}
+
+} // namespace mfem

examples/multigriddd.hpp

@@ -0,0 +1,869 @@
+#ifndef BGMULTIGRID
+#define BGMULTIGRID
+
+#include "mfem.hpp"
+
+using namespace std;
+
+//namespace mfem {
+namespace blockgmg
+{
+
+// Row and column offsets are assumed to be the same, for each process.
+// Array offsets stores process-local offsets with respect to the blocks. Process offsets are not included.
+HypreParMatrix* CreateHypreParMatrixFromBlocks2(MPI_Comm comm,
+                                                Array<int> const& offsets, Array2D<HypreParMatrix*> const& blocks,
+                                                Array2D<SparseMatrix*> const& blocksSp,
+                                                Array2D<double> const& coefficient,
+                                                std::vector<std::vector<int> > const& blockProcOffsets,
+                                                std::vector<std::vector<int> > const& all_block_num_loc_rows);
+
+class BlockMGSolver : public Solver
+{
+private:
+   /// The linear system matrix
+   Array2D<HypreParMatrix *>&
+   Af;  // TODO: remove this, as it is used only in the constructor
+   Array2D<double>&
+   Acoef;  // TODO: remove this, as it is used only in the constructor
+   vector<Array<int>> Aoffsets;
+   vector<Array<int>> Poffsets_i;
+   vector<Array<int>> Poffsets_j;
+   std::vector<Array2D<HypreParMatrix *>> A;
+   std::vector<HypreParMatrix *> const& P;
+   std::vector<BlockOperator *> BlkP;
+   std::vector<BlockOperator *> BlkA;
+   std::vector<BlockOperator *> S;
+   HypreParMatrix * Ac;
+   SparseMatrix AcSp;
+   int numGrids, numBlocks;
+   //STRUMPACKSolver *invAc = nullptr;
+   Operator *invAc = nullptr;
+   double theta = 0.5;
+
+public:
+   BlockMGSolver(MPI_Comm comm, const int height, const int width,
+                 Array2D<HypreParMatrix *>& Af_,
+                 Array2D<double>& Acoef_, std::vector<HypreParMatrix *> const& P_)
+      : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_)
+   {
+      numBlocks = Af.NumRows();
+      MFEM_VERIFY(Af.NumCols() == numBlocks, "");
+      numGrids = P.size();
+      BlkP.resize(numGrids);
+      BlkA.resize(numGrids+1);
+      S.resize(numGrids);
+      A.resize(numGrids + 1);
+      A[numGrids] = Af;
+      Aoffsets.resize(numGrids+1);
+      Poffsets_i.resize(numGrids);
+      Poffsets_j.resize(numGrids);
+      // Construct Bilinear form Matrices on each level
+      for (int k = numGrids ; k > 0; k--)
+      {
+         A[k - 1].SetSize(numBlocks,numBlocks);
+         Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
+         for (int i=0; i<numBlocks; i++)
+         {
+            Aoffsets[k][i+1] = A[k](i,i)->Height();
+         }
+
+         Aoffsets[k].PartialSum();
+         BlkA[k] = new BlockOperator(Aoffsets[k]);
+         S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
+
+         for (int i=0; i<numBlocks; i++)
+         {
+            for (int j=0; j<numBlocks; j++)
+            {
+               if (A[k](i,j) == NULL)
+               {
+                  A[k - 1](i,j) = NULL;
+               }
+               else
+               {
+                  A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
+                  BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
+               }
+            }
+
+            HypreSmoother *S_i = new HypreSmoother;
+            S_i->SetType(HypreSmoother::Jacobi);
+            S_i->SetOperator(*(A[k](i,i)));
+
+            S[k - 1]->SetBlock(i,i,S_i);
+         }
+
+         Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
+         Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
+         for (int i=0; i<numBlocks; i++)
+         {
+            Poffsets_i[k-1][i+1] = P[k-1]->Height();
+            Poffsets_j[k-1][i+1] = P[k-1]->Width();
+         }
+         Poffsets_i[k-1].PartialSum();
+         Poffsets_j[k-1].PartialSum();
+
+         BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
+         for (int i=0; i<numBlocks; i++)
+         {
+            BlkP[k-1]->SetBlock(i,i,P[k-1]);
+         }
+      }
+      // Set up coarse solve operator
+      // Convert the coarse grid blockmatrix to a HypreParMatrix
+      Array<int> offsets(numBlocks+1);
+      offsets[0]=0;
+      for (int i=0; i<numBlocks; i++)
+      {
+         offsets[i+1]=A[0](i,i)->Height();
+      }
+
+      offsets.PartialSum();
+
+      BlkA[0] = new BlockOperator(offsets);
+
+      Array2D<SparseMatrix*> Asp;
+      //Array2D<double> Acoef;
+      Asp.SetSize(numBlocks,numBlocks);
+      //Acoef.SetSize(numBlocks,numBlocks);
+      for (int i=0; i<numBlocks; i++)
+      {
+         for (int j=0; j<numBlocks; j++)
+         {
+            if (A[0](i,j) != NULL)
+            {
+               BlkA[0]->SetBlock(i, j, A[0](i,j), Acoef(i,j));
+            }
+
+            Asp(i,j) = NULL;
+            //Acoef(i,j) = 1.0;
+         }
+      }
+
+      // Convert to HypreParMatrix
+      //HypreParMatrix * Ac;
+
+      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
+      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
+
+      {
+         int nprocs, rank;
+         MPI_Comm_rank(comm, &rank);
+         MPI_Comm_size(comm, &nprocs);
+
+         std::vector<int> allnumrows(nprocs);
+         const int blockNumRows = A[0](0,
+                                       0)->Height();    // TODO: Not valid if blocks are of different size
+         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT, comm);
+
+         for (int b=0; b<numBlocks; ++b)
+         {
+            blockProcOffsets[b].resize(nprocs);
+            all_block_num_loc_rows[b].resize(nprocs);
+         }
+
+         blockProcOffsets[0][0] = 0;
+         for (int i=0; i<nprocs-1; ++i)
+         {
+            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
+         }
+
+         for (int i=0; i<nprocs; ++i)
+         {
+            for (int b=0; b<numBlocks; ++b)
+            {
+               all_block_num_loc_rows[b][i] = allnumrows[i];
+            }
+
+            for (int b=1; b<numBlocks; ++b)
+            {
+               blockProcOffsets[b][i] = blockProcOffsets[0][i];
+            }
+         }
+      }
+
+      Ac = CreateHypreParMatrixFromBlocks2(comm, offsets, A[0], Asp,
+                                           Acoef, blockProcOffsets, all_block_num_loc_rows);
+
+#ifdef MFEM_USE_STRUMPACK
+      invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), comm);
+      delete Ac;
+#else
+      Ac->GetDiag(AcSp);  // AcSp does not own the data
+      UMFPackSolver *umf_solver = new UMFPackSolver();
+      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver->SetOperator(AcSp);
+      invAc = umf_solver;
+#endif
+   }
+
+   virtual void SetOperator(const Operator &op) {}
+
+   virtual void SetTheta(const double a) { theta = a; }
+
+   virtual void Mult(const Vector &r, Vector &z) const
+   {
+      // Residual vectors
+      std::vector<Vector> rv(numGrids + 1);
+      // correction vectors
+      std::vector<Vector> zv(numGrids + 1);
+      // allocation
+      for (int i = 0; i <= numGrids ; i++)
+      {
+         int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
+         rv[i].SetSize(n);
+         zv[i].SetSize(n);
+      }
+      // Initial residual
+      rv[numGrids] = r;
+
+      // smooth and update residuals down to the coarsest level
+      for (int i = numGrids; i > 0 ; i--)
+      {
+         // Pre smooth
+         S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+         // compute residual
+         int n = BlkA[i]->Width();
+         Vector w(n);
+         BlkA[i]->Mult(zv[i], w);
+         rv[i] -= w;
+         // Restrict
+         BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
+      }
+
+      // Coarse grid Solve
+      invAc->Mult(rv[0], zv[0]);
+      //
+      for (int i = 1; i <= numGrids ; i++)
+      {
+         // Prolong correction
+         Vector u(BlkP[i - 1]->Height());
+         BlkP[i - 1]->Mult(zv[i - 1], u);
+         // Update correction
+         zv[i] += u;
+         // Update residual
+         Vector v(BlkA[i]->Height());
+         BlkA[i]->Mult(u, v); rv[i] -= v;
+         // Post smooth
+         S[i - 1]->Mult(rv[i], v); v *= theta;
+         // Update correction
+         zv[i] += v;
+      }
+      z = zv[numGrids];
+   }
+
+   virtual ~BlockMGSolver()
+   {
+      for (int i = numGrids - 1; i >= 0 ; i--)
+      {
+         delete S[i];
+         delete BlkP[i];
+         delete BlkA[i];
+         for (int j=0; j<numBlocks; j++)
+         {
+            for (int k=0; k<numBlocks; k++)
+            {
+               delete A[i](j,k);
+            }
+         }
+         A[i].DeleteAll();
+      }
+      delete BlkA[numGrids];
+      delete invAc;
+      A.clear();
+   }
+
+#ifdef MFEM_USE_STRUMPACK
+   STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
+   {
+      //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      return strumpack;
+   }
+#endif
+
+};
+
+#define SWTIMING
+#define SERIAL_PROLONGATION
+#define ITERATIVE_COARSE_SOLVE
+#define SPARSE_JACOBI
+//#define SPARSE_ICHOLESKY
+//#define SPARSE_ILU
+//#define COARSE_PA
+//#define BLOCK_DIAG
+
+//#define COARSE_AMS
+
+class BlockMGPASolver : public Solver
+{
+private:
+   /// The linear system matrix
+   Array2D<Operator*>&
+   Af;  // TODO: remove this, as it is used only in the constructor
+   Array2D<double>&
+   Acoef;  // TODO: remove this, as it is used only in the constructor
+   vector<Array<int>> Aoffsets;
+   vector<Array<int>> Poffsets_i;
+   vector<Array<int>> Poffsets_j;
+   Array<int> Coffsets;
+
+   std::vector<Array2D<Operator*>> A;
+#ifdef SERIAL_PROLONGATION
+   std::vector<SparseMatrix *> P;
+#else
+   std::vector<HypreParMatrix *> const& P;
+#endif
+   std::vector<Vector*> const& diag;
+   std::vector<Operator*> Pt;
+   std::vector<BlockOperator *> BlkP;
+   std::vector<BlockOperator *> BlkA;
+   std::vector<BlockOperator *> S;
+   std::vector<OperatorJacobiSmoother*> Jacobi;
+
+   HypreParMatrix * Ac;
+   SparseMatrix AcSp;
+   int numGrids, numBlocks;
+   //STRUMPACKSolver *invAc = nullptr;
+   Operator *invAc = nullptr;
+   double theta = 0.5;
+
+   mutable std::vector<Vector> rv, zv;
+   mutable Vector u, v, w;
+
+   Vector diagAc;
+   Array<int> emptyEssDof;
+   OperatorJacobiSmoother *JacobiAc;
+#ifdef SPARSE_ICHOLESKY
+   Solver *iCholAc;
+#endif
+#ifdef SPARSE_ILU
+   Solver *iluAc;
+#endif
+
+#ifdef BLOCK_DIAG
+   HypreParMatrix *AcDiag;
+   SparseMatrix AcDiagSp;
+#endif
+
+public:
+   BlockMGPASolver(MPI_Comm comm, const int height, const int width,
+                   Array2D<Operator*>& Af_, Array2D<double>& Acoef_,
+#ifdef COARSE_PA
+                   BlockOperator *BlkAc,
+#else
+                   Array2D<HypreParMatrix*> const& BlkAc,
+#endif
+#ifdef COARSE_AMS
+                   ParFiniteElementSpace* coarseFespace,
+#endif
+                   std::vector<HypreParMatrix *> const& P_, std::vector<Vector*> const& diag_,
+                   Array<int>& ess_tdof_list)
+#ifdef SERIAL_PROLONGATION
+      : Solver(height, width), Af(Af_), Acoef(Acoef_), diag(diag_)
+#else
+      : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_), diag(diag_)
+#endif
+   {
+      timeMult = 0.0;
+      timeMultAc = 0.0;
+      timeMultPresmooth = 0.0;
+      timeMultResidual = 0.0;
+      timeMultRestrict = 0.0;
+
+#ifdef SERIAL_PROLONGATION
+      P.resize(P_.size());
+      for (int i=0; i<P_.size(); ++i)
+      {
+         P[i] = new SparseMatrix();
+         P_[i]->GetDiag(*P[i]);
+         P[i]->BuildTranspose();
+      }
+#endif
+
+      numBlocks = Af.NumRows();
+      MFEM_VERIFY(Af.NumCols() == numBlocks, "");
+#ifndef COARSE_PA
+      MFEM_VERIFY(BlkAc.NumCols() == numBlocks && BlkAc.NumRows() == numBlocks, "");
+#endif
+      numGrids = P.size();
+      MFEM_VERIFY(diag.size() == numBlocks, "");
+      BlkP.resize(numGrids);
+      BlkA.resize(numGrids+1);
+      Pt.resize(numGrids);
+      S.resize(numGrids);
+      A.resize(numGrids + 1);
+      A[numGrids] = Af;
+      Aoffsets.resize(numGrids+1);
+      Poffsets_i.resize(numGrids);
+      Poffsets_j.resize(numGrids);
+
+      Jacobi.resize(numBlocks);
+      for (int i=0; i<numBlocks; ++i)
+      {
+         Jacobi[i] = new OperatorJacobiSmoother(*(diag[i]), ess_tdof_list, 1.0);
+      }
+
+      // Construct Bilinear form Matrices on each level
+      for (int k = numGrids ; k > 0; k--)
+      {
+         A[k - 1].SetSize(numBlocks,numBlocks);
+         Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
+         for (int i=0; i<numBlocks; i++)
+         {
+            Aoffsets[k][i+1] = A[k](i,i)->Height();
+         }
+
+         Aoffsets[k].PartialSum();
+         BlkA[k] = new BlockOperator(Aoffsets[k]);
+         S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
+         Pt[k - 1] = new TransposeOperator(P[k - 1]);
+
+         for (int i=0; i<numBlocks; i++)
+         {
+            for (int j=0; j<numBlocks; j++)
+            {
+               if (A[k](i,j) == NULL)
+               {
+                  A[k - 1](i,j) = NULL;
+               }
+               else
+               {
+                  //A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
+                  A[k - 1](i,j) = new TripleProductOperator(Pt[k - 1], A[k](i,j), P[k - 1], false,
+                                                            false, false);
+                  BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
+               }
+            }
+
+            if (k == numGrids)  // finest level
+            {
+               S[k - 1]->SetBlock(i,i,Jacobi[i]);
+            }
+            else
+            {
+               S[k - 1]->SetBlock(i,i, new TripleProductOperator(Pt[k], &(S[k]->GetBlock(i,i)),
+                                                                 P[k], false,
+                                                                 false, false));   // TODO: get a diagonal on each level
+            }
+         }
+
+         Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
+         Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
+         for (int i=0; i<numBlocks; i++)
+         {
+            Poffsets_i[k-1][i+1] = P[k-1]->Height();
+            Poffsets_j[k-1][i+1] = P[k-1]->Width();
+         }
+         Poffsets_i[k-1].PartialSum();
+         Poffsets_j[k-1].PartialSum();
+
+         BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
+         for (int i=0; i<numBlocks; i++)
+         {
+            BlkP[k-1]->SetBlock(i,i,P[k-1]);
+         }
+      }
+      // Set up coarse solve operator
+      // Convert the coarse grid blockmatrix to a HypreParMatrix
+      Coffsets.SetSize(numBlocks+1);
+      Coffsets[0]=0;
+      for (int i=0; i<numBlocks; i++)
+      {
+         Coffsets[i+1]=A[0](i,i)->Height();
+#ifndef COARSE_PA
+         MFEM_VERIFY(BlkAc(i,i)->Height() == A[0](i,i)->Height(), "");
+         MFEM_VERIFY(BlkAc(i,i)->Width() == A[0](i,i)->Width(), "");
+#endif
+      }
+      Coffsets.PartialSum();
+
+#ifdef COARSE_PA
+      BlkA[0] = BlkAc;
+#else
+      BlkA[0] = new BlockOperator(Coffsets);
+#endif
+
+      Array2D<SparseMatrix*> Asp;
+      //Array2D<double> Acoef;
+      Asp.SetSize(numBlocks,numBlocks);
+      //Acoef.SetSize(numBlocks,numBlocks);
+      for (int i=0; i<numBlocks; i++)
+      {
+         for (int j=0; j<numBlocks; j++)
+         {
+#ifndef COARSE_PA
+            MFEM_VERIFY((A[0](i,j) == NULL) == (BlkAc(i,j) == NULL), "");
+
+            if (BlkAc(i,j) != NULL)
+            {
+               BlkA[0]->SetBlock(i, j, BlkAc(i,j), Acoef(i,j));
+            }
+#endif
+
+            Asp(i,j) = NULL;
+            //Acoef(i,j) = 1.0;
+         }
+      }
+
+#ifdef COARSE_PA
+      CGSolver *cg_solver = new CGSolver();
+      cg_solver->SetAbsTol(1.0e-6);
+      cg_solver->SetRelTol(1.0e-6);
+      cg_solver->SetMaxIter(1000);
+      cg_solver->SetOperator(*BlkAc);
+      cg_solver->SetPrintLevel(0);
+      cg_solver->iterative_mode = false;
+      invAc = cg_solver;
+#else
+      // Convert to HypreParMatrix
+      HypreParMatrix * Ac;
+
+      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
+      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
+
+      {
+         int nprocs, rank;
+         MPI_Comm_rank(comm, &rank);
+         MPI_Comm_size(comm, &nprocs);
+
+         std::vector<int> allnumrows(nprocs);
+         // TODO: Not valid if blocks are of different size
+         const int blockNumRows = BlkAc(0,0)->Height();
+
+         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT, comm);
+
+         for (int b=0; b<numBlocks; ++b)
+         {
+            blockProcOffsets[b].resize(nprocs);
+            all_block_num_loc_rows[b].resize(nprocs);
+         }
+
+         blockProcOffsets[0][0] = 0;
+         for (int i=0; i<nprocs-1; ++i)
+         {
+            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
+         }
+
+         for (int i=0; i<nprocs; ++i)
+         {
+            for (int b=0; b<numBlocks; ++b)
+            {
+               all_block_num_loc_rows[b][i] = allnumrows[i];
+            }
+
+            for (int b=1; b<numBlocks; ++b)
+            {
+               blockProcOffsets[b][i] = blockProcOffsets[0][i];
+            }
+         }
+      }
+
+      Ac = CreateHypreParMatrixFromBlocks2(comm, Coffsets, BlkAc, Asp,
+                                           Acoef, blockProcOffsets, all_block_num_loc_rows);
+
+#ifdef BLOCK_DIAG
+      {
+         Array2D<HypreParMatrix*> BlkAcDiag(numBlocks, numBlocks);
+         Array2D<double> DiagCoef(numBlocks, numBlocks);
+         DiagCoef = 0.0;
+         for (int i=0; i<numBlocks; ++i)
+         {
+            DiagCoef(i,i) = Acoef(i,i);
+            for (int j=0; j<numBlocks; ++j)
+            {
+               BlkAcDiag(i,j) = NULL;
+            }
+
+            BlkAcDiag(i,i) = BlkAc(i,i);
+         }
+
+         AcDiag = CreateHypreParMatrixFromBlocks2(comm, Coffsets, BlkAcDiag, Asp,
+                                                  DiagCoef, blockProcOffsets, all_block_num_loc_rows);
+
+         AcDiag->GetDiag(AcDiagSp);
+         //delete AcDiag;
+      }
+#endif
+
+#ifdef MFEM_USE_STRUMPACK
+      invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), comm);
+      delete Ac;
+#else
+      Ac->GetDiag(AcSp);  // AcSp does not own the data
+      delete Ac;
+
+#ifdef ITERATIVE_COARSE_SOLVE
+      //CGSolver *cg_solver = new CGSolver(comm);
+      CGSolver *cg_solver = new CGSolver();
+      cg_solver->SetAbsTol(1.0e-6);
+      cg_solver->SetRelTol(1.0e-6);
+      cg_solver->SetMaxIter(1000);
+      cg_solver->SetOperator(AcSp);
+      cg_solver->SetPrintLevel(0);
+      cg_solver->iterative_mode = false;
+#ifdef SPARSE_JACOBI
+      AcSp.GetDiag(diagAc);
+      JacobiAc = new OperatorJacobiSmoother(diagAc, emptyEssDof);
+      cg_solver->SetPreconditioner(*JacobiAc);
+#endif
+#ifdef SPARSE_ICHOLESKY
+      {
+         Vector tmpX(AcSp.Height());
+         Vector tmpY(AcSp.Height());
+         tmpX = 1.0;
+         tmpY = 0.0;
+#ifdef BLOCK_DIAG
+         AcDiagSp.Finalize();
+         AcDiagSp.SortColumnIndices();
+
+         AcDiagSp.Mult(tmpX, tmpY);
+#else
+         AcSp.Finalize();
+         AcSp.SortColumnIndices();
+
+         AcSp.Mult(tmpX, tmpY);
+#endif
+      }
+#ifdef BLOCK_DIAG
+      iCholAc = new IncompleteCholesky(AcDiagSp);
+#else
+      iCholAc = new IncompleteCholesky(AcSp);
+#endif
+      cg_solver->SetPreconditioner(*iCholAc);
+      cg_solver->SetPrintLevel(-1);
+#endif // SPARSE_ICHOLESKY
+
+#ifdef SPARSE_ILU
+      {
+         Vector tmpX(AcSp.Height());
+         Vector tmpY(AcSp.Height());
+         tmpX = 1.0;
+         tmpY = 0.0;
+#ifdef BLOCK_DIAG
+         AcDiagSp.Finalize();
+         AcDiagSp.SortColumnIndices();
+
+         AcDiagSp.Mult(tmpX, tmpY);
+#else
+         AcSp.Finalize();
+         AcSp.SortColumnIndices();
+
+         AcSp.Mult(tmpX, tmpY);
+#endif
+      }
+#ifdef BLOCK_DIAG
+      iluAc = new ILUcusparse(AcDiagSp);
+#else
+      iluAc = new ILUcusparse(AcSp);
+#endif
+      cg_solver->SetPreconditioner(*iluAc);
+      cg_solver->SetPrintLevel(0);
+#endif // SPARSE_ILU
+
+#ifdef COARSE_AMS
+#ifndef COARSE_PA
+      MFEM_VERIFY(numBlocks == 4, "");
+      // TODO: just set 2 AMS solvers for E and H.
+
+      if (coarseFespace != NULL)
+      {
+         BlockDiagonalPreconditioner *blkAMS = new BlockDiagonalPreconditioner(Coffsets);
+
+         for (int i=0; i<numBlocks; ++i)
+         {
+            HypreAMS *ams = new HypreAMS(*BlkAc(i,i), coarseFespace);
+            blkAMS->SetDiagonalBlock(i, ams);
+         }
+
+         cg_solver->SetPreconditioner(*blkAMS);
+      }
+#endif
+#endif
+
+      invAc = cg_solver;
+#else
+      UMFPackSolver *umf_solver = new UMFPackSolver();
+      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver->SetOperator(AcSp);
+      invAc = umf_solver;
+#endif
+#endif
+#endif  // COARSE_PA
+
+      // Residual vectors
+      rv.resize(numGrids + 1);
+      // correction vectors
+      zv.resize(numGrids + 1);
+      // allocation
+      for (int i = 0; i <= numGrids ; i++)
+      {
+         int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
+
+         rv[i].SetSize(n);
+         zv[i].SetSize(n);
+         rv[i].UseDevice(true);
+         zv[i].UseDevice(true);
+      }
+   }
+
+   virtual void SetOperator(const Operator &op) {}
+
+   virtual void SetTheta(const double a) { theta = a; }
+
+   virtual void Mult(const Vector &r, Vector &z) const
+   {
+#ifdef SWTIMING
+      StopWatch sw;
+      sw.Clear();
+      sw.Start();
+#endif
+
+      // Initial residual
+      rv[numGrids] = r;
+
+      // smooth and update residuals down to the coarsest level
+      for (int i = numGrids; i > 0 ; i--)
+      {
+         // Pre smooth
+#ifdef SWTIMING
+         StopWatch sws;
+         sws.Clear();
+         sws.Start();
+#endif
+
+         S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+
+#ifdef SWTIMING
+         sws.Stop();
+         timeMultPresmooth += sws.RealTime();
+#endif
+
+         // compute residual
+         int n = BlkA[i]->Width();
+         w.SetSize(n);
+         w.UseDevice(true);
+#ifdef SWTIMING
+         StopWatch swop;
+         swop.Clear();
+         swop.Start();
+#endif
+         BlkA[i]->Mult(zv[i], w);
+         rv[i] -= w;
+#ifdef SWTIMING
+         swop.Stop();
+         timeMultResidual += swop.RealTime();
+#endif
+         // Restrict
+#ifdef SWTIMING
+         StopWatch swr;
+         swr.Clear();
+         swr.Start();
+#endif
+         BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
+#ifdef SWTIMING
+         swr.Stop();
+         timeMultRestrict += swr.RealTime();
+#endif
+      }
+
+#ifdef SWTIMING
+      StopWatch swAc;
+      swAc.Clear();
+      swAc.Start();
+#endif
+
+      // Coarse grid Solve
+      invAc->Mult(rv[0], zv[0]);
+      //
+
+#ifdef SWTIMING
+      swAc.Stop();
+      timeMultAc += swAc.RealTime();
+#endif
+
+      for (int i = 1; i <= numGrids ; i++)
+      {
+         // Prolong correction
+         u.SetSize(BlkP[i - 1]->Height());
+         u.UseDevice(true);
+
+         BlkP[i - 1]->Mult(zv[i - 1], u);
+         // Update correction
+         zv[i] += u;
+         // Update residual
+         v.SetSize(BlkA[i]->Height());
+         v.UseDevice(true);
+
+         BlkA[i]->Mult(u, v); rv[i] -= v;
+         // Post smooth
+         S[i - 1]->Mult(rv[i], v); v *= theta;
+         // Update correction
+         zv[i] += v;
+      }
+      z = zv[numGrids];
+
+#ifdef SWTIMING
+      sw.Stop();
+      timeMult += sw.RealTime();
+#endif
+   }
+
+   virtual ~BlockMGPASolver()
+   {
+      for (int i = numGrids - 1; i >= 0 ; i--)
+      {
+         delete S[i];
+         delete BlkP[i];
+         delete BlkA[i];
+         for (int j=0; j<numBlocks; j++)
+         {
+            for (int k=0; k<numBlocks; k++)
+            {
+               delete A[i](j,k);
+            }
+         }
+         A[i].DeleteAll();
+      }
+      delete BlkA[numGrids];
+      delete invAc;
+      A.clear();
+
+#ifdef SPARSE_ICHOLESKY
+      delete iCholAc;
+#endif
+#ifdef SPARSE_ILU
+      delete iluAc;
+#endif
+   }
+
+#ifdef MFEM_USE_STRUMPACK
+   STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
+   {
+      //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      return strumpack;
+   }
+#endif
+
+   mutable double timeMult, timeMultAc, timeMultPresmooth, timeMultResidual,
+   timeMultRestrict;
+};
+
+}
+//} // namespace mfem
+
+#endif // BGMULTIGRID

examples/remapBlockMatrix.cpp

@@ -0,0 +1,363 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <vector>
+#include <cstdlib>
+#include <cmath>
+
+using namespace std;
+
+class HypreMat
+{
+private:
+   std::vector<int> I, J;
+   std::vector<double> d;
+   int size;
+
+public:
+   HypreMat() : size(0)
+   {
+
+   }
+
+   int Size() const { return size; }
+
+   void ReadMatrixParallel(std::string filename, const int np)
+   {
+      int lastRow = 0;
+      int count = 0;
+
+      std::vector<int> R;
+
+      for (int p=0; p<np; ++p)
+      {
+         std::ifstream f(filename.c_str() + std::to_string(p), std::ifstream::in);
+
+         int rowFirst, rowLast, colFirst, colLast, row, col;
+         double v;
+
+         f >> rowFirst >> rowLast >> colFirst >> colLast;
+
+         I.resize(rowLast+2);
+
+         for (row=rowFirst+1; row<=rowLast+1; ++row)
+         {
+            I[row] = 0;
+         }
+
+         while (f.good())
+         {
+            f >> row >> col >> v;
+
+            if (row >= 2485)
+            {
+               abort();
+            }
+
+            //if (row < lastRow)  // verify ascending order of rows
+            //  abort();
+
+            lastRow = row;
+
+            I[row+1]++;
+            R.push_back(row);
+            J.push_back(col);
+            d.push_back(v);
+
+            count++;
+         }
+
+         f.close();
+      }
+
+      // Partial sum of I
+
+      size = I.size() - 1;
+
+      I[0] = 0;
+      for (int j=1; j<size; ++j)
+      {
+         I[j+1] += I[j];
+      }
+
+      //int *Idata = I.data();
+
+      if (I[size] != J.size() || I[size] != d.size())
+      {
+         abort();
+      }
+
+      {
+         // Reorder J and d
+         const int nnz = J.size();
+         if (nnz != d.size() || nnz != R.size())
+         {
+            abort();
+         }
+
+         std::vector<int> JJ(nnz);
+         std::vector<double> dd(nnz);
+         std::vector<int> cnt;
+
+         JJ = J;
+         dd = d;
+
+         cnt.assign(I.size(), 0); // slightly larger than necessary
+
+         for (int j=0; j<nnz; ++j)
+         {
+            J[I[R[j]] + cnt[R[j]]] = JJ[j];
+            d[I[R[j]] + cnt[R[j]]] = dd[j];
+            cnt[R[j]]++;
+         }
+      }
+   }
+
+   void CopyReorder(HypreMat const& A, std::vector<int> const& permRow,
+                    std::vector<int> const& permCol)
+   {
+      if (permRow.size() != A.Size() || permCol.size() != A.Size())
+      {
+         abort();
+      }
+
+      size = A.Size();
+      I.resize(size+1);
+
+      I[0] = 0;
+
+      for (int j=0; j<size; ++j)
+      {
+         const std::size_t nnz_j = A.I[j+1] - A.I[j];
+         I[permRow[j]+1] = nnz_j;
+      }
+
+      for (int j=1; j<size; ++j)
+      {
+         I[j+1] += I[j];
+      }
+
+      const std::size_t nnz = I[size];
+      J.resize(nnz);
+      d.resize(nnz);
+
+      std::vector<std::size_t> cnt;
+      cnt.assign(size, 0);
+
+      for (int j=0; j<size; ++j)
+      {
+         const int pj = permRow[j];
+         const std::size_t nnz_j = A.I[j+1] - A.I[j];
+
+         for (int k=0; k<nnz_j; ++k, cnt[pj]++)
+         {
+            J[I[pj] + cnt[pj]] = permCol[A.J[A.I[j] + k]];
+            d[I[pj] + cnt[pj]] = A.d[A.I[j] + k];
+         }
+      }
+   }
+
+   void Print(std::string filename)
+   {
+      std::ofstream f(filename.c_str());
+
+      for (int j=0; j<size; ++j)
+      {
+         for (int k=I[j]; k<I[j+1]; ++k)
+         {
+            f << j << " " << J[k] << " " << d[k] << endl;
+         }
+      }
+
+      f.close();
+   }
+
+   void Compare(HypreMat const& A)
+   {
+      if (size != A.Size())
+      {
+         abort();
+      }
+
+      for (int j=0; j<size; ++j)
+      {
+         for (int k=I[j]; k<I[j+1]; ++k)
+         {
+            int m = -1;
+
+            for (int l=A.I[j]; l<A.I[j+1]; ++l)
+            {
+               if (A.J[l] == J[k])
+               {
+                  m = l;
+               }
+            }
+
+            if (m < 0)
+            {
+               cout << "row " << j << " column " << J[k] << " not found" << endl;
+            }
+            //else if (fabs(A.d[m] - d[k]) > 1.0e-8)
+            else if (fabs(fabs(A.d[m]) - fabs(d[k])) > 0.0)
+            {
+               cout << "entry (" << j << ", " << J[k] << ") " << d[k] << " != " << A.d[m] <<
+                    endl;
+            }
+         }
+      }
+   }
+};
+
+void ReadCrd(std::string filename, std::vector<double>& crd)
+{
+   std::ifstream f(filename.c_str(), std::ifstream::in);
+
+   double c;
+
+   while (f.good())
+   {
+      f >> c;
+      crd.push_back(c);
+   }
+
+   // for some reason, the last value gets pushed twice
+   crd.pop_back();
+
+   f.close();
+}
+
+void ReadCrdParallel(std::string filenamebase, const int np,
+                     std::vector<double>& crd)
+{
+   for (int p=0; p<np; ++p)
+   {
+      ReadCrd(filenamebase + std::to_string(p),
+              crd);  // results get concatenated in crd
+   }
+}
+
+// crd is the DOF coordinates in serial, crdp in parallel.
+void SerialToParallelDOFMap(std::vector<double> const& crd,
+                            std::vector<double> const& crdp,
+                            std::vector<int>& sp)
+{
+   if (crdp.size() != crd.size())
+   {
+      abort();
+   }
+
+   const int dim = 3;
+   const int n = crd.size() / dim;  // number of DOF's
+
+   sp.resize(n);
+
+   const double tol = 1.0e-8;
+
+   // Simple O(n^2) search
+
+   for (int i=0; i<n; ++i)
+   {
+      sp[i] = -1;
+
+      for (int j=0; j<n; ++j)
+      {
+         bool eq = true;
+         for (int l=0; l<dim; ++l)
+         {
+            if (fabs(crd[(dim*i)+l] - crdp[(dim*j)+l]) > tol)
+            {
+               eq = false;
+            }
+         }
+
+         if (eq)
+         {
+            if (sp[i] != -1)
+            {
+               abort();
+            }
+
+            sp[i] = j;
+         }
+      }
+
+      if (sp[i] < 0)
+      {
+         abort();
+      }
+   }
+}
+
+int main()
+{
+   const int np = 2;
+
+   std::vector<double> crd0, crdp0, crd1, crdp1;
+
+   ReadCrd("dofcrd0Ser", crd0);
+   ReadCrd("dofcrd1Ser", crd1);
+
+   const int numCrd = crd0.size() / 3;
+
+   if (3*numCrd != crd0.size())
+   {
+      abort();
+   }
+
+   //for (int i=7440; i<crd.size(); ++i)
+   //cout << "crd[" << i << "] " << crd[i] << endl;
+
+   ReadCrdParallel("dofcrd0Par", np, crdp0);
+   ReadCrdParallel("dofcrd1Par", np, crdp1);
+
+   if (3*numCrd != crdp0.size())
+   {
+      abort();
+   }
+
+   std::vector<int> sp0, sp1;  // serial to parallel DOF map for a subdomain
+   SerialToParallelDOFMap(crd0, crdp0, sp0);
+   SerialToParallelDOFMap(crd1, crdp1, sp1);
+
+   /*
+   for (int i=0; i<sp.size(); ++i)
+     cout << i << " " << sp[i] << endl;
+   */
+
+   cout << "sp0" << endl;
+   for (int i=0; i<sp0.size(); ++i)
+   {
+      cout << i << " " << sp0[i] << endl;
+   }
+
+   cout << "sp1" << endl;
+   for (int i=0; i<sp1.size(); ++i)
+   {
+      cout << i << " " << sp1[i] << endl;
+   }
+
+   HypreMat Aser, Apar, AserToPar;
+
+   /*
+   Aser.ReadMatrixParallel("HypreAsdComplexIm1_Serial.0000", 1);
+   Apar.ReadMatrixParallel("HypreAsdComplexIm1_Par5.0000", np);
+   */
+
+   Aser.ReadMatrixParallel("ifopSer.0000", 1);
+   Apar.ReadMatrixParallel("ifopPar.0000", np);
+
+   //AserToPar.CopyReorder(Aser, sp, sp);
+   AserToPar.CopyReorder(Aser, sp0, sp1);
+
+   /*
+   Aser.Print("rbSer");
+   AserToPar.Print("rbSerToPar");
+   Apar.Print("rbPar");
+   */
+
+   cout << "Comparing " << endl;
+
+   AserToPar.Compare(Apar);
+
+   return 0;
+}

examples/testStrumpack.hpp

@@ -0,0 +1,105 @@
+#ifndef TESTSTRUMPACK_HPP
+#define TESTSTRUMPACK_HPP
+
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+
+void TestStrumpackConstructor()
+{
+   int num_procs, rank;
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+   const int num_loc_rows = 100;
+   const int first_loc_row = num_loc_rows * rank;
+   const int glob_nrows = num_loc_rows * num_procs;
+   const int glob_ncols = glob_nrows;
+
+   int *opI = new int[num_loc_rows+1];
+
+   for (int i=0; i<num_loc_rows+1; ++i)
+   {
+      opI[i] = 0;
+   }
+
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      int nnz_i = 3;
+
+      if ((first_loc_row + i) == 0 ||
+          (first_loc_row + i) == glob_nrows-1)  // if first or last row
+      {
+         nnz_i = 2;
+      }
+
+      opI[i+1] = opI[i] + nnz_i;
+   }
+
+   const int nnz = opI[num_loc_rows];
+
+   int *opJ = new int[nnz];
+   double *data = new double[nnz];
+
+   int cnt = 0;
+   for (int i=0; i<num_loc_rows; ++i)
+   {
+      const int globalRow = first_loc_row + i;
+
+      // Diagonal entry
+
+      opJ[cnt] = first_loc_row + i;
+
+      if (globalRow == 0 || globalRow == glob_nrows-1)
+      {
+         data[cnt] = 2.0;
+      }
+      else
+      {
+         data[cnt] = 3.0;
+      }
+
+      cnt++;
+
+      // Upper diagonal
+      if (globalRow < glob_nrows-1)
+      {
+         opJ[cnt] = first_loc_row + i + 1;
+         data[cnt] = -1.0;
+         cnt++;
+      }
+
+      // Lower diagonal
+      if (globalRow > 0)
+      {
+         opJ[cnt] = first_loc_row + i - 1;
+         data[cnt] = -1.0;
+         cnt++;
+      }
+   }
+
+   Operator *op = new STRUMPACKRowLocMatrix(MPI_COMM_WORLD, num_loc_rows,
+                                            first_loc_row, glob_nrows, glob_ncols, opI, opJ, data);
+
+   STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, MPI_COMM_WORLD);
+   strumpack->SetPrintFactorStatistics(true);
+   strumpack->SetPrintSolveStatistics(false);
+   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+   strumpack->SetOperator(*op);
+   strumpack->SetFromCommandLine();
+
+   Vector x(num_loc_rows);
+   Vector y(num_loc_rows);
+
+   x = 1.0;
+   strumpack->Mult(x, y);
+
+   delete opI;
+   delete opJ;
+   delete data;
+}
+
+#endif // TESTSTRUMPACK_HPP

fem/bilinearform.cpp

@@ -1771,9 +1771,40 @@ MixedBilinearForm::~MixedBilinearForm()
    delete ext;
 }
 
+void DiscreteLinearOperator::SetAssemblyLevel(AssemblyLevel assembly_level)
+{
+   if (ext)
+   {
+      MFEM_ABORT("the assembly level has already been set!");
+   }
+   assembly = assembly_level;
+   switch (assembly)
+   {
+      case AssemblyLevel::FULL:
+         // Use the original BilinearForm implementation for now
+         break;
+      case AssemblyLevel::ELEMENT:
+         mfem_error("Element assembly not supported yet... stay tuned!");
+         break;
+      case AssemblyLevel::PARTIAL:
+         ext = new PADiscreteLinearOperatorExtension(this);
+         break;
+      case AssemblyLevel::NONE:
+         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         break;
+      default:
+         mfem_error("Unknown assembly level");
+   }
+}
 
 void DiscreteLinearOperator::Assemble(int skip_zeros)
 {
+   if (ext)
+   {
+      ext->Assemble();
+      return;
+   }
+
    Array<int> dom_vdofs, ran_vdofs;
    ElementTransformation *T;
    const FiniteElement *dom_fe, *ran_fe;

fem/bilinearform.hpp

@@ -375,6 +375,9 @@ public:
    /// Get the output finite element space prolongation matrix
    virtual const Operator *GetOutputProlongation() const
    { return GetProlongation(); }
+   /// Get the output finite element space prolongation matrix (local diagonal)
+   virtual const Operator *GetLocalOutputProlongation() const
+   { return GetOutputProlongation(); }
    /// Get the output finite element space restriction matrix
    virtual const Operator *GetOutputRestriction() const
    { return GetRestriction(); }
@@ -591,6 +594,18 @@ public:
    /// Indicate that integrators are not owned by the BilinearForm
    void UseExternalIntegrators() { extern_bfs = 1; };
 
+   void GetTimings(double& t1, double& t2)
+   {
+      t1 = 0.0;
+      t2 = 0.0;
+      PABilinearFormExtension *paext = dynamic_cast<PABilinearFormExtension*>(ext);
+      if (paext != NULL)
+      {
+         t1 = paext->timingDomain;
+         t2 = paext->timingBoundary;
+      }
+   }
+
    /// Destroys bilinear form.
    virtual ~BilinearForm();
 };
@@ -984,9 +999,17 @@ public:
    /// Access all interpolators added with AddDomainInterpolator().
    Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
 
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
+   /** This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level);
+
    /** @brief Construct the internal matrix representation of the discrete
        linear operator. */
    virtual void Assemble(int skip_zeros = 1);
+
+   /// Get the output finite element space prolongation matrix (local diagonal)
+   virtual const Operator *GetLocalOutputProlongation() const
+   { return test_fes->GetLocalProlongationMatrix(); }
 };
 
 }

fem/bilinearform_ext.cpp

@@ -17,6 +17,8 @@
 #include "libceed/ceed.hpp"
 #include "pgridfunc.hpp"
 
+#include "../general/tic_toc.hpp"
+
 namespace mfem
 {
 
@@ -46,6 +48,9 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
    elem_restrict = NULL;
    int_face_restrict_lex = NULL;
    bdr_face_restrict_lex = NULL;
+
+   timingDomain = 0.0;
+   timingBoundary = 0.0;
 }
 
 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
@@ -96,8 +101,12 @@ void PABilinearFormExtension::Assemble()
       integrators[i]->AssemblePA(*a->FESpace());
    }
 
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
+   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
+   const int bdryIntegratorCount = bdryIntegrators.Size();
+   for (int i = 0; i < bdryIntegratorCount; ++i)
+   {
+      bdryIntegrators[i]->AssemblePA(*a->FESpace());
+   }
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int intFaceIntegratorCount = intFaceIntegrators.Size();
@@ -112,6 +121,96 @@ void PABilinearFormExtension::Assemble()
    {
       bdrFaceIntegrators[i]->AssemblePABoundaryFaces(*a->FESpace());
    }
+
+   timingDomain = 0.0;
+   timingBoundary = 0.0;
+
+   if (bdryIntegratorCount > 0)
+   {
+      FiniteElementSpace *fes = a->GetFES();
+      const int nbe = fes->GetNBE();
+      const int bedofs = nbe > 0 ? nbe * fes->GetVDim() * fes->GetBE(0)->GetDof() : 0;
+      gatherMap.SetSize(bedofs);
+      indices.SetSize(bedofs);
+
+      ndofs = fes->GetNDofs();
+      offsets.SetSize(ndofs+1);
+      for (int i = 0; i <= ndofs; ++i)
+      {
+         offsets[i] = 0;
+      }
+
+      for (int i = 0; i < nbe; i++)
+      {
+         const FiniteElement &be = *fes->GetBE(i);
+         Array<int> vdofs;
+         fes -> GetBdrElementVDofs (i, vdofs);
+
+         const TensorBasisElement* el =
+            dynamic_cast<const TensorBasisElement*>(&be);
+
+         MFEM_VERIFY(el != NULL, "");
+
+         const Array<int> &fe_dof_map = el->GetDofMap();
+
+         MFEM_VERIFY(fe_dof_map.Size() == fes->GetBE(i)->GetDof(), "");
+         MFEM_VERIFY(vdofs.Size() == fes->GetBE(i)->GetDof(), "");
+
+         for (int j=0; j<vdofs.Size(); ++j)
+         {
+            const int sidj = fe_dof_map[j];
+            const int idj = sidj >= 0 ? sidj : -1 - sidj;
+            const int dof_j = vdofs[idj];
+            const int d = dof_j >= 0 ? dof_j : -1-dof_j;
+            offsets[d+1]++;
+         }
+      }
+
+      for (int i = 1; i <= ndofs; ++i)
+      {
+         // Partial sum
+         offsets[i] += offsets[i - 1];
+      }
+
+      int os = 0;
+      Array<int> cnt(ndofs);
+      cnt = 0;
+      indices = 0;
+
+      for (int i = 0; i < nbe; i++)
+      {
+         const FiniteElement &be = *fes->GetBE(i);
+         Array<int> vdofs;
+         fes -> GetBdrElementVDofs (i, vdofs);
+
+         const TensorBasisElement* el =
+            dynamic_cast<const TensorBasisElement*>(&be);
+
+         MFEM_VERIFY(el != NULL, "");
+
+         const Array<int> &fe_dof_map = el->GetDofMap();
+
+         MFEM_VERIFY(fe_dof_map.Size() == fes->GetBE(i)->GetDof(), "");
+         MFEM_VERIFY(vdofs.Size() == fes->GetBE(i)->GetDof(), "");
+
+         for (int j=0; j<vdofs.Size(); ++j)
+         {
+            const int sidj = fe_dof_map[j];
+            const int idj = sidj >= 0 ? sidj : -1 - sidj;
+            const int dof_j = vdofs[idj];
+            const bool plus = (sidj >= 0 && dof_j >= 0) || (sidj < 0 && dof_j < 0);
+            const int d = dof_j >= 0 ? dof_j : -1-dof_j;
+            const int lid = os + j;
+            gatherMap[lid] = plus ? d : -1-d;
+            indices[offsets[d] + cnt[d]] = plus ? lid : -1-lid;
+            cnt[d]++;
+         }
+
+         os += vdofs.Size();
+      }
+
+      MFEM_VERIFY(os == bedofs, "");
+   }
 }
 
 void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
@@ -179,10 +278,18 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
    A.Reset(oper); // A will own oper
 }
 
+//#define SWTIMING
+
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
 
+#ifdef SWTIMING
+   StopWatch swd;
+   swd.Clear();
+   swd.Start();
+#endif
+
    const int iSz = integrators.Size();
    if (DeviceCanUseCeed() || !elem_restrict)
    {
@@ -204,6 +311,93 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
       elem_restrict->MultTranspose(localY, y);
    }
 
+#ifdef SWTIMING
+   swd.Stop();
+   timingDomain += swd.RealTime();
+
+   StopWatch swb;
+   swb.Clear();
+   swb.Start();
+#endif
+
+   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
+
+   const int biSz = bdryIntegrators.Size();
+   if (DeviceCanUseCeed() || !elem_restrict)
+   {
+      MFEM_ABORT("Not implemented");
+   }
+   else if (biSz > 0)
+   {
+      FiniteElementSpace *fes = a->GetFES();
+
+      const int nbe = fes->GetNBE();
+      const int bedofs = nbe > 0 ? nbe * fes->GetVDim() * fes->GetBE(0)->GetDof() : 0;
+      bdryX.SetSize(bedofs);
+      bdryY.SetSize(bedofs);
+      bdryX.UseDevice(true);
+      bdryY.UseDevice(true);
+      bdryX = 0.0;
+      bdryY = 0.0;
+
+      {
+         MFEM_VERIFY(gatherMap.Size() == bedofs, "");
+         auto d_gatherMap = gatherMap.Read();
+         auto d_x = x.Read();
+         auto d_bdryX = bdryX.Write();
+
+         MFEM_FORALL(i, bedofs,
+         {
+            const int gid = d_gatherMap[i];
+            const bool plus = gid >= 0;
+            const int j = plus ? gid : -1-gid;
+
+            d_bdryX[i] = plus ? d_x[j] : -d_x[j];
+         });
+      }
+
+      for (int i = 0; i < biSz; ++i)
+      {
+         bdryIntegrators[i]->AddMultPA(bdryX, bdryY);
+      }
+      //elem_restrict->MultTranspose(bdryY, y);
+
+      // bdryY contains quantities on all boundary elements. Now add them to y.
+
+      {
+         const int xsize = x.Size();
+         MFEM_VERIFY(y.Size() == xsize, "");
+         MFEM_VERIFY(gatherMap.Size() == bedofs, "");
+         auto d_offsets = offsets.Read();
+         auto d_indices = indices.Read();
+         auto d_y = y.ReadWrite();
+         auto d_bdryY = bdryY.Read();
+
+         MFEM_FORALL(i, ndofs,
+         {
+            const int offset = d_offsets[i];
+            const int nextOffset = d_offsets[i + 1];
+
+            double val = 0.0;
+            for (int j = offset; j < nextOffset; ++j)
+            {
+               const int id = d_indices[j];
+               const bool plus = id >= 0;
+               const int idj = plus ? id : -1-id;
+               const double yval = d_bdryY[idj];
+               val += plus ? yval : -yval;
+            }
+
+            d_y[i] += val;
+         });
+      }
+   }
+
+#ifdef SWTIMING
+   swb.Stop();
+   timingBoundary += swb.RealTime();
+#endif
+
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int iFISz = intFaceIntegrators.Size();
    if (int_face_restrict_lex && iFISz>0)
@@ -261,6 +455,10 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
       }
    }
 
+   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
+   const int biSz = bdryIntegrators.Size();
+   MFEM_VERIFY(biSz == 0, "TODO");
+
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int iFISz = intFaceIntegrators.Size();
    if (int_face_restrict_lex && iFISz>0)
@@ -821,7 +1019,6 @@ void PAMixedBilinearFormExtension::Update()
       localTrial.UseDevice(true);
       localTrial.SetSize(elem_restrict_trial->Height(),
                          Device::GetMemoryType());
-
    }
    if (elem_restrict_test)
    {
@@ -1021,4 +1218,133 @@ void PAMixedBilinearFormExtension::AssembleDiagonal_ADAt(const Vector &D,
    }
 }
 
+PADiscreteLinearOperatorExtension::PADiscreteLinearOperatorExtension(
+   DiscreteLinearOperator *linop) :
+   PAMixedBilinearFormExtension(linop)
+{
+}
+
+const Operator *PADiscreteLinearOperatorExtension::GetLocalOutputProlongation()
+const
+{
+   return a->GetLocalOutputProlongation();
+}
+
+void PADiscreteLinearOperatorExtension::Assemble()
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integratorCount = integrators.Size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->AssemblePA(*trialFes, *testFes);
+   }
+
+   test_multiplicity.UseDevice(true);
+   test_multiplicity.SetSize(elem_restrict_test->Width()); // l-vector
+   Vector ones(elem_restrict_test->Height()); // e-vector
+   ones = 1.0;
+
+   const ElementRestriction* elem_restrict =
+      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
+   if (elem_restrict)
+   {
+      elem_restrict->MultTransposeUnsigned(ones, test_multiplicity);
+   }
+   else
+   {
+      mfem_error("A real ElementRestriction is required in this setting!");
+   }
+
+   auto tm = test_multiplicity.ReadWrite();
+   MFEM_FORALL(i, test_multiplicity.Size(),
+   {
+      tm[i] = 1.0 / tm[i];
+   });
+}
+
+void PADiscreteLinearOperatorExtension::AddMult(
+   const Vector &x, Vector &y, const double c) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+
+   // * G operation
+   SetupMultInputs(elem_restrict_trial, x, localTrial,
+                   elem_restrict_test, y, localTest, c);
+
+   // * B^TDB operation
+   for (int i = 0; i < iSz; ++i)
+   {
+      integrators[i]->AddMultPA(localTrial, localTest);
+   }
+
+   // do a kind of "set" rather than "add" in the below
+   // operation as compared to the BilinearForm case
+   // * G^T operation (kind of...)
+   const ElementRestriction* elem_restrict =
+      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
+   if (elem_restrict)
+   {
+      tempY.SetSize(y.Size());
+      elem_restrict->MultLeftInverse(localTest, tempY);
+      y += tempY;
+   }
+   else
+   {
+      mfem_error("In this setting you need a real ElementRestriction!");
+   }
+}
+
+void PADiscreteLinearOperatorExtension::AddMultTranspose(
+   const Vector &x, Vector &y, const double c) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+
+   // do a kind of "set" rather than "add" in the below
+   // operation as compared to the BilinearForm case
+   // * G operation (kinda)
+   Vector xscaled(x);
+   MFEM_VERIFY(x.Size() == test_multiplicity.Size(), "Input vector of wrong size");
+   auto xs = xscaled.ReadWrite();
+   auto tm = test_multiplicity.Read();
+   MFEM_FORALL(i, x.Size(),
+   {
+      xs[i] *= tm[i];
+   });
+   SetupMultInputs(elem_restrict_test, xscaled, localTest,
+                   elem_restrict_trial, y, localTrial, c);
+
+   // * B^TD^TB operation
+   for (int i = 0; i < iSz; ++i)
+   {
+      integrators[i]->AddMultTransposePA(localTest, localTrial);
+   }
+
+   // * G^T operation
+   if (elem_restrict_trial)
+   {
+      tempY.SetSize(y.Size());
+      elem_restrict_trial->MultTranspose(localTrial, tempY);
+      y += tempY;
+   }
+   else
+   {
+      mfem_error("Trial ElementRestriction not defined");
+   }
+}
+
+void PADiscreteLinearOperatorExtension::FormRectangularSystemOperator(
+   const Array<int>& ess1, const Array<int>& ess2, OperatorHandle &A)
+{
+   const Operator *Pi = this->GetProlongation();
+   const Operator *Po = this->GetLocalOutputProlongation();
+   Operator *rap = SetupRAP(Pi, Po);
+
+   RectangularConstrainedOperator *Arco
+      = new RectangularConstrainedOperator(rap, ess1, ess2, rap != this);
+
+   A.Reset(Arco);
+}
+
 } // namespace mfem

fem/bilinearform_ext.hpp

@@ -21,6 +21,7 @@ namespace mfem
 
 class BilinearForm;
 class MixedBilinearForm;
+class DiscreteLinearOperator;
 
 /// Class extending the BilinearForm class to support different AssemblyLevels.
 /**  FA - Full Assembly
@@ -68,11 +69,16 @@ class PABilinearFormExtension : public BilinearFormExtension
 protected:
    const FiniteElementSpace *trialFes, *testFes; // Not owned
    mutable Vector localX, localY;
+   mutable Vector bdryX, bdryY;
    mutable Vector faceIntX, faceIntY;
    mutable Vector faceBdrX, faceBdrY;
    const Operator *elem_restrict; // Not owned
    const Operator *int_face_restrict_lex; // Not owned
    const Operator *bdr_face_restrict_lex; // Not owned
+   Array<int> gatherMap;
+   Array<int> indices;
+   Array<int> offsets;
+   int ndofs;
 
 public:
    PABilinearFormExtension(BilinearForm*);
@@ -88,6 +94,8 @@ public:
    void MultTranspose(const Vector &x, Vector &y) const;
    void Update();
 
+   mutable double timingDomain, timingBoundary;
+
 protected:
    void SetupRestrictionOperators(const L2FaceValues m);
 };
@@ -205,7 +213,7 @@ protected:
    mutable Vector localTrial, localTest, tempY;
    const Operator *elem_restrict_trial; // Not owned
    const Operator *elem_restrict_test;  // Not owned
-private:
+
    /// Helper function to set up inputs/outputs for Mult or MultTranspose
    void SetupMultInputs(const Operator *elem_restrict_x,
                         const Vector &x, Vector &localX,
@@ -251,6 +259,33 @@ public:
    void Update();
 };
 
+
+/**
+   This acts very much like PAMixedBilinearFormExtension, but its
+   FormRectangularSystemOperator implementation emulates 'Set' rather than
+   'Add' in the assembly case.
+*/
+class PADiscreteLinearOperatorExtension : public PAMixedBilinearFormExtension
+{
+public:
+   PADiscreteLinearOperatorExtension(DiscreteLinearOperator *linop);
+
+   /// Partial assembly of all internal integrators
+   void Assemble();
+
+   void AddMult(const Vector &x, Vector &y, const double c) const;
+
+   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
+
+   void FormRectangularSystemOperator(const Array<int>&, const Array<int>&,
+                                      OperatorHandle& A);
+
+   const Operator * GetLocalOutputProlongation() const;
+
+private:
+   Vector test_multiplicity;
+};
+
 }
 
 #endif

fem/bilininteg.cpp

@@ -22,14 +22,14 @@ namespace mfem
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(fes)\n"
                "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                         const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
                "   is not implemented for this class.");
 }
 
@@ -92,7 +92,7 @@ void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
 
 void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose(...)\n"
+   mfem_error ("BilinearFormIntegrator::AddMultTransposePA(...)\n"
                "   is not implemented for this class.");
 }
 
@@ -900,6 +900,44 @@ const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
    return IntRules.Get(trial_fe.GetGeomType(), order);
 }
 
+/// alpha (n x u, v)
+void VectorFEBoundaryTangentIntegrator::AssembleElementMatrix
+(const FiniteElement &el, ElementTransformation &Trans,
+ DenseMatrix &elmat)
+{
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
+
+   const int nd1 = el.GetDof();
+
+   DenseMatrix vshape(nd1, 3);
+   DenseMatrix vshapeRotated(nd1, 3);
+
+   elmat.SetSize(nd1);
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint (&ip);
+
+      Vector n(3);
+      CalcOrtho(Trans.Jacobian(), n);
+
+      el.CalcVShape(Trans, vshape);
+
+      for (int j=0; j<nd1; ++j)
+      {
+         // Set vshapeRotated(j) = n x vshape
+         vshapeRotated(j, 0) = (n[1] * vshape(j, 2)) - (n[2] * vshape(j, 1));
+         vshapeRotated(j, 1) = (n[2] * vshape(j, 0)) - (n[0] * vshape(j, 2));
+         vshapeRotated(j, 2) = (n[0] * vshape(j, 1)) - (n[1] * vshape(j, 0));
+      }
+
+      const double w = alpha * ip.weight; // Trans.Weight() is included in n
+
+      AddMult_a_ABt(w, vshape, vshapeRotated, elmat);
+   }
+}
 
 void BoundaryMassIntegrator::AssembleFaceMatrix(
    const FiniteElement &el1, const FiniteElement &el2,

fem/bilininteg.hpp

@@ -17,6 +17,8 @@
 #include "fespace.hpp"
 #include "libceed/ceed.hpp"
 
+//#define SETUPONHOST
+
 namespace mfem
 {
 
@@ -207,6 +209,9 @@ public:
    { return 0.0; }
 
    virtual ~BilinearFormIntegrator() { }
+
+   bool isBdryInteg = false;
+   Array<int> *el_marker = NULL;
 };
 
 /** Wraps a given @a BilinearFormIntegrator and transposes the resulting element
@@ -1809,8 +1814,10 @@ protected:
 };
 
 /** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same space
-    as u. */
+    scalar coefficient, and v is a vector with components v_i in the same (H1) space
+    as u.
+
+    See also MixedVectorGradientIntegrator when v is in H(curl). */
 class GradientIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2062,6 +2069,20 @@ public:
                                    DenseMatrix &elmat);
 };
 
+/// alpha (n x u, v)
+class VectorFEBoundaryTangentIntegrator : public MassIntegrator
+{
+private:
+   double alpha;
+
+public:
+   VectorFEBoundaryTangentIntegrator(double a = 1.0) : alpha(a)
+   { }
+
+   void AssembleElementMatrix(const FiniteElement &el,
+                              ElementTransformation &Trans, DenseMatrix &elmat);
+};
+
 /// alpha (q . grad u, v)
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
@@ -2878,11 +2899,33 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };
 class GradientInterpolator : public DiscreteInterpolator
 {
 public:
+   GradientInterpolator() : fake_fe(NULL) { }
+   virtual ~GradientInterpolator() { delete fake_fe; }
+
    virtual void AssembleElementMatrix2(const FiniteElement &h1_fe,
                                        const FiniteElement &nd_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat)
    { nd_fe.ProjectGrad(h1_fe, Trans, elmat); }
+
+   using BilinearFormIntegrator::AssemblePA;
+   /**
+      trial_fes should be H1 Lagrange
+      test_fes should be Nedelec
+   */
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+private:
+   /// 1D finit element that generates and owns the 1D DofToQuad maps below
+   FiniteElement * fake_fe;
+
+   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
+   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
+   int dim, ne, o_dofs1D, c_dofs1D;
 };
 
 
@@ -2897,6 +2940,24 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat)
    { ran_fe.Project(dom_fe, Trans, elmat); }
+
+   using BilinearFormIntegrator::AssemblePA;
+
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+private:
+   /// 1D finit element that generates and owns the 1D DofToQuad maps below
+   FiniteElement * fake_fe;
+
+   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
+   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
+   int dim, ne, o_dofs1D, c_dofs1D;
+
+   Vector pa_data;
 };
 
 

fem/bilininteg_gradient.cpp

@@ -334,7 +334,7 @@ static void PAGradientApplyTranspose2D(const int NE,
                                        const int q1d = 0)
 {
    // TODO
-   MFEM_ASSERT(false, "GradientPAApplyTranspose 3D not implemented.");
+   MFEM_ASSERT(false, "PAGradientApplyTranspose2D not implemented.");
 }
 
 // PA Gradient Apply 3D kernel

fem/bilininteg_mass_pa.cpp

@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
 
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
@@ -50,7 +50,7 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
+   pa_data.SetSize(ne*nq, Device::GetMemoryType());
    Vector coeff;
    if (Q == nullptr)
    {
@@ -950,8 +950,9 @@ static void SmemPAMassApply3D(const int NE,
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
    {
+      const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -968,188 +969,130 @@ static void SmemPAMassApply3D(const int NE,
       double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
       double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
       double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
                X[dz][dy][dx] = x(dx,dy,dz,e);
             }
          }
-         MFEM_FOREACH_THREAD(dx,x,Q1D)
+      }
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            B[dx][dy] = b(dx,dy);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               u[dz] = 0;
+               double u = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  u += X[dz][dy][dx] * B[qx][dx];
+               }
+               DDQ[dz][dy][qx] = u;
             }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               MFEM_UNROLL(MD1)
+               double u = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  u += DDQ[dz][dy][qx] * B[qy][dy];
+               }
+               DQQ[dz][qy][qx] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  u[dz] += X[dz][dy][dx] * B[qx][dx];
+                  u += DQQ[dz][qy][qx] * B[qz][dz];
                }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ[dz][dy][qx] = u[dz];
+               QQQ[qz][qy][qx] = u * d(qx,qy,qz,e);
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
+            MFEM_FOREACH_THREAD(q,x,Q1D)
             {
-               u[dz] = 0;
+               Bt[d][q] = b(q,d);
             }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
+               double u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
                {
-                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+                  u += QQQ[qz][qy][qx] * Bt[dx][qx];
                }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ[dz][qy][qx] = u[dz];
+               QQD[qz][qy][dx] = u;
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
+               double u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
                {
-                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+                  u += QQD[qz][qy][dx] * Bt[dy][qy];
                }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+               QDD[qz][dy][dx] = u;
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
          {
-            Bt[d][q] = b(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
             {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               MFEM_UNROLL(MQ1)
+               double u = 0.0;
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+                  u += QDD[qz][dy][dx] * Bt[dz][qz];
                }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD[qz][qy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD[qz][dy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += u[dz];
+               y(dx,dy,dz,e) += u;
             }
          }
       }
@@ -1180,20 +1123,14 @@ static void PAMassApply(const int dim,
       MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
    }
 #endif // MFEM_USE_OCCA
-   const int id = (D1D << 4) | Q1D;
    if (dim == 2)
    {
-      switch (id)
+      switch ((D1D << 4) | Q1D)
       {
          case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
          case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
          case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
          case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
          case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
          case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
          case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
@@ -1203,25 +1140,18 @@ static void PAMassApply(const int dim,
    }
    else if (dim == 3)
    {
-      switch (id)
+      switch ((D1D << 4) | Q1D)
       {
          case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
          case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
          case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
          case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
          case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
          case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
          case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
          default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
    }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
    MFEM_ABORT("Unknown kernel.");
 }
 

fem/bilininteg_vectorfe.cpp

@@ -23,6 +23,13 @@ void PAHcurlSetup2D(const int Q1D,
                     Vector &coeff,
                     Vector &op);
 
+void PAHcurlSetup2Din3D(const int Q1D,
+                        const int NE,
+                        const Array<double> &w,
+                        const Vector &j,
+                        Vector &_coeff,
+                        Vector &op);
+
 void PAHcurlSetup3D(const int Q1D,
                     const int coeffDim,
                     const int NE,
@@ -49,6 +56,14 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                    const Vector &pa_data,
                                    Vector &diag);
 
+void PAHcurlMassAssembleDiagonal3DHost(const int D1D,
+                                       const int Q1D,
+                                       const int NE,
+                                       const Array<double> &_Bo,
+                                       const Array<double> &_Bc,
+                                       const Vector &_op,
+                                       Vector &_diag);
+
 template<int T_D1D = 0, int T_Q1D = 0>
 void SmemPAHcurlMassAssembleDiagonal3D(const int D1D,
                                        const int Q1D,
@@ -722,8 +737,14 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 {
    // Assumes tensor-product elements
    Mesh *mesh = trial_fes.GetMesh();
+   dim = mesh->Dimension();
+   const bool bdryInteg = isBdryInteg;
+
+   // TODO: this implementation applies boundary markers just by setting coeff to zero on boundary elements. It would be better to skip those elements in AddMultPA.
+
+   const FiniteElement *trial_fel = bdryInteg ? trial_fes.GetBE(
+                                       0) : trial_fes.GetFE(0);
 
-   const FiniteElement *trial_fel = trial_fes.GetFE(0);
    const VectorTensorFiniteElement *trial_el =
       dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
    MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
@@ -733,21 +754,38 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
    MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
 
+   if (bdryInteg)
+   {
+      MFEM_VERIFY(trial_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3 &&
+                  mesh->SpaceDimension() == 3 && trial_el->GetDim() == 2, "");
+   }
+
    const IntegrationRule *ir
       = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
+                                                     bdryInteg ? *mesh->GetBdrElementTransformation(0) :
                                                      *mesh->GetElementTransformation(0));
    const int dims = trial_el->GetDim();
+
    MFEM_VERIFY(dims == 2 || dims == 3, "");
 
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
    const int nq = ir->GetNPoints();
-   dim = mesh->Dimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "");
 
-   ne = trial_fes.GetNE();
-   MFEM_VERIFY(ne == test_fes.GetNE(),
+   ne = bdryInteg ? trial_fes.GetNBE() : trial_fes.GetNE();
+   Vector bdryJac;
+   if (bdryInteg)
+   {
+      bdryJac.SetSize(ne*nq*6);
+   }
+   else
+   {
+      geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   }
+
+   MFEM_VERIFY(ne == (bdryInteg ? test_fes.GetNBE() : test_fes.GetNE()),
                "Different meshes for test and trial spaces");
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+
    mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
    mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
    dofs1D = mapsC->ndof;
@@ -781,7 +819,12 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
                       Device::GetMemoryType());
 
-   Vector coeff(coeffDim * ne * nq);
+   if (bdryInteg)
+   {
+      MFEM_VERIFY(coeffDim == 1 && dims == 2, "");
+   }
+
+   Vector coeff(coeffDim * nq * ne);
    coeff = 1.0;
    auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
    if (Q || VQ || MQ)
@@ -813,7 +856,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
       for (int e=0; e<ne; ++e)
       {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         ElementTransformation *tr = bdryInteg ? mesh->GetBdrElementTransformation(
+                                        e) : mesh->GetElementTransformation(e);
          for (int p=0; p<nq; ++p)
          {
             if (MQ)
@@ -854,10 +898,72 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       }
    }
 
-   if (trial_curl && test_curl && dim == 3)
+   if (el_marker)
    {
+      MFEM_VERIFY(bdryInteg, "");
+      MFEM_VERIFY(el_marker->Size() == mesh->bdr_attributes.Max(), "");
+
+      for (int e=0; e<ne; ++e)
+      {
+         const int el_attr = (bdryInteg) ? mesh->GetBdrAttribute(e) : -1;
+         MFEM_VERIFY(el_attr > 0, "");
+
+         if ((*el_marker)[el_attr-1] == 0)
+         {
+            for (int p=0; p<nq; ++p)
+            {
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = 0.0;
+               }
+            }
+         }
+      }
+   }
+
+   if (isBdryInteg || (trial_el->GetDerivType() == mfem::FiniteElement::CURL &&
+                       dim == 2 && mesh->SpaceDimension() == 3))
+   {
+      MFEM_VERIFY(coeffDim == 1,
+                  "");  // Vector coefficient not implemented in this case
+
+      if (isBdryInteg)
+      {
+         for (int e=0; e<ne; ++e)
+         {
+            ElementTransformation *tr = mesh->GetBdrElementTransformation(e);
+            for (int q = 0; q < nq; ++q)
+            {
+               tr->SetIntPoint(&(ir->IntPoint(q)));
+               const DenseMatrix& jac = tr->Jacobian();
+               for (int i=0; i<3; ++i)
+                  for (int j=0; j<2; ++j)
+                  {
+                     bdryJac[(6*nq*e) + (3*nq*j) + (i*nq) + q] = jac(i,j);
+                  }
+            }
+         }
+      }
+      else
+      {
+         // J is allocated as 3x3, but entries (i,j) are set only for 0 <= i < 3, 0 <= j < 2.
+         // That is, on each element J is 3x2 with 6 entries, not 9, so the last third of geom->J is unused.
+         MFEM_VERIFY(geom->J.Size() == 6 * ne * nq, "");
+      }
+
+      PAHcurlSetup2Din3D(quad1D, ne, ir->GetWeights(),
+                         isBdryInteg ? bdryJac : geom->J,
+                         coeff, pa_data);
+   }
+   else if (trial_curl && test_curl && dim == 3)
+   {
+#ifdef SETUPONHOST
+      PAHcurlSetup3DHost(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                         coeff, pa_data);
+#else
       PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
+#endif
    }
    else if (trial_curl && test_curl && dim == 2)
    {
@@ -967,7 +1073,7 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
    const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
-   if (dim == 3)
+   if (dim == 3 && !isBdryInteg)
    {
       if (trial_curl && test_curl)
       {

fem/complex_fem.cpp

@@ -968,7 +968,7 @@ ParComplexLinearForm::ParallelAssemble()
                                            2*(pfes->GlobalTrueVSize()),
                                            tdof_offsets);
 
-   tv->Write();
+   ((Vector*) tv)->Write(true);
    Vector tvr; tvr.MakeRef(*tv, 0, tvsize);
    Vector tvi; tvi.MakeRef(*tv, tvsize, tvsize);
 

fem/fespace.hpp

@@ -330,6 +330,11 @@ public:
    virtual const Operator *GetProlongationMatrix() const
    { return GetConformingProlongation(); }
 
+   /// Return a prolongation operator that only acts on local dofs
+   /** The returned operator is owned by the FiniteElementSpace. */
+   virtual const Operator *GetLocalProlongationMatrix() const
+   { return GetConformingProlongation(); }
+
    /// The returned SparseMatrix is owned by the FiniteElementSpace.
    virtual const SparseMatrix *GetRestrictionMatrix() const
    { return GetConformingRestriction(); }
@@ -984,7 +989,6 @@ inline bool UsesTensorBasis(const FiniteElementSpace& fes)
 {
    return dynamic_cast<const mfem::TensorBasisElement *>(fes.GetFE(0))!=nullptr;
 }
-
 }
 
 #endif

fem/pbilinearform.cpp

@@ -565,6 +565,53 @@ HypreParMatrix* ParDiscreteLinearOperator::ParallelAssemble() const
    return RAP;
 }
 
+void ParDiscreteLinearOperator::ParallelAssemble(OperatorHandle &A)
+{
+   // construct the rectangular block-diagonal matrix dA
+   OperatorHandle dA(A.Type());
+   dA.MakeRectangularBlockDiag(domain_fes->GetComm(),
+                               range_fes->GlobalVSize(),
+                               domain_fes->GlobalVSize(),
+                               range_fes->GetDofOffsets(),
+                               domain_fes->GetDofOffsets(),
+                               mat);
+
+   OperatorHandle P_test(A.Type()), P_trial(A.Type());
+
+   // TODO - construct the Dof_TrueDof_Matrix directly in the required format.
+   P_test.ConvertFrom(range_fes->Dof_TrueDof_Matrix());
+   P_trial.ConvertFrom(domain_fes->Dof_TrueDof_Matrix());
+
+   A.MakeRAP(P_test, dA, P_trial);
+}
+
+//// @todo copied from ParMixedBilinearForm, should be some inheritance?
+void ParDiscreteLinearOperator::FormRectangularSystemMatrix(OperatorHandle &A)
+{
+   if (ext)
+   {
+      Array<int> empty;
+      ext->FormRectangularSystemOperator(empty, empty, A);
+      return;
+   }
+
+   /*
+   if (mat)
+   {
+      Finalize();
+      ParallelAssemble(p_mat);
+      delete mat;
+      mat = NULL;
+      delete mat_e;
+      mat_e = NULL;
+      p_mat_e = NULL;
+   }
+   A = p_mat;
+   */
+
+   mfem_error("not implemented!");
+}
+
 void ParDiscreteLinearOperator::GetParBlocks(Array2D<HypreParMatrix *> &blocks)
 const
 {

fem/pbilinearform.hpp

@@ -160,6 +160,9 @@ public:
    /// Get the parallel finite element space prolongation matrix
    virtual const Operator *GetProlongation() const
    { return pfes->GetProlongationMatrix(); }
+   /// Get the diagonal part of the parallel finite element space prolongation matrix
+   virtual const Operator *GetLocalProlongation() const
+   { return pfes->GetLocalProlongationMatrix(); }
    /// Get the parallel finite element space restriction matrix
    virtual const Operator *GetRestriction() const
    { return pfes->GetRestrictionMatrix(); }
@@ -301,10 +304,18 @@ public:
    /// Returns the matrix "assembled" on the true dofs
    HypreParMatrix *ParallelAssemble() const;
 
+   /** @brief Returns the matrix assembled on the true dofs, i.e.
+       @a A = P_test^t A_local P_trial, in the format (type id) specified by
+       @a A. */
+   void ParallelAssemble(OperatorHandle &A);
+
    /** Extract the parallel blocks corresponding to the vector dimensions of the
        domain and range parallel finite element spaces */
    void GetParBlocks(Array2D<HypreParMatrix *> &blocks) const;
 
+   /** @brief Return in @a A a parallel (on truedofs) version of this operator. */
+   virtual void FormRectangularSystemMatrix(OperatorHandle &A);
+
    virtual ~ParDiscreteLinearOperator() { }
 };
 

fem/pfespace.cpp

@@ -101,6 +101,7 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
 
    P = NULL;
    Pconf = NULL;
+   Pconf_local = NULL;
    R = NULL;
 
    num_face_nbr_dofs = -1;
@@ -921,6 +922,39 @@ const Operator *ParFiniteElementSpace::GetProlongationMatrix() const
    }
 }
 
+const Operator *ParFiniteElementSpace::GetLocalProlongationMatrix() const
+{
+   if (Conforming())
+   {
+      if (Pconf_local) { return Pconf_local; }
+
+      if (NRanks == 1)
+      {
+         Pconf_local = new IdentityOperator(GetTrueVSize());
+      }
+      else
+      {
+         if (!Device::Allows(Backend::DEVICE_MASK))
+         {
+            Pconf_local = new ConformingProlongationOperator(*this, true);
+         }
+         else
+         {
+            // Pconf = new DeviceConformingProlongationOperator(*this);
+            mfem_error("Not implemented!");
+         }
+      }
+      return Pconf_local;
+   }
+   else
+   {
+      // return Dof_TrueDof_Matrix();
+      // just need diagonal portion, not too hard
+      mfem_error("Not implemented!");
+      return NULL;
+   }
+}
+
 void ParFiniteElementSpace::ExchangeFaceNbrData()
 {
    if (num_face_nbr_dofs >= 0) { return; }
@@ -2834,6 +2868,7 @@ void ParFiniteElementSpace::Destroy()
 
    delete P; P = NULL;
    delete Pconf; Pconf = NULL;
+   delete Pconf_local; Pconf_local = NULL;
    delete R; R = NULL;
 
    delete gcomm; gcomm = NULL;
@@ -2959,12 +2994,12 @@ void ParFiniteElementSpace::Update(bool want_transform)
    }
 }
 
-
 ConformingProlongationOperator::ConformingProlongationOperator(
-   const ParFiniteElementSpace &pfes)
+   const ParFiniteElementSpace &pfes, bool local_)
    : Operator(pfes.GetVSize(), pfes.GetTrueVSize()),
      external_ldofs(),
-     gc(pfes.GroupComm())
+     gc(pfes.GroupComm()),
+     local(local_)
 {
    MFEM_VERIFY(pfes.Conforming(), "");
    const Table &group_ldof = gc.GroupLDofTable();
@@ -3013,7 +3048,14 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
    const int m = external_ldofs.Size();
 
    const int in_layout = 2; // 2 - input is ltdofs array
-   gc.BcastBegin(const_cast<double*>(xdata), in_layout);
+   if (local)
+   {
+      y = 0.0;
+   }
+   else
+   {
+      gc.BcastBegin(const_cast<double*>(xdata), in_layout);
+   }
 
    int j = 0;
    for (int i = 0; i < m; i++)
@@ -3025,7 +3067,10 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
    std::copy(xdata+j-m, xdata+Width(), ydata+j);
 
    const int out_layout = 0; // 0 - output is ldofs array
-   gc.BcastEnd(ydata, out_layout);
+   if (!local)
+   {
+      gc.BcastEnd(ydata, out_layout);
+   }
 }
 
 void ConformingProlongationOperator::MultTranspose(
@@ -3038,7 +3083,10 @@ void ConformingProlongationOperator::MultTranspose(
    double *ydata = y.HostWrite();
    const int m = external_ldofs.Size();
 
-   gc.ReduceBegin(xdata);
+   if (!local)
+   {
+      gc.ReduceBegin(xdata);
+   }
 
    int j = 0;
    for (int i = 0; i < m; i++)
@@ -3050,7 +3098,10 @@ void ConformingProlongationOperator::MultTranspose(
    std::copy(xdata+j, xdata+Height(), ydata+j-m);
 
    const int out_layout = 2; // 2 - output is an array on all ltdofs
-   gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
+   if (!local)
+   {
+      gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
+   }
 }
 
 DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(

fem/pfespace.hpp

@@ -72,6 +72,9 @@ private:
    mutable HypreParMatrix *P;
    /// Optimized action-only prolongation operator for conforming meshes. Owned.
    mutable Operator *Pconf;
+   /// Optimized action-only diagonal prolongation operator on ldofs for
+   /// conforming meshes. Owned.
+   mutable Operator *Pconf_local;
 
    /// The (block-diagonal) matrix R (restriction of dof to true dof). Owned.
    mutable SparseMatrix *R;
@@ -335,6 +338,7 @@ public:
    HYPRE_Int GetMyTDofOffset() const;
 
    virtual const Operator *GetProlongationMatrix() const;
+   virtual const Operator *GetLocalProlongationMatrix() const;
    /// Get the R matrix which restricts a local dof vector to true dof vector.
    virtual const SparseMatrix *GetRestrictionMatrix() const
    { Dof_TrueDof_Matrix(); return R; }
@@ -389,9 +393,11 @@ class ConformingProlongationOperator : public Operator
 protected:
    Array<int> external_ldofs;
    const GroupCommunicator &gc;
+   bool local;
 
 public:
-   ConformingProlongationOperator(const ParFiniteElementSpace &pfes);
+   ConformingProlongationOperator(const ParFiniteElementSpace &pfes,
+                                  bool local_=false);
 
    virtual void Mult(const Vector &x, Vector &y) const;
 

fem/quadinterpolator.cpp

@@ -403,6 +403,19 @@ void QuadratureInterpolator::Mult(
          }
       }
    }
+   else if (vdim == 3 && dim == 2)
+   {
+      switch (100*nd + nq)
+      {
+         // Q1
+         case 404: eval_func = &Eval2D<3,4,4>; break;
+         case 409: eval_func = &Eval2D<3,4,9>; break;
+         // Q2
+         case 416: eval_func = &Eval2D<3,4,16>; break;
+         // Q3
+         case 425: eval_func = &Eval2D<3,4,25>; break;
+      }
+   }
    if (eval_func)
    {
       eval_func(ne, vdim, maps, e_vec, q_val, q_der, q_det, eval_flags);

fem/quadinterpolator_face.hpp

@@ -42,7 +42,7 @@ protected:
 
    static const int MAX_NQ2D = 100;
    static const int MAX_ND2D = 100;
-   static const int MAX_VDIM2D = 2;
+   static const int MAX_VDIM2D = 3;
 
    static const int MAX_NQ3D = 1000;
    static const int MAX_ND3D = 1000;

fem/restriction.cpp

@@ -17,6 +17,90 @@
 namespace mfem
 {
 
+/*
+// TODO: implement this based on PABilinearFormExtension::FormLinearSystem
+BoundaryElementRestriction::BoundaryElementRestriction(const FiniteElementSpace &f,
+                                     ElementDofOrdering e_ordering)
+ : fes(f),
+   nbe(fes.GetNBE()),
+   vdim(fes.GetVDim()),
+   byvdim(fes.GetOrdering() == Ordering::byVDIM),
+   ndofs(fes.GetNDofs()),
+   dof(nbe > 0 ? fes.GetBE(0)->GetDof() : 0),
+   nbedofs(nbe*dof),
+   offsets(ndofs+1),
+   indices(nbe*dof),
+   gatherMap(nbe*dof)
+{
+ // Assuming all finite elements are the same.
+ height = vdim*nbe*dof;
+ width = fes.GetVSize();
+ const bool dof_reorder = (e_ordering == ElementDofOrdering::LEXICOGRAPHIC);
+ const int *dof_map = NULL;
+ if (dof_reorder && nbe > 0)
+ {
+    for (int e = 0; e < nbe; ++e)
+    {
+       const FiniteElement *fe = fes.GetBE(e);
+       const TensorBasisElement* el =
+          dynamic_cast<const TensorBasisElement*>(fe);
+       if (el) { continue; }
+       mfem_error("Finite element not suitable for lexicographic ordering");
+    }
+    const FiniteElement *fe = fes.GetBE(0);
+    const TensorBasisElement* el =
+       dynamic_cast<const TensorBasisElement*>(fe);
+    const Array<int> &fe_dof_map = el->GetDofMap();
+    MFEM_VERIFY(fe_dof_map.Size() > 0, "invalid dof map");
+    dof_map = fe_dof_map.GetData();
+ }
+ const Table& e2dTable = fes.GetElementToDofTable();
+ const int* elementMap = e2dTable.GetJ();
+ // We will be keeping a count of how many local nodes point to its global dof
+ for (int i = 0; i <= ndofs; ++i)
+ {
+    offsets[i] = 0;
+ }
+ for (int e = 0; e < ne; ++e)
+ {
+    for (int d = 0; d < dof; ++d)
+    {
+       const int sgid = elementMap[dof*e + d];  // signed
+       const int gid = (sgid >= 0) ? sgid : -1 - sgid;
+       ++offsets[gid + 1];
+    }
+ }
+ // Aggregate to find offsets for each global dof
+ for (int i = 1; i <= ndofs; ++i)
+ {
+    offsets[i] += offsets[i - 1];
+ }
+ // For each global dof, fill in all local nodes that point to it
+ for (int e = 0; e < ne; ++e)
+ {
+    for (int d = 0; d < dof; ++d)
+    {
+       const int sdid = dof_reorder ? dof_map[d] : 0;  // signed
+       const int did = (!dof_reorder)?d:(sdid >= 0 ? sdid : -1-sdid);
+       const int sgid = elementMap[dof*e + did];  // signed
+       const int gid = (sgid >= 0) ? sgid : -1-sgid;
+       const int lid = dof*e + d;
+       const bool plus = (sgid >= 0 && sdid >= 0) || (sgid < 0 && sdid < 0);
+       gatherMap[lid] = plus ? gid : -1-gid;
+       indices[offsets[gid]++] = plus ? lid : -1-lid;
+    }
+ }
+ // We shifted the offsets vector by 1 by using it as a counter.
+ // Now we shift it back.
+ for (int i = ndofs; i > 0; --i)
+ {
+    offsets[i] = offsets[i - 1];
+ }
+ offsets[0] = 0;
+
+}
+*/
+
 ElementRestriction::ElementRestriction(const FiniteElementSpace &f,
                                        ElementDofOrdering e_ordering)
    : fes(f),
@@ -195,6 +279,32 @@ void ElementRestriction::MultTransposeUnsigned(const Vector& x, Vector& y) const
    });
 }
 
+void ElementRestriction::MultLeftInverse(const Vector& x, Vector& y) const
+{
+   // Assumes all elements have the same number of dofs
+   const int nd = dof;
+   const int vd = vdim;
+   const bool t = byvdim;
+   auto d_offsets = offsets.Read();
+   auto d_indices = indices.Read();
+   auto d_x = Reshape(x.Read(), nd, vd, ne);
+   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   MFEM_FORALL(i, ndofs,
+   {
+      const int nextOffset = d_offsets[i + 1];
+      for (int c = 0; c < vd; ++c)
+      {
+         double dofValue = 0;
+         const int j = nextOffset - 1;
+         const int idx_j = (d_indices[j] >= 0) ? d_indices[j] : -1 - d_indices[j];
+         dofValue =
+         (d_indices[j] >= 0) ?
+         d_x(idx_j % nd, c, idx_j / nd) : -d_x(idx_j % nd, c, idx_j / nd);
+         d_y(t?c:i,t?i:c) = dofValue;
+      }
+   });
+}
+
 void ElementRestriction::BooleanMask(Vector& y) const
 {
    // Assumes all elements have the same number of dofs

fem/restriction.hpp

@@ -57,6 +57,10 @@ public:
    /// Compute MultTranspose without applying signs based on DOF orientations.
    void MultTransposeUnsigned(const Vector &x, Vector &y) const;
 
+   /// Compute MultTranspose by setting (rather than adding) element
+   /// contributions; this is a left inverse of the Mult() operation
+   void MultLeftInverse(const Vector &x, Vector &y) const;
+
    /// @brief Fills the E-vector y with `boolean` values 0.0 and 1.0 such that each
    /// each entry of the L-vector is uniquely represented in `y`.
    /** This means, the sum of the E-vector `y` is equal to the sum of the

general/array.hpp

@@ -108,6 +108,8 @@ public:
    /// Return the device flag of the Memory object used by the Array
    bool UseDevice() const { return data.UseDevice(); }
 
+   inline MemoryType GetMemoryType() const { return data.GetMemoryType(); }
+
    /// Return true if the data will be deleted by the array
    inline bool OwnsData() const { return data.OwnsHostPtr(); }
 

general/mem_manager.cpp

@@ -794,8 +794,9 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
       }
       case MemoryClass::MANAGED:
       {
-         MFEM_VERIFY((h_mt == MemoryType::MANAGED &&
+         MFEM_VERIFY((d_mt == MemoryType::DEVICE ||
                       d_mt == MemoryType::MANAGED),"");
+
          return true;
       }
       default: break;
@@ -1046,10 +1047,10 @@ void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
    {
       if (dest_h_ptr != src_h_ptr && bytes != 0)
       {
-         MFEM_ASSERT((char*)dest_h_ptr + bytes <= src_h_ptr ||
+         MFEM_ASSERT((char*)dest_h_ptr + bytes <= (const char*)src_h_ptr ||
                      (const char*)src_h_ptr + bytes <= dest_h_ptr,
                      "data overlaps!");
-         std::memcpy(dest_h_ptr, src_h_ptr, bytes);
+         std::memcpy(dest_h_ptr, (const char*)src_h_ptr, bytes);
       }
    }
    else
@@ -1059,7 +1060,7 @@ void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
                          mm.GetDevicePtr(dest_h_ptr, bytes, false);
       const internal::Memory &base = maps->memories.at(dest_h_ptr);
       const MemoryType d_mt = base.d_mt;
-      ctrl->Device(d_mt)->HtoD(dest_d_ptr, src_h_ptr, bytes);
+      ctrl->Device(d_mt)->HtoD(dest_d_ptr, (const char*)src_h_ptr, bytes);
    }
    dest_flags = dest_flags &
                 ~(dest_on_host ? Mem::VALID_DEVICE : Mem::VALID_HOST);
@@ -1290,7 +1291,8 @@ void MemoryManager::Destroy()
    for (auto& n : maps->memories)
    {
       internal::Memory &mem = n.second;
-      bool mem_h_ptr = mem.h_mt != MemoryType::HOST && mem.h_ptr;
+      bool mem_h_ptr = mem.h_mt != MemoryType::HOST &&
+                       mem.h_mt != MemoryType::MANAGED && mem.h_ptr;
       if (mem_h_ptr) { ctrl->Host(mem.h_mt)->Dealloc(mem.h_ptr); }
       if (mem.d_ptr) { ctrl->Device(mem.d_mt)->Dealloc(mem); }
    }

linalg/amgxsolver.cpp

@@ -807,6 +807,9 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,
 
 void AmgXSolver::SetOperator(const Operator& op)
 {
+   height = op.Height();
+   width = op.Width();
+
    if (const SparseMatrix* Aptr =
           dynamic_cast<const SparseMatrix*>(&op))
    {

linalg/auxiliary.cpp

@@ -0,0 +1,591 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "linalg.hpp"
+#include "../fem/pfespace.hpp"
+#include "../fem/pbilinearform.hpp"
+
+namespace mfem
+{
+
+GeneralAMS::GeneralAMS(const mfem::Operator& A,
+                       const mfem::Operator& pi,
+                       const mfem::Operator& g,
+                       const mfem::Operator& pispacesolver,
+                       const mfem::Operator& gspacesolver,
+                       const mfem::Operator& smoother,
+                       const mfem::Array<int>& ess_tdof_list)
+   :
+   mfem::Solver(A.Height()),
+   A_(A),
+   pi_(pi),
+   g_(g),
+   pispacesolver_(pispacesolver),
+   gspacesolver_(gspacesolver),
+   smoother_(smoother),
+   ess_tdof_list_(ess_tdof_list),
+   residual_time_(0.0),
+   smooth_time_(0.0),
+   gspacesolver_time_(0.0),
+   pispacesolver_time_(0.0)
+{
+   // could assert a bunch of sizes...
+}
+
+GeneralAMS::~GeneralAMS()
+{
+}
+
+void GeneralAMS::FormResidual(const mfem::Vector& rhs, const mfem::Vector& x,
+                              mfem::Vector& residual) const
+{
+   chrono_.Clear();
+   chrono_.Start();
+
+   A_.Mult(x, residual);
+   residual *= -1.0;
+   residual += rhs;
+
+   chrono_.Stop();
+   residual_time_ += chrono_.RealTime();
+}
+
+void GeneralAMS::Mult(const Vector& x, Vector& y) const
+{
+   /* REMOVE?
+    int rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+   */
+
+   // see hypre_ParCSRSubspacePrec() in hypre/src/parcsr_ls/ams.c
+   // and also hypre_AMSSolve() in the same file
+
+   // default cyle (cycle 1) is "01210"
+   // ie, smooth, correct in space 1, correct in space 2, correct in space 1, smooth
+   // Bi[0] = ams_data -> B_G;    HBi[0] = (HYPRE_PtrToSolverFcn) hypre_BoomerAMGSolve;
+   // Bi[1] = ams_data -> B_Pi;   HBi[1] = (HYPRE_PtrToSolverFcn) hypre_BoomerAMGBlockSolve;
+   // (suggests space 1 is G, space 2 is Pi)
+
+   // but I think mfem::HypreAMS picks cycle 13, or 034515430
+   // with 0 smooth, three separate coordinate Pi solves, grad solve, then back up
+
+   // maybe we should do Pi-G-Pi instead of G-Pi-G (doesn't seem to make much difference)
+
+   // cycle would be more like 0102010 if below is true
+   const bool extra_smoothing = false;
+
+   mfem::StopWatch chrono;
+
+   MFEM_ASSERT(x.Size() == y.Size(), "Sizes don't match!");
+   MFEM_ASSERT(A_.Height() == x.Size(), "Sizes don't match!");
+
+   Vector residual(x.Size());
+   residual = 0.0;
+   residual.UseDevice(true);
+   y = 0.0;
+
+   // smooth (exactly what smoother is HypreAMS using?)
+   chrono.Clear();
+   chrono.Start();
+   smoother_.Mult(x, y);
+   chrono.Stop();
+   smooth_time_ += chrono.RealTime();
+
+   // g-space correction
+   FormResidual(x, y, residual);
+   Vector gspacetemp(g_.Width());
+   gspacetemp = 0.0;
+   gspacetemp.UseDevice(true);
+   g_.MultTranspose(residual, gspacetemp);
+   Vector gspacecorrection(g_.Width());
+   gspacecorrection = 0.0;
+   gspacecorrection.UseDevice(true);
+   chrono.Clear();
+   chrono.Start();
+   gspacesolver_.Mult(gspacetemp, gspacecorrection);
+   chrono.Stop();
+   gspacesolver_time_ += chrono.RealTime();
+   g_.Mult(gspacecorrection, residual);
+   y += residual;
+
+   Vector temp(x.Size());
+   if (extra_smoothing)
+   {
+      FormResidual(x, y, residual);
+      smoother_.Mult(residual, temp);
+      y += temp;
+   }
+
+   // pi-space correction
+   FormResidual(x, y, residual);
+#ifdef HYPRE_USING_CUDA
+   pispacetemp.SetSize(pi_.Width(), Device::GetDeviceMemoryType());
+#else
+   //Vector pispacetemp(pi_.Width());
+   pispacetemp.SetSize(pi_.Width());
+   pispacetemp = 0.0;
+   pispacetemp.UseDevice(true);
+#endif
+
+   pi_.MultTranspose(residual, pispacetemp);
+
+   //Vector pispacecorrection(pi_.Width());
+   pispacecorrection.SetSize(pi_.Width());
+   pispacecorrection = 0.0;
+   pispacecorrection.UseDevice(true);
+   chrono.Clear();
+   chrono.Start();
+   pispacesolver_.Mult(pispacetemp, pispacecorrection);
+   chrono.Stop();
+   pispacesolver_time_ += chrono.RealTime();
+   pi_.Mult(pispacecorrection, residual);
+   y += residual;
+
+   if (extra_smoothing)
+   {
+      FormResidual(x, y, residual);
+      smoother_.Mult(residual, temp);
+      y += temp;
+   }
+
+   // g-space correction
+   FormResidual(x, y, residual);
+   g_.MultTranspose(residual, gspacetemp);
+   gspacecorrection = 0.0;
+   chrono.Clear();
+   chrono.Start();
+   gspacesolver_.Mult(gspacetemp, gspacecorrection);
+   chrono.Stop();
+   gspacesolver_time_ += chrono.RealTime();
+   g_.Mult(gspacecorrection, residual);
+   y += residual;
+
+   // smooth (don't need the residual if smoother_ has iterative_mode ?)
+   FormResidual(x, y, residual);
+   chrono.Clear();
+   chrono.Start();
+   smoother_.Mult(residual, temp);
+   y += temp;
+   chrono.Stop();
+   smooth_time_ += chrono.RealTime();
+}
+
+// Pi-space constructor
+MatrixFreeAuxiliarySpace::MatrixFreeAuxiliarySpace(
+   MPI_Comm comm_,
+   mfem::ParMesh& mesh_lor,
+   mfem::Coefficient* alpha_coeff, mfem::Coefficient* beta_coeff,
+   mfem::MatrixCoefficient* beta_mcoeff,
+   Array<int>& ess_bdr,
+   mfem::Operator& curlcurl_oper,
+   mfem::Operator& pi,
+   int cg_iterations)
+   :
+   Solver(pi.Width()),
+   comm(comm_),
+   matfree_(NULL),
+   cg_(NULL),
+   inner_aux_iterations_(0)
+{
+   H1_FECollection * fec_lor = new H1_FECollection(1, mesh_lor.Dimension());
+   ParFiniteElementSpace fespace_lor_d(&mesh_lor, fec_lor, mesh_lor.Dimension(),
+                                       Ordering::byVDIM);
+
+   // build LOR AMG v-cycle
+   if (ess_bdr.Size())
+   {
+      fespace_lor_d.GetEssentialTrueDofs(ess_bdr, ess_tdof_list_);
+   }
+   ParBilinearForm a_space(&fespace_lor_d);
+
+   // this choice of policy is super-important for the G-space solver, but
+   // also can make some difference here
+   const Matrix::DiagonalPolicy policy = Matrix::DIAG_KEEP;
+   a_space.SetDiagonalPolicy(policy); // doesn't do anything, see Eliminate() below
+   if (alpha_coeff)
+   {
+      a_space.AddDomainIntegrator(new VectorDiffusionIntegrator(*alpha_coeff));
+   }
+   else
+   {
+      mfem_error("VectorCoefficient support not implemented?");
+   }
+
+   if (beta_coeff)
+   {
+      MFEM_VERIFY(!beta_mcoeff, "");
+      a_space.AddDomainIntegrator(new VectorMassIntegrator(*beta_coeff));
+   }
+   else if (beta_mcoeff)
+   {
+      a_space.AddDomainIntegrator(new VectorMassIntegrator(*beta_mcoeff));
+   }
+   else
+   {
+      mfem_error("Coefficient not supported");
+   }
+
+   a_space.UsePrecomputedSparsity();
+   a_space.Assemble();
+   if (ess_bdr.Size())
+   {
+      a_space.EliminateEssentialBC(ess_bdr, policy);
+   }
+   a_space.Finalize();
+   aspacematrix_ = a_space.ParallelAssemble();
+   aspacematrix_->CopyRowStarts();
+   aspacematrix_->CopyColStarts();
+
+   SetupBoomerAMG(fespace_lor_d.GetMesh()->Dimension());
+
+   if (cg_iterations > 0)
+   {
+      const bool super_duper_extra_verbose = false;
+      SetupCG(curlcurl_oper, pi, cg_iterations, super_duper_extra_verbose);
+   }
+   else
+   {
+      SetupVCycle();
+   }
+   delete fec_lor;
+}
+
+// G-space constructor
+MatrixFreeAuxiliarySpace::MatrixFreeAuxiliarySpace(
+   MPI_Comm comm_,
+   mfem::ParMesh& mesh_lor,
+   mfem::Coefficient* beta_coeff, MatrixCoefficient* beta_mcoeff,
+   Array<int>& ess_bdr, mfem::Operator& curlcurl_oper,
+   mfem::Operator& g, int cg_iterations)
+   :
+   Solver(curlcurl_oper.Height()),
+   comm(comm_),
+   matfree_(NULL),
+   cg_(NULL),
+   inner_aux_iterations_(0)
+{
+   H1_FECollection * fec_lor = new H1_FECollection(1, mesh_lor.Dimension());
+   ParFiniteElementSpace fespace_lor(&mesh_lor, fec_lor);
+
+   // build LOR AMG v-cycle
+   ParBilinearForm a_space(&fespace_lor);
+
+   // we need something like DIAG_ZERO in the solver, but explicitly doing
+   // that makes BoomerAMG setup complain, so instead we constrain the boundary
+   // in the CG solver
+   const Matrix::DiagonalPolicy policy = Matrix::DIAG_ONE;
+
+   a_space.SetDiagonalPolicy(policy);
+   if (beta_mcoeff)
+   {
+      MFEM_VERIFY(beta_coeff == NULL, "");
+      a_space.AddDomainIntegrator(new DiffusionIntegrator(*beta_mcoeff));
+   }
+   else if (beta_coeff)
+   {
+      a_space.AddDomainIntegrator(new DiffusionIntegrator(*beta_coeff));
+   }
+   else
+   {
+      mfem_error("Coefficient not supported");
+   }
+
+   a_space.UsePrecomputedSparsity();
+   a_space.Assemble();
+   if (ess_bdr.Size())
+   {
+      fespace_lor.GetEssentialTrueDofs(ess_bdr, ess_tdof_list_);
+   }
+
+   // you have to use (serial) BilinearForm eliminate routines to get
+   // diag policy DIAG_ZERO all the ParallelEliminateTDofs etc. routines
+   // implicitly have a Matrix::DIAG_KEEP policy
+   if (ess_bdr.Size())
+   {
+      a_space.EliminateEssentialBC(ess_bdr, policy);
+   }
+   a_space.Finalize();
+   aspacematrix_ = a_space.ParallelAssemble();
+
+   aspacematrix_->CopyRowStarts();
+   aspacematrix_->CopyColStarts();
+
+   SetupBoomerAMG(0);
+
+   if (cg_iterations > 0)
+   {
+      // inner CG seems necessary in G-space solver
+      SetupCG(curlcurl_oper, g, cg_iterations);
+   }
+   else
+   {
+      // this would probably be more efficient, but there are boundary condition issues?
+      SetupVCycle();
+   }
+
+   delete fec_lor;
+}
+
+void MatrixFreeAuxiliarySpace::SetupCG(
+   mfem::Operator& curlcurl_oper, mfem::Operator& conn,
+   int inner_cg_iterations, bool very_verbose)
+{
+   MFEM_ASSERT(conn.Height() == curlcurl_oper.Width(),
+               "Operators don't match!");
+   matfree_ = new RAPOperator(conn, curlcurl_oper, conn);
+   MFEM_ASSERT(matfree_->Height() == aspacepc_->Height(),
+               "Operators don't match!");
+
+   cg_ = new CGSolver(comm);
+   cg_->SetOperator(*matfree_);
+   cg_->SetPreconditioner(*aspacepc_);
+   if (inner_cg_iterations > 99)
+   {
+      cg_->SetRelTol(1.e-14);
+      cg_->SetMaxIter(100);
+   }
+   else
+   {
+      cg_->SetRelTol(0.0);
+      cg_->SetMaxIter(inner_cg_iterations);
+   }
+   if (very_verbose)
+   {
+      cg_->SetPrintLevel(1);
+   }
+   else
+   {
+      cg_->SetPrintLevel(-1);
+   }
+
+   aspacewrapper_ = cg_;
+}
+
+void MatrixFreeAuxiliarySpace::SetupVCycle()
+{
+   aspacewrapper_ = aspacepc_;
+}
+
+class ZeroWrap : public Solver
+{
+public:
+   ZeroWrap(HypreParMatrix& mat, Array<int>& ess_tdof_list, const bool useAMGx) :
+      Solver(mat.Height()),  ess_tdof_list_(ess_tdof_list)
+      //Solver(mat.Height()), amg_(mat), ess_tdof_list_(ess_tdof_list)
+   {
+#ifdef MFEM_USE_AMGX
+      if (useAMGx)
+      {
+         const bool amgx_verbose = false;
+         AmgXSolver *amgx = new AmgXSolver(mat.GetComm(), AmgXSolver::PRECONDITIONER,
+                                           amgx_verbose);
+         amgx->SetOperator(mat);
+         s = amgx;
+      }
+      else
+#endif
+      {
+         HypreBoomerAMG *amg = new HypreBoomerAMG(mat);
+         amg->SetPrintLevel(0);
+         s = amg;
+      }
+   }
+
+   void Mult(const Vector& x, Vector& y) const
+   {
+      //amg_.Mult(x, y);
+      s->Mult(x, y);
+      auto Y = y.HostReadWrite();
+      for (int k : ess_tdof_list_)
+      {
+         //y(k) = 0.0;
+         Y[k] = 0.0;
+      }
+   }
+
+   void SetOperator(const Operator&) { }
+
+   ~ZeroWrap()
+   {
+      delete s;
+   }
+
+private:
+   //HypreBoomerAMG amg_;
+   Solver *s = NULL;
+   Array<int>& ess_tdof_list_;
+};
+
+void MatrixFreeAuxiliarySpace::SetupBoomerAMG(int system_dimension)
+{
+   if (system_dimension == 0)
+   {
+      // boundary condition tweak for G-space solver
+      aspacepc_ = new ZeroWrap(*aspacematrix_, ess_tdof_list_, useAMGx);
+   }
+   else if (directSolve)
+   {
+      aspacematrix_->GetDiag(aspacematrixSP_);
+      UMFPackSolver *umf_solver = new UMFPackSolver();
+      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver->SetOperator(aspacematrixSP_);
+      aspacepc_ = umf_solver;
+   }
+   else // if (system_dimension > 0)
+   {
+      // Pi-space solver is a vector space
+#ifdef MFEM_USE_AMGX
+      if (useAMGx)
+      {
+         const bool amgx_verbose = false;
+         AmgXSolver *amgx = new AmgXSolver(aspacematrix_->GetComm(),
+                                           AmgXSolver::PRECONDITIONER, amgx_verbose);
+         amgx->SetOperator(*aspacematrix_);
+         aspacepc_ = amgx;
+      }
+      else
+#endif
+      {
+         HypreBoomerAMG* hpc = new HypreBoomerAMG(*aspacematrix_);
+         hpc->SetSystemsOptions(system_dimension);
+         hpc->SetPrintLevel(0);
+         aspacepc_ = hpc;
+      }
+   }
+}
+
+void MatrixFreeAuxiliarySpace::Mult(const mfem::Vector& x,
+                                    mfem::Vector& y) const
+{
+   int rank;
+   //MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+   MPI_Comm_rank(comm, &rank);
+
+   y = 0.0;
+   aspacewrapper_->Mult(x, y);
+   if (cg_ && rank == 0)
+   {
+      int q = cg_->GetNumIterations();
+      inner_aux_iterations_ += q;
+   }
+}
+
+MatrixFreeAuxiliarySpace::~MatrixFreeAuxiliarySpace()
+{
+   /*
+    int rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+   */
+
+   delete aspacematrix_;
+   delete aspacepc_;
+   delete matfree_;
+   if (aspacepc_ != aspacewrapper_) { delete aspacewrapper_; }
+   if (cg_ != aspacewrapper_) { delete cg_; }
+}
+
+MatrixFreeAMS::MatrixFreeAMS(
+   ParBilinearForm& aform, Operator& oper, ParFiniteElementSpace& nd_fespace,
+   Coefficient* alpha_coeff, Coefficient* beta_coeff,
+   MatrixCoefficient* beta_mcoeff,
+   Array<int>& ess_bdr, int inner_pi_iterations,
+   int inner_g_iterations)
+   :
+   Solver(oper.Height())
+{
+   int order = nd_fespace.GetFE(0)->GetOrder();
+   ParMesh *mesh = nd_fespace.GetParMesh();
+   int dim = mesh->Dimension();
+
+   // smoother
+   const double scale = 0.25; // not so clear what exactly to put here...
+   Array<int> ess_tdof_list;
+   if (ess_bdr.Size())
+   {
+      nd_fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+   smoother_ = new OperatorJacobiSmoother(aform, ess_tdof_list, scale);
+
+   // get H1 space
+   FiniteElementCollection *h1_fec = new H1_FECollection(order, dim);
+   h1_fespace_ = new ParFiniteElementSpace(mesh, h1_fec);
+   h1_fespace_d_ = new ParFiniteElementSpace(mesh, h1_fec, dim, Ordering::byVDIM);
+
+   // build G operator
+   pa_grad_ = new ParDiscreteLinearOperator(h1_fespace_, &nd_fespace);
+   pa_grad_->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   pa_grad_->AddDomainInterpolator(new GradientInterpolator);
+   pa_grad_->Assemble();
+   pa_grad_->FormRectangularSystemMatrix(G_);
+
+   // build Pi operator
+   pa_interp_ = new ParDiscreteLinearOperator(h1_fespace_d_, &nd_fespace);
+   pa_interp_->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   pa_interp_->AddDomainInterpolator(new IdentityInterpolator);
+   pa_interp_->Assemble();
+   pa_interp_->FormRectangularSystemMatrix(Pi_);
+
+   // build LOR space
+   ParMesh mesh_lor(mesh, order, BasisType::GaussLobatto);
+
+   /* A lot depends on the quality of the auxiliary space solves.
+      For high-contrast coefficients, and other difficult problems,
+      inner iteration counts may need to be increased.
+
+      Boundary conditions can matter as well (see DIAG_ZERO policy) */
+
+   // build G space solver
+   Gspacesolver_ = new MatrixFreeAuxiliarySpace(nd_fespace.GetComm(), mesh_lor,
+                                                beta_coeff,
+                                                beta_mcoeff, ess_bdr, oper, *G_, inner_g_iterations);
+
+   // build Pi space solver
+   Pispacesolver_ = new MatrixFreeAuxiliarySpace(nd_fespace.GetComm(), mesh_lor,
+                                                 alpha_coeff, beta_coeff,
+                                                 beta_mcoeff,
+                                                 ess_bdr, oper, *Pi_,
+                                                 inner_pi_iterations);
+
+   general_ams_ = new GeneralAMS(oper, *Pi_, *G_, *Pispacesolver_,
+                                 *Gspacesolver_, *smoother_, ess_tdof_list);
+
+   delete h1_fec;
+}
+
+MatrixFreeAMS::~MatrixFreeAMS()
+{
+   delete smoother_;
+   // delete serialPi_;
+   // delete Pi_;
+   // delete serialG_;
+   // delete G_;
+   delete pa_grad_;
+   delete pa_interp_;
+   delete Gspacesolver_;
+   delete Pispacesolver_;
+   delete general_ams_;
+   delete h1_fespace_;
+   delete h1_fespace_d_;
+}
+
+void MatrixFreeAMS::PrintTimings(const int myid)
+{
+   general_ams_->PrintTimings(myid);
+}
+
+void GeneralAMS::PrintTimings(const int myid)
+{
+   std::cout << myid << ": AMS timing for residual " << residual_time_ <<
+             ", smooth " << smooth_time_
+             << ", G " << gspacesolver_time_ << ", Pi " << pispacesolver_time_ << std::endl;
+}
+
+} // namespace mfem

linalg/auxiliary.hpp

@@ -0,0 +1,201 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_AUXILIARY
+#define MFEM_AUXILIARY
+
+#include "../config/config.hpp"
+#include "../general/tic_toc.hpp"
+
+#include "solvers.hpp"
+
+namespace mfem
+{
+
+// forward declarations (can probably be reduced/simplified
+class Coefficient;
+class MatrixCoefficient;
+class ParMesh;
+class ParBilinearForm;
+class ParDiscreteLinearOperator;
+
+/**
+   The basic idea is that given an operator A and a transfer
+   G, this will create a solver that approximates (G^T A G)^{-1}
+
+   In practice we only use this for an AMS cycle, so some of
+   the notation and algorithmic choices are specific to that.
+*/
+class MatrixFreeAuxiliarySpace : public mfem::Solver
+{
+public:
+   /**
+      Pi space constructor (two coefficients)
+
+      cg_iterations = 0 means a single V-cycle
+      otherwise we wrap BoomerAMG in CG
+
+      rap_in_lor does a RAP product in the LOR space
+      for building the matrix
+   */
+   MatrixFreeAuxiliarySpace(MPI_Comm comm_,
+                            mfem::ParMesh& mesh_lor,
+                            mfem::Coefficient* alpha_coeff, mfem::Coefficient* beta_coeff,
+                            MatrixCoefficient* beta_mcoeff,
+                            Array<int>& ess_bdr,
+                            mfem::Operator& curlcurl_oper, mfem::Operator& pi,
+                            int cg_iterations = 0);
+
+   /**
+      G space constructor (one coefficient)
+
+      cg_iterations = 0 means a single V-cycle
+      otherwise we wrap BoomerAMG in CG
+
+      rap_in_lor does a RAP product in the LOR space
+      for building the matrix
+   */
+   MatrixFreeAuxiliarySpace(MPI_Comm comm_,
+                            mfem::ParMesh& mesh_lor,
+                            mfem::Coefficient* beta_coeff, MatrixCoefficient* beta_mcoeff,
+                            Array<int>& ess_bdr, mfem::Operator& curlcurl_oper,
+                            mfem::Operator& g, int cg_iterations = 1);
+
+   ~MatrixFreeAuxiliarySpace();
+
+   void Mult(const mfem::Vector& x, mfem::Vector& y) const;
+
+   void SetOperator(const mfem::Operator& op) {}
+
+private:
+   void SetupBoomerAMG(int system_dimension);
+   void SetupVCycle();
+
+   /// inner_cg_iterations > 99 applies an exact solve here
+   void SetupCG(
+      mfem::Operator& curlcurl_oper, mfem::Operator& conn,
+      int inner_cg_iterations, bool very_verbose=false);
+
+   mfem::Array<int> ess_tdof_list_;
+   mfem::HypreParMatrix * aspacematrix_;
+   // mfem::HypreBoomerAMG * aspacepc_;
+   Solver * aspacepc_;
+   mfem::Operator* matfree_;
+   mfem::CGSolver* cg_;
+   mfem::Operator* aspacewrapper_;
+
+   mutable int inner_aux_iterations_;
+
+   MPI_Comm comm;
+
+   const bool directSolve = false;
+   SparseMatrix aspacematrixSP_;
+
+   const bool useAMGx = true;
+};
+
+/**
+   Perform AMS cycle with generic Operator objects.
+
+   Most users should use MatrixFreeAMS, which wraps this.
+*/
+class GeneralAMS : public mfem::Solver
+{
+public:
+   /**
+      pi and g should have Mult() and MultTranspose()
+
+      the rest just nead Mult()
+   */
+   GeneralAMS(const mfem::Operator& A,
+              const mfem::Operator& pi,
+              const mfem::Operator& g,
+              const mfem::Operator& pispacesolver,
+              const mfem::Operator& gspacesolver,
+              const mfem::Operator& smoother,
+              const mfem::Array<int>& ess_tdof_list);
+   virtual ~GeneralAMS();
+
+   /// in principle this should set A_ = op;
+   void SetOperator(const mfem::Operator &op) {}
+
+   virtual void Mult(const mfem::Vector& x, mfem::Vector& y) const;
+
+   void PrintTimings(const int myid);
+
+private:
+   const mfem::Operator& A_;
+   const mfem::Operator& pi_;
+   const mfem::Operator& g_;
+   const mfem::Operator& pispacesolver_;
+   const mfem::Operator& gspacesolver_;
+   const mfem::Operator& smoother_;
+   const mfem::Array<int> ess_tdof_list_;
+
+   mutable mfem::StopWatch chrono_;
+
+   mutable double residual_time_;
+   mutable double smooth_time_;
+   mutable double gspacesolver_time_;
+   mutable double pispacesolver_time_;
+
+   mutable Vector pispacetemp;
+   mutable Vector pispacecorrection;
+
+   void FormResidual(const mfem::Vector& rhs, const mfem::Vector& x,
+                     mfem::Vector& residual) const;
+};
+
+/**
+   An auxiliary Maxwell solver for high-order finite element operators without
+   high-order assembly.
+
+   The auxiliary space solves are done using a low-order refined approach, but
+   all the interpolation operators, residuals, etc. are done in a matrix-free
+   manner.
+*/
+class MatrixFreeAMS : public mfem::Solver
+{
+public:
+   /// ess_bdr is the boundary attributes that are essential (not the dofs, the attributes)
+   MatrixFreeAMS(ParBilinearForm& aform, mfem::Operator& oper,
+                 mfem::ParFiniteElementSpace& nd_fespace,
+                 mfem::Coefficient* alpha_coeff, mfem::Coefficient* beta_coeff,
+                 mfem::MatrixCoefficient* beta_mcoeff,
+                 mfem::Array<int>& ess_bdr,
+                 int inner_pi_iterations = 0, int inner_g_iterations = 1);
+   ~MatrixFreeAMS();
+
+   void SetOperator(const mfem::Operator &op) {}
+
+   void Mult(const mfem::Vector& x, mfem::Vector& y) const { general_ams_->Mult(x, y); }
+
+   void PrintTimings(const int myid);
+
+private:
+   GeneralAMS * general_ams_;
+
+   Solver * smoother_;
+   ParDiscreteLinearOperator * pa_grad_;
+   OperatorPtr G_;
+   ParDiscreteLinearOperator * pa_interp_;
+   OperatorPtr Pi_;
+
+   mfem::Solver * Gspacesolver_;
+   mfem::Solver * Pispacesolver_;
+
+   mfem::ParFiniteElementSpace * h1_fespace_;
+   mfem::ParFiniteElementSpace * h1_fespace_d_;
+};
+
+} // namespace mfem
+
+#endif

linalg/blockoperator.cpp

@@ -80,9 +80,13 @@ void BlockOperator::Mult (const Vector & x, Vector & y) const
    xblock.Update(const_cast<Vector&>(x),col_offsets);
    yblock.Update(y,row_offsets);
 
+   // TODO: this should not be necessary, but it is in case tmp.SetSize(0) is called for first row.
+   tmp.SetSize(10);
+
    for (int iRow=0; iRow < nRowBlocks; ++iRow)
    {
       tmp.SetSize(row_offsets[iRow+1] - row_offsets[iRow]);
+
       for (int jCol=0; jCol < nColBlocks; ++jCol)
       {
          if (op(iRow,jCol))
@@ -95,6 +99,7 @@ void BlockOperator::Mult (const Vector & x, Vector & y) const
 
    for (int iRow=0; iRow < nRowBlocks; ++iRow)
    {
+      if (yblock.BlockSize(iRow) == 0) { continue; }
       yblock.GetBlock(iRow).SyncAliasMemory(y);
    }
 
@@ -131,6 +136,7 @@ void BlockOperator::MultTranspose (const Vector & x, Vector & y) const
 
    for (int iRow=0; iRow < nColBlocks; ++iRow)
    {
+      if (yblock.BlockSize(iRow) == 0) { continue; }
       yblock.GetBlock(iRow).SyncAliasMemory(y);
    }
 
@@ -206,6 +212,7 @@ void BlockDiagonalPreconditioner::Mult (const Vector & x, Vector & y) const
 
    for (int i=0; i<nBlocks; ++i)
    {
+      if (yblock.BlockSize(i) == 0) { continue; }
       yblock.GetBlock(i).SyncAliasMemory(y);
    }
 
@@ -242,6 +249,7 @@ void BlockDiagonalPreconditioner::MultTranspose (const Vector & x,
 
    for (int i=0; i<nBlocks; ++i)
    {
+      if (yblock.BlockSize(i) == 0) { continue; }
       yblock.GetBlock(i).SyncAliasMemory(y);
    }
 
@@ -381,4 +389,55 @@ BlockLowerTriangularPreconditioner::~BlockLowerTriangularPreconditioner()
    }
 }
 
+void BlockGSPreconditioner::Mult(const Vector & x, Vector & y) const
+{
+   MFEM_ASSERT(x.Size() == width, "incorrect input Vector size");
+   MFEM_ASSERT(y.Size() == height, "incorrect output Vector size");
+
+   x.Read();
+   y.Write(); y = 0.0;
+
+   xblock.Update(const_cast<Vector&>(x),col_offsets);
+   yblock.Update(y,row_offsets);
+
+   // TODO: this should not be necessary, but it is in case tmp.SetSize(0) is called for first row.
+   tmp.SetSize(10);
+
+   for (int iter=0; iter<numIter; ++iter)
+   {
+      for (int iRow=0; iRow < nRowBlocks; ++iRow)
+      {
+         tmp.SetSize(row_offsets[iRow+1] - row_offsets[iRow]);
+
+         yblock.GetBlock(iRow) = xblock.GetBlock(iRow);
+
+         for (int jCol=0; jCol < nColBlocks; ++jCol)
+         {
+            if (iRow != jCol && !blocks->IsZeroBlock(iRow,jCol))
+            {
+               blocks->GetBlock(iRow,jCol).Mult(yblock.GetBlock(jCol), tmp);
+               yblock.GetBlock(iRow).Add(-blocks->GetBlockCoef(iRow,jCol), tmp);
+            }
+         }
+
+         blocks->GetBlock(iRow,iRow).Mult(yblock.GetBlock(iRow),
+                                          tmp); // inverse or preconditioner
+         MFEM_VERIFY(blocks->GetBlockCoef(iRow,iRow) == 1.0,
+                     "If not 1, choose a convention for how to scale");
+         yblock.GetBlock(iRow) = tmp;
+      }
+   }
+
+   for (int iRow=0; iRow < nRowBlocks; ++iRow)
+   {
+      if (yblock.BlockSize(iRow) == 0) { continue; }
+      yblock.GetBlock(iRow).SyncAliasMemory(y);
+   }
+
+   // Destroy alias vectors to prevent dangling aliases when the base vectors
+   // are deleted
+   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
+   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
+}
+
 }

linalg/blockoperator.hpp

@@ -282,5 +282,41 @@ private:
    mutable Vector tmp2;
 };
 
+class BlockGSPreconditioner : public Solver
+{
+public:
+   //! Constructor that specifies the block structure
+   BlockGSPreconditioner(BlockOperator *blocks_, const int iter)
+      : blocks(blocks_), row_offsets(blocks_->RowOffsets()),
+        col_offsets(blocks_->ColOffsets()), nRowBlocks(blocks_->NumRowBlocks()),
+        nColBlocks(blocks_->NumColBlocks()), numIter(iter)
+   { }
+
+   /// Operator application
+   virtual void Mult(const Vector & x, Vector & y) const;
+
+   //! This method is present since required by the abstract base class Solver
+   virtual void SetOperator(const Operator &op) { }
+
+private:
+
+   BlockOperator
+   *blocks;  // Assumption: diagonal blocks are inverses or preconditioners, while off-diagonal blocks are operator blocks.
+
+   //! Number of Blocks
+   int nRowBlocks;
+   int nColBlocks;
+
+   Array<int> row_offsets;
+   Array<int> col_offsets;
+
+   const int numIter;
+
+   mutable BlockVector xblock;
+   mutable BlockVector yblock;
+
+   mutable Vector tmp;
+};
+
 }
 #endif /* MFEM_BLOCKOPERATOR */

linalg/blockvector.cpp

@@ -99,6 +99,7 @@ void BlockVector::Update(Vector & data, const Array<int> & bOffsets)
 
    for (int i = 0; i < numBlocks; ++i)
    {
+      if (BlockSize(i) == 0) { continue; }
       blocks[i].MakeRef(data, blockOffsets[i], BlockSize(i));
    }
 }

linalg/hypre.cpp

@@ -34,13 +34,23 @@ template<typename TargetT, typename SourceT>
 static TargetT *DuplicateAs(const SourceT *array, int size,
                             bool cplusplus = true)
 {
-   TargetT *target_array = cplusplus ? (TargetT*) Memory<TargetT>(size)
-                           /*     */ : mfem_hypre_TAlloc(TargetT, size);
-   for (int i = 0; i < size; i++)
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
    {
-      target_array[i] = array[i];
+      TargetT *target_array = cplusplus ? (TargetT*) Memory<TargetT>(size)
+                              /*     */ : mfem_hypre_TAlloc(TargetT, size);
+      for (int i = 0; i < size; i++)
+      {
+         target_array[i] = array[i];
+      }
+      return target_array;
+   }
+   else
+   {
+      Memory<TargetT> mm(size, GetHypreMemoryType());
+      mm.CopyFromHost(array, size);
+      TargetT *target_array = mm;
+      return target_array;
    }
-   return target_array;
 }
 
 inline void HypreParVector::_SetDataAndSize_()
@@ -251,13 +261,34 @@ HypreParMatrix::HypreParMatrix()
 
 char HypreParMatrix::CopyCSR(SparseMatrix *csr, hypre_CSRMatrix *hypre_csr)
 {
-   hypre_CSRMatrixData(hypre_csr) = csr->GetData();
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      hypre_CSRMatrixData(hypre_csr) = csr->GetData();
 #ifndef HYPRE_BIGINT
-   hypre_CSRMatrixI(hypre_csr) = csr->GetI();
-   hypre_CSRMatrixJ(hypre_csr) = csr->GetJ();
-   // Prevent hypre from destroying hypre_csr->{i,j,data}
-   return 0;
-#else
+      hypre_CSRMatrixI(hypre_csr) = csr->GetI();
+      hypre_CSRMatrixJ(hypre_csr) = csr->GetJ();
+      // Prevent hypre from destroying hypre_csr->{i,j,data}
+      return 0;
+#endif
+   }
+   else
+   {
+      const int nnz = csr->GetI()[csr->NumRows()];
+      Memory<double> hypre_mem_data(nnz, GetHypreMemoryType());
+      hypre_mem_data.CopyFromHost(csr->GetData(), nnz);
+
+      hypre_CSRMatrixData(hypre_csr) = hypre_mem_data;
+#ifndef HYPRE_BIGINT
+      hypre_CSRMatrixI(hypre_csr) = DuplicateAs<int>(csr->GetI(),
+                                                     csr->Height()+1);
+      hypre_CSRMatrixJ(hypre_csr) = DuplicateAs<int>(csr->GetJ(),
+                                                     csr->NumNonZeroElems());
+      // Prevent hypre from destroying hypre_csr->{i,j,data}
+      return 0;
+#endif
+   }
+
+#ifdef HYPRE_BIGINT
    hypre_CSRMatrixI(hypre_csr) =
       DuplicateAs<HYPRE_Int>(csr->GetI(), csr->Height()+1);
    hypre_CSRMatrixJ(hypre_csr) =
@@ -306,8 +337,20 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, HYPRE_Int glob_size,
    : Operator(diag->Height(), diag->Width())
 {
    Init();
-   A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, row_starts,
-                                row_starts, 0, diag->NumNonZeroElems(), 0);
+
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, row_starts,
+                                   row_starts, 0, diag->NumNonZeroElems(), 0);
+   }
+   else
+   {
+      hypre_mem_row.New(3, GetHypreMemoryType());
+      hypre_mem_row.CopyFromHost(row_starts, 3);
+      A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, hypre_mem_row,
+                                   hypre_mem_row, 0, diag->NumNonZeroElems(), 0);
+   }
+
    hypre_ParCSRMatrixSetDataOwner(A,1);
    hypre_ParCSRMatrixSetRowStartsOwner(A,0);
    hypre_ParCSRMatrixSetColStartsOwner(A,0);
@@ -565,11 +608,30 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
       diag_nnz = i_diag[row[1]-row[0]];
       offd_nnz = i_offd[row[1]-row[0]];
 
-      A = hypre_ParCSRMatrixCreate(comm, row[2], col[2], row, col,
-                                   cmap_size, diag_nnz, offd_nnz);
+      if (GetHypreMemoryClass() == MemoryClass::HOST)
+      {
+         A = hypre_ParCSRMatrixCreate(comm, row[2], col[2], row, col,
+                                      cmap_size, diag_nnz, offd_nnz);
+      }
+      else
+      {
+         // Assumption: all input pointers to host memory have valid data already on host, so we can just use CopyFromHost.
+         // Copy row, col to new arrays with hypre memory type.
+
+         hypre_mem_row.New(3, GetHypreMemoryType());
+         hypre_mem_col.New(3, GetHypreMemoryType());
+
+         hypre_mem_row.CopyFromHost(row, 3);
+         hypre_mem_col.CopyFromHost(col, 3);
+
+         A = hypre_ParCSRMatrixCreate(comm, row[2], col[2], hypre_mem_row, hypre_mem_col,
+                                      cmap_size, diag_nnz, offd_nnz);
+      }
    }
    else
    {
+      MFEM_VERIFY(GetHypreMemoryClass() == MemoryClass::HOST, "TODO");
+
       diag_nnz = i_diag[row[id+1]-row[id]];
       offd_nnz = i_offd[row[id+1]-row[id]];
 
@@ -583,35 +645,94 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
 
    HYPRE_Int i;
 
-   double *a_diag = Memory<double>(diag_nnz);
+   double *a_diag = Memory<double>(diag_nnz, GetHypreMemoryType());
+
    for (i = 0; i < diag_nnz; i++)
    {
       a_diag[i] = 1.0;
    }
 
-   double *a_offd = Memory<double>(offd_nnz);
+   double *a_offd = Memory<double>(offd_nnz, GetHypreMemoryType());
    for (i = 0; i < offd_nnz; i++)
    {
       a_offd[i] = 1.0;
    }
 
    hypre_CSRMatrixSetDataOwner(A->diag,0);
-   hypre_CSRMatrixI(A->diag)    = i_diag;
-   hypre_CSRMatrixJ(A->diag)    = j_diag;
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      hypre_CSRMatrixI(A->diag)    = i_diag;
+      hypre_CSRMatrixJ(A->diag)    = j_diag;
+   }
+   else
+   {
+      Memory<HYPRE_Int> hypre_mem_i_diag, hypre_mem_j_diag;
+
+      if (HYPRE_AssumedPartitionCheck())
+      {
+         const int rowSize = row[1]-row[0];
+         hypre_mem_i_diag.New(rowSize+1, GetHypreMemoryType());
+         hypre_mem_j_diag.New(diag_nnz, GetHypreMemoryType());
+
+         hypre_mem_i_diag.CopyFromHost(i_diag, rowSize+1);
+         hypre_mem_j_diag.CopyFromHost(j_diag, diag_nnz);
+      }
+      else
+      {
+         MFEM_VERIFY(false, "TODO");
+      }
+
+      hypre_CSRMatrixI(A->diag)    = hypre_mem_i_diag;
+      hypre_CSRMatrixJ(A->diag)    = hypre_mem_j_diag;
+   }
    hypre_CSRMatrixData(A->diag) = a_diag;
    hypre_CSRMatrixSetRownnz(A->diag);
    // Prevent hypre from destroying A->diag->{i,j,data}, own A->diag->{i,j,data}
    diagOwner = 3;
 
    hypre_CSRMatrixSetDataOwner(A->offd,0);
-   hypre_CSRMatrixI(A->offd)    = i_offd;
-   hypre_CSRMatrixJ(A->offd)    = j_offd;
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      hypre_CSRMatrixI(A->offd)    = i_offd;
+      hypre_CSRMatrixJ(A->offd)    = j_offd;
+   }
+   else
+   {
+      Memory<HYPRE_Int> hypre_mem_i_offd, hypre_mem_j_offd;
+
+      if (HYPRE_AssumedPartitionCheck())
+      {
+         const int rowSize = row[1]-row[0];
+         hypre_mem_i_offd.New(rowSize+1, GetHypreMemoryType());
+         hypre_mem_j_offd.New(offd_nnz, GetHypreMemoryType());
+
+         hypre_mem_i_offd.CopyFromHost(i_offd, rowSize+1);
+         hypre_mem_j_offd.CopyFromHost(j_offd, offd_nnz);
+      }
+      else
+      {
+         MFEM_VERIFY(false, "TODO");
+      }
+
+      hypre_CSRMatrixI(A->offd)    = hypre_mem_i_offd;
+      hypre_CSRMatrixJ(A->offd)    = hypre_mem_j_offd;
+   }
    hypre_CSRMatrixData(A->offd) = a_offd;
    hypre_CSRMatrixSetRownnz(A->offd);
    // Prevent hypre from destroying A->offd->{i,j,data}, own A->offd->{i,j,data}
    offdOwner = 3;
 
-   hypre_ParCSRMatrixColMapOffd(A) = cmap;
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      hypre_ParCSRMatrixColMapOffd(A) = cmap;
+   }
+   else
+   {
+      hypre_mem_cmap.New(cmap_size, GetHypreMemoryType());
+      hypre_mem_cmap.CopyFromHost(cmap, cmap_size);
+      hypre_ParCSRMatrixColMapOffd(A) = hypre_mem_cmap;
+   }
+
    // Prevent hypre from destroying A->col_map_offd, own A->col_map_offd
    colMapOwner = 1;
 
@@ -1394,6 +1515,7 @@ HypreParMatrix* HypreParMatrix::EliminateCols(const Array<int> &cols)
    get_sorted_rows_cols(cols, rc_sorted);
 
    hypre_ParCSRMatrix* Ae;
+
    internal::hypre_ParCSRMatrixEliminateAAe(
       A, &Ae, rc_sorted.Size(), rc_sorted.GetData(), 1);
 
@@ -1492,28 +1614,58 @@ void HypreParMatrix::PrintCommPkg(std::ostream &out) const
 inline void delete_hypre_CSRMatrixData(hypre_CSRMatrix *M)
 {
    HYPRE_Complex *data = hypre_CSRMatrixData(M);
-   Memory<HYPRE_Complex>(data, M->num_nonzeros, true).Delete();
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      Memory<HYPRE_Complex>(data, M->num_nonzeros, true).Delete();
+   }
+   else
+   {
+      Memory<HYPRE_Complex>(data, M->num_nonzeros, GetHypreMemoryType(),
+                            true).Delete();
+   }
 }
 
 inline void delete_hypre_ParCSRMatrixColMapOffd(hypre_ParCSRMatrix *A)
 {
    HYPRE_Int  *A_col_map_offd = hypre_ParCSRMatrixColMapOffd(A);
    int size = hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(A));
-   Memory<HYPRE_Int>(A_col_map_offd, size, true).Delete();
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      Memory<HYPRE_Int>(A_col_map_offd, size, true).Delete();
+   }
+   else
+   {
+      Memory<HYPRE_Int>(A_col_map_offd, size, GetHypreMemoryType(), true).Delete();
+   }
 }
 
 inline void delete_hypre_CSRMatrixI(hypre_CSRMatrix *M)
 {
    HYPRE_Int *I = hypre_CSRMatrixI(M);
    int size = hypre_CSRMatrixNumRows(M) + 1;
-   Memory<HYPRE_Int>(I, size, true).Delete();
+
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      Memory<HYPRE_Int>(I, size, true).Delete();
+   }
+   else
+   {
+      Memory<HYPRE_Int>(I, size, GetHypreMemoryType(), true).Delete();
+   }
 }
 
 inline void delete_hypre_CSRMatrixJ(hypre_CSRMatrix *M)
 {
    HYPRE_Int *J = hypre_CSRMatrixJ(M);
    int size = hypre_CSRMatrixNumNonzeros(M);
-   Memory<HYPRE_Int>(J, size, true).Delete();
+   if (GetHypreMemoryClass() == MemoryClass::HOST)
+   {
+      Memory<HYPRE_Int>(J, size, true).Delete();
+   }
+   else
+   {
+      Memory<HYPRE_Int>(J, size, GetHypreMemoryType(), true).Delete();
+   }
 }
 
 void HypreParMatrix::Destroy()
@@ -1643,7 +1795,16 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
       hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
 
    hypre_ParCSRMatrix * rap;
+#ifdef HYPRE_USING_CUDA
+   {
+      hypre_ParCSRMatrix *Q = hypre_ParCSRMatMat(*A,*P);
+      const bool keepTranspose = false;
+      rap = hypre_ParCSRTMatMatKT(*P,Q,keepTranspose);
+      delete Q;
+   }
+#else
    hypre_BoomerAMGBuildCoarseOperator(*P,*A,*P,&rap);
+#endif
    hypre_ParCSRMatrixSetNumNonzeros(rap);
    // hypre_MatvecCommPkgCreate(rap);
 
@@ -1669,7 +1830,15 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
       hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*Rt));
 
    hypre_ParCSRMatrix * rap;
+#ifdef HYPRE_USING_CUDA
+   {
+      hypre_ParCSRMatrix *Q = hypre_ParCSRMatMat(*A,*P);
+      rap = hypre_ParCSRTMatMat(*Rt,Q);
+      delete Q;
+   }
+#else
    hypre_BoomerAMGBuildCoarseOperator(*Rt,*A,*P,&rap);
+#endif
 
    hypre_ParCSRMatrixSetNumNonzeros(rap);
    // hypre_MatvecCommPkgCreate(rap);
@@ -2546,26 +2715,22 @@ void HypreSolver::Mult(const Vector &b, Vector &x) const
       mfem_error("HypreSolver::Mult (...) : HypreParMatrix A is missing");
       return;
    }
-   auto b_data = b.HostRead();
-   auto x_data = x.HostWrite();
    if (B == NULL)
    {
       B = new HypreParVector(A->GetComm(),
                              A -> GetGlobalNumRows(),
-                             const_cast<double*>(b_data),
+                             nullptr,
                              A -> GetRowStarts());
       X = new HypreParVector(A->GetComm(),
                              A -> GetGlobalNumCols(),
-                             x_data,
+                             nullptr,
                              A -> GetColStarts());
    }
-   else
-   {
-      B -> SetData(const_cast<double*>(b_data));
-      X -> SetData(x_data);
-   }
+   B->Read(b);
+   X->Write(x);
 
    Mult(*B, *X);
+   X->WriteCopy(x);
 }
 
 HypreSolver::~HypreSolver()
@@ -3010,6 +3175,10 @@ HypreParaSails::~HypreParaSails()
    HYPRE_ParaSailsDestroy(sai_precond);
 }
 
+HypreEuclid::~HypreEuclid()
+{
+   HYPRE_EuclidDestroy(euc_precond);
+}
 
 HypreEuclid::HypreEuclid(MPI_Comm comm)
 {
@@ -3074,11 +3243,6 @@ void HypreEuclid::SetOperator(const Operator &op)
    B = X = NULL;
 }
 
-HypreEuclid::~HypreEuclid()
-{
-   HYPRE_EuclidDestroy(euc_precond);
-}
-
 
 HypreBoomerAMG::HypreBoomerAMG()
 {
@@ -3094,6 +3258,7 @@ HypreBoomerAMG::HypreBoomerAMG(HypreParMatrix &A) : HypreSolver(&A)
 
 void HypreBoomerAMG::SetDefaultOptions()
 {
+#ifndef HYPRE_USING_CUDA
    // AMG coarsening options:
    int coarsen_type = 10;   // 10 = HMIS, 8 = PMIS, 6 = Falgout, 0 = CLJP
    int agg_levels   = 1;    // number of aggressive coarsening levels
@@ -3110,10 +3275,32 @@ void HypreBoomerAMG::SetDefaultOptions()
    // Additional options:
    int print_level  = 1;    // print AMG iterations? 1 = no, 2 = yes
    int max_levels   = 25;   // max number of levels in AMG hierarchy
+#else
+   // AMG coarsening options:
+   int coarsen_type = 8;    // 10 = HMIS, 8 = PMIS, 6 = Falgout, 0 = CLJP
+   int agg_levels   = 0;    // number of aggressive coarsening levels
+   double theta     = 0.25; // strength threshold: 0.25, 0.5, 0.8
+
+   // AMG interpolation options:
+   int interp_type  = 15;   // or 3 = direct
+   int Pmax         = 4;    // max number of elements per row in P
+
+   // AMG relaxation options:
+   int relax_type   = 7;    // or 18 = l1-Jacobi
+   int relax_sweeps = 1;    // relaxation sweeps on each level
+
+   // Additional options:
+   int print_level  = 1;    // print AMG iterations? 1 = no, 2 = yes
+   int max_levels   = 25;   // max number of levels in AMG hierarchy
+#endif
 
    HYPRE_BoomerAMGSetCoarsenType(amg_precond, coarsen_type);
    HYPRE_BoomerAMGSetAggNumLevels(amg_precond, agg_levels);
    HYPRE_BoomerAMGSetRelaxType(amg_precond, relax_type);
+   if (relax_type == 7)
+   {
+      HYPRE_BoomerAMGSetRelaxWt(amg_precond, 1.0); // this is the hypre default
+   }
    HYPRE_BoomerAMGSetNumSweeps(amg_precond, relax_sweeps);
    HYPRE_BoomerAMGSetStrongThreshold(amg_precond, theta);
    HYPRE_BoomerAMGSetInterpType(amg_precond, interp_type);
@@ -4421,6 +4608,471 @@ HypreAME::StealEigenvectors()
    return vecs;
 }
 
+#define HELMHOLTZ_AMS
+#define HELMHOLTZ_AMS_BC
+#define CSL_CORRECTION
+
+#ifdef HYPRE_DYLAN
+HypreIAMS::HypreIAMS(HypreParMatrix &A, HypreParMatrix *H, STRUMPACKSolver *CSL,
+                     BlockVector *trueBlockX, BlockVector *trueBlockY,
+                     HypreAMS *ams, int argc, char *argv[])
+   : m_ams(ams), m_Pix(ams->Get_Pix(), false), m_Piy(ams->Get_Piy(), false),
+     m_Piz(ams->Get_Piz(), false), m_G(ams->Get_G(), false),
+     z(ams->Get_Pix()->comm, hypre_ParCSRMatrixGlobalNumCols(ams->Get_Pix()),
+       hypre_ParCSRMatrixColStarts(ams->Get_Pix())),
+     w(ams->Get_Pix()->comm, hypre_ParCSRMatrixGlobalNumCols(ams->Get_Pix()),
+       hypre_ParCSRMatrixColStarts(ams->Get_Pix())),
+     v(ams->Get_Pix()->comm, hypre_ParCSRMatrixGlobalNumRows(ams->Get_Pix()),
+       hypre_ParCSRMatrixRowStarts(ams->Get_Pix())),
+     r(ams->Get_Pix()->comm, hypre_ParCSRMatrixGlobalNumRows(ams->Get_Pix()),
+       hypre_ParCSRMatrixRowStarts(ams->Get_Pix())),
+     smoother(A, HypreSmoother::Kaczmarz), m_A(&A), m_CSL(CSL),
+     m_trueBlockX(trueBlockX), m_trueBlockY(trueBlockY)
+{
+   HypreParMatrix A_G(ams->Get_A_G());
+
+   int myid = -1;
+   MPI_Comm_rank(ams->Get_Pix()->comm, &myid);
+
+   //cout << myid << ": r size " << r.Size() << endl;
+
+#ifdef HELMHOLTZ_AMS
+#ifdef HELMHOLTZ_AMS_BC
+   Arow[0] = new STRUMPACKRowLocMatrix(H[0]);
+   Arow[1] = new STRUMPACKRowLocMatrix(H[1]);
+   Arow[2] = new STRUMPACKRowLocMatrix(H[2]);
+   Arow[3] = new STRUMPACKRowLocMatrix(A_G);
+
+   for (int i=0; i<4; ++i)
+#else
+   Arow[0] = new STRUMPACKRowLocMatrix(*H);
+   Arow[1] = new STRUMPACKRowLocMatrix(A_G);
+
+   for (int i=0; i<2; ++i)
+#endif
+#else
+   HypreParMatrix A_Pix(ams->Get_A_Pix());
+   HypreParMatrix A_Piy(ams->Get_A_Piy());
+   HypreParMatrix A_Piz(ams->Get_A_Piz());
+   HypreParMatrix A_G(ams->Get_A_G());
+
+   Arow[0] = new STRUMPACKRowLocMatrix(A_Pix);
+   Arow[1] = new STRUMPACKRowLocMatrix(A_Piy);
+   Arow[2] = new STRUMPACKRowLocMatrix(A_Piz);
+   Arow[3] = new STRUMPACKRowLocMatrix(A_G);
+
+   for (int i=0; i<4; ++i)
+#endif
+   {
+      strumpack[i] = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack[i]->SetPrintFactorStatistics(true);
+      strumpack[i]->SetPrintSolveStatistics(false);
+      strumpack[i]->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack[i]->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack[i]->SetOperator(*Arow[i]);
+      strumpack[i]->SetFromCommandLine();
+   }
+}
+
+void HypreIAMS::SetOperator(const Operator &op)
+{
+
+}
+
+void HypreIAMS::MultAdditive(const mfem::Vector &x, mfem::Vector &y) const
+{
+   r = 0.0;
+   r += x;
+
+   m_Pix.MultTranspose(r, w);
+   strumpack[0]->Mult(w, z);
+   m_Pix.Mult(z, v);
+   y += v;
+
+   m_Piy.MultTranspose(r, w);
+   strumpack[1]->Mult(w, z);
+   m_Piy.Mult(z, v);
+   y += v;
+
+   m_Piz.MultTranspose(r, w);
+   strumpack[2]->Mult(w, z);
+   m_Piz.Mult(z, v);
+   y += v;
+
+   m_G.MultTranspose(r, w);
+   strumpack[3]->Mult(w, z);
+   m_G.Mult(z, v);
+   y += v;
+
+   smoother.Mult(r, v);
+   y += v;
+}
+
+void HypreIAMS::Smooth(const int n, const mfem::Vector &x,
+                       mfem::Vector &y) const
+{
+   for (int i=0; i<n; ++i)
+   {
+      m_A->Mult(y, r);
+      r -= x;
+
+      smoother.Mult(r, v);
+      y -= v;
+   }
+}
+
+void HypreIAMS::CorrectionCSL(const mfem::Vector &x, mfem::Vector &y) const
+{
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_trueBlockX->GetBlock(0) = r;
+   m_trueBlockX->GetBlock(1) = 0.0;
+
+   m_CSL->Mult(*m_trueBlockX, *m_trueBlockY);
+   IdentityOperator identity(v.Size());
+   identity.Mult(m_trueBlockY->GetBlock(0), v);
+   //v = (mfem::Vector) m_trueBlockY->GetBlock(0);  // Why doesn't this work?
+   y -= v;
+}
+
+void HypreIAMS::CorrectionGradient(const mfem::Vector &x, mfem::Vector &y) const
+{
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_G.MultTranspose(r, w);
+   strumpack[3]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+}
+
+void HypreIAMS::CorrectionPix(const mfem::Vector &x, mfem::Vector &y) const
+{
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_Pix.MultTranspose(r, w);
+   strumpack[0]->Mult(w, z);
+   m_Pix.Mult(z, v);
+   y -= v;
+}
+
+void HypreIAMS::CorrectionPiy(const mfem::Vector &x, mfem::Vector &y) const
+{
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_Piy.MultTranspose(r, w);
+   strumpack[1]->Mult(w, z);
+   m_Piy.Mult(z, v);
+   y -= v;
+}
+
+void HypreIAMS::CorrectionPiz(const mfem::Vector &x, mfem::Vector &y) const
+{
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_Piz.MultTranspose(r, w);
+   strumpack[2]->Mult(w, z);
+   m_Piz.Mult(z, v);
+   y -= v;
+}
+
+void HypreIAMS::MultCSL(const mfem::Vector &x, mfem::Vector &y) const
+{
+   const int id_x = 0;
+   const int id_y = 1;
+   const int id_z = 2;
+   const int id_G = 3;
+
+   y = 0.0;
+
+   Smooth(1, x, y);
+   CorrectionGradient(x, y);
+
+   CorrectionCSL(x, y);
+   CorrectionGradient(x, y);
+
+   CorrectionPix(x, y);
+   CorrectionGradient(x, y);
+
+   CorrectionPiy(x, y);
+   CorrectionGradient(x, y);
+
+   CorrectionPiz(x, y);
+   CorrectionGradient(x, y);
+
+   /*
+   CorrectionPiy(x, y);
+   CorrectionGradient(x, y);
+
+   CorrectionPix(x, y);
+   CorrectionGradient(x, y);
+
+   CorrectionCSL(x, y);
+   CorrectionGradient(x, y);
+
+   Smooth(1, x, y);
+   */
+}
+
+void HypreIAMS::MultMultiplicative(const mfem::Vector &x, mfem::Vector &y) const
+{
+#ifdef HELMHOLTZ_AMS
+#ifdef HELMHOLTZ_AMS_BC
+   const int id_x = 0;
+   const int id_y = 1;
+   const int id_z = 2;
+   const int id_G = 3;
+#else
+   const int id_x = 0;
+   const int id_y = 0;
+   const int id_z = 0;
+   const int id_G = 1;
+#endif
+#else
+   const int id_x = 0;
+   const int id_y = 1;
+   const int id_z = 2;
+   const int id_G = 3;
+#endif
+
+   // Pre-smoothing
+   //smoother.Mult(x, y);
+   y = 0.0;
+   Smooth(1, x, y);
+
+   // Compute residual
+   m_A->Mult(y, r);
+   r -= x;  // r = Ay - x  ==> A^{-1} r = y - A^{-1}x = sol_{iter} - sol_{exact}
+
+   m_G.MultTranspose(r, w);
+   strumpack[id_G]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+
+   //Smooth(2, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+#ifdef CSL_CORRECTION
+   m_trueBlockX->GetBlock(0) = r;
+   m_trueBlockX->GetBlock(1) = 0.0;
+
+   m_CSL->Mult(*m_trueBlockX, *m_trueBlockY);
+   IdentityOperator identity(v.Size());
+   identity.Mult(m_trueBlockY->GetBlock(0), v);
+   //v = (mfem::Vector) m_trueBlockY->GetBlock(0);  // Why doesn't this work?
+   y -= v;
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_G.MultTranspose(r, w);
+   strumpack[id_G]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+
+   //Smooth(2, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+#endif
+
+   m_Pix.MultTranspose(r, w);
+   strumpack[id_x]->Mult(w, z);
+   m_Pix.Mult(z, v);
+   y -= v;
+
+   //Smooth(1, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_G.MultTranspose(r, w);
+   strumpack[id_G]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+
+   //Smooth(2, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_Piy.MultTranspose(r, w);
+   strumpack[id_y]->Mult(w, z);
+   m_Piy.Mult(z, v);
+   y -= v;
+
+   //Smooth(1, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_G.MultTranspose(r, w);
+   strumpack[id_G]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+
+   //Smooth(2, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_Piz.MultTranspose(r, w);
+   strumpack[id_z]->Mult(w, z);
+   m_Piz.Mult(z, v);
+   y -= v;
+
+   //Smooth(1, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_G.MultTranspose(r, w);
+   strumpack[id_G]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+
+   //Smooth(2, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_Piy.MultTranspose(r, w);
+   strumpack[id_y]->Mult(w, z);
+   m_Piy.Mult(z, v);
+   y -= v;
+
+   //Smooth(1, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_G.MultTranspose(r, w);
+   strumpack[id_G]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+
+   //Smooth(2, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_Pix.MultTranspose(r, w);
+   strumpack[id_x]->Mult(w, z);
+   m_Pix.Mult(z, v);
+   y -= v;
+
+   //Smooth(1, x, y);
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_G.MultTranspose(r, w);
+   strumpack[id_G]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+
+#ifdef CSL_CORRECTION
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_trueBlockX->GetBlock(0) = r;
+   m_trueBlockX->GetBlock(1) = 0.0;
+
+   m_CSL->Mult(*m_trueBlockX, *m_trueBlockY);
+   identity.Mult(m_trueBlockY->GetBlock(0), v);
+   //v = m_trueBlockY->GetBlock(0);
+   y -= v;
+
+   m_A->Mult(y, r);
+   r -= x;
+
+   m_G.MultTranspose(r, w);
+   strumpack[id_G]->Mult(w, z);
+   m_G.Mult(z, v);
+   y -= v;
+#endif
+
+   // Post-smoothing
+   Smooth(1, x, y);
+}
+
+void HypreIAMS::Mult(const mfem::Vector &x, mfem::Vector &y) const
+{
+   //MultAdditive(x, y);
+   //MultMultiplicative(x, y);
+   MultCSL(x, y);
+}
+
+HypreIAMS::~HypreIAMS()
+{
+   for (int i=0; i<4; ++i)
+   {
+      delete strumpack[i];
+      delete Arow[i];
+   }
+}
+
+HypreAMSG::HypreAMSG(HypreAMS *ams, int argc, char *argv[])
+   : Solver(ams->Height(), ams->Width()), m_G(ams->Get_G(), false),
+     z(ams->Get_G()->comm, hypre_ParCSRMatrixGlobalNumCols(ams->Get_G()),
+       hypre_ParCSRMatrixColStarts(ams->Get_G())),
+     w(ams->Get_G()->comm, hypre_ParCSRMatrixGlobalNumCols(ams->Get_G()),
+       hypre_ParCSRMatrixColStarts(ams->Get_G())),
+     v(ams->Get_G()->comm, hypre_ParCSRMatrixGlobalNumRows(ams->Get_G()),
+       hypre_ParCSRMatrixRowStarts(ams->Get_G()))
+{
+   HypreParMatrix A_G(ams->Get_A_G());
+
+   Arow = new STRUMPACKRowLocMatrix(A_G);
+
+   strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+   strumpack->SetPrintFactorStatistics(true);
+   strumpack->SetPrintSolveStatistics(false);
+   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+   // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+   // strumpack->SetSymmetricPattern(true);
+   strumpack->SetOperator(*Arow);
+   strumpack->SetFromCommandLine();
+}
+
+void HypreAMSG::SetOperator(const Operator &op)
+{
+
+}
+
+void HypreAMSG::Mult(const mfem::Vector &x, mfem::Vector &y) const
+{
+   m_G.MultTranspose(x, w);
+   strumpack->Mult(w, z);
+   m_G.Mult(z, v);
+
+   y = x;
+   y -= v;
+}
+
+
+void HypreAMSG::SetPrintLevel(int print_lvl)
+{
+
+}
+
+HypreAMSG::~HypreAMSG()
+{
+   delete strumpack;
+   delete Arow;
+}
+
+#endif
+
 }
 
 #endif

linalg/hypre.hpp

@@ -14,6 +14,14 @@
 
 #include "../config/config.hpp"
 
+//#define HYPRE_DYLAN
+
+#ifdef MFEM_USE_STRUMPACK
+#include "strumpack.hpp"
+#endif
+
+#include "blockvector.hpp"
+
 #ifdef MFEM_USE_MPI
 
 #include <mpi.h>
@@ -40,6 +48,7 @@ namespace mfem
 
 class ParFiniteElementSpace;
 class HypreParMatrix;
+class STRUMPACKSolver;
 
 namespace internal
 {
@@ -66,6 +75,30 @@ inline int to_int(HYPRE_Int i)
 
 }
 
+
+/// The MemoryClass used by Hypre objects.
+inline constexpr MemoryClass GetHypreMemoryClass()
+{
+#ifndef HYPRE_USING_CUDA
+   return MemoryClass::HOST;
+#elif defined(HYPRE_USING_UNIFIED_MEMORY)
+   return MemoryClass::MANAGED;
+#else
+   return MemoryClass::DEVICE;
+#endif
+}
+
+inline constexpr MemoryType GetHypreMemoryType()
+{
+#ifndef HYPRE_USING_CUDA
+   return MemoryType::HOST;
+#elif defined(HYPRE_USING_UNIFIED_MEMORY)
+   return MemoryType::MANAGED;
+#else
+   return MemoryType::DEVICE;
+#endif
+}
+
 /// Wrapper for hypre's parallel vector class
 class HypreParVector : public Vector
 {
@@ -80,6 +113,8 @@ private:
    // Set Vector::data and Vector::size from *x
    inline void _SetDataAndSize_();
 
+   Vector hypre_mem_base;
+
 public:
    /** @brief Creates vector with given global size and parallel partitioning of
        the rows/columns given by @a col. */
@@ -150,6 +185,73 @@ public:
        HypreParVector(MPI_Comm, HYPRE_Int, double *, HYPRE_Int *). */
    void SetData(double *_data);
 
+   /// TODO: documentation
+   inline const HypreParVector &Read(const Vector &base)
+   {
+      // TODO: we may need to copy the data if the MemoryTypes of base are not
+      // suitable for GetHypreMemoryClass()
+      if (GetHypreMemoryClass() != MemoryClass::HOST &&
+          base.GetMemory().GetMemoryType() == MemoryType::HOST)
+      {
+         if (hypre_mem_base.Size() == 0)
+         {
+            hypre_mem_base.SetSize(base.Size(), GetHypreMemoryType());
+         }
+         else
+         {
+            MFEM_VERIFY(hypre_mem_base.Size() == base.Size(), "");
+         }
+         hypre_mem_base.GetMemory().CopyFromHost(base.HostRead(), base.Size());
+         hypre_VectorData(hypre_ParVectorLocalVector(x)) = hypre_mem_base.GetMemory();
+      }
+      else
+      {
+         MakeRef(const_cast<Vector&>(base), 0);
+         UseDevice(true);
+         hypre_VectorData(hypre_ParVectorLocalVector(x)) =
+            const_cast<double*>(data.Read(GetHypreMemoryClass(), size));
+      }
+      return *this;
+   }
+
+   /// TODO: documentation
+   inline HypreParVector &Write(Vector &base)
+   {
+      // TODO: we may need to allocate memory if the MemoryTypes of base are not
+      // suitable for GetHypreMemoryClass(). Then the data will need to be
+      // copied back to base with a separate call to a new method.
+      if (GetHypreMemoryClass() == MemoryClass::HOST)
+      {
+         MakeRef(base, 0);
+         UseDevice(true);
+         hypre_VectorData(hypre_ParVectorLocalVector(x)) =
+            data.Write(GetHypreMemoryClass(), size);
+      }
+      else
+      {
+         if (hypre_mem_base.Size() == 0)
+         {
+            hypre_mem_base.SetSize(base.Size(), GetHypreMemoryType());
+         }
+         else
+         {
+            MFEM_VERIFY(hypre_mem_base.Size() == base.Size(), "");
+         }
+         MakeRef(const_cast<Vector&>(hypre_mem_base), 0);
+         hypre_VectorData(hypre_ParVectorLocalVector(x)) = data;
+      }
+      return *this;
+   }
+
+   inline void WriteCopy(Vector &base)
+   {
+      if (GetHypreMemoryClass() != MemoryClass::HOST)
+      {
+         MFEM_VERIFY(hypre_mem_base.Size() == base.Size(), "");
+         base.GetMemory().CopyFrom(hypre_mem_base.GetMemory(), base.Size());
+      }
+   }
+
    /// Set random values
    HYPRE_Int Randomize(HYPRE_Int seed);
 
@@ -223,6 +325,8 @@ private:
    // the indices from HYPRE_Int to int.
    static void CopyCSR_J(hypre_CSRMatrix *hypre_csr, int *J);
 
+   Memory<HYPRE_Int> hypre_mem_row, hypre_mem_col, hypre_mem_cmap;
+
 public:
    /// An empty matrix to be used as a reference to an existing matrix
    HypreParMatrix();
@@ -546,6 +650,8 @@ public:
    virtual ~HypreParMatrix() { Destroy(); }
 
    Type GetType() const { return Hypre_ParCSR; }
+
+   virtual MemoryClass GetMemoryClass() const { return GetHypreMemoryClass(); }
 };
 
 /** @brief Return a new matrix `C = alpha*A + beta*B`, assuming that both `A`
@@ -648,7 +754,7 @@ public:
        16   = Chebyshev
        1001 = Taubin polynomial smoother
        1002 = FIR polynomial smoother. */
-   enum Type { Jacobi = 0, l1Jacobi = 1, l1GS = 2, l1GStr = 4, lumpedJacobi = 5,
+   enum Type { Jacobi = 0, l1Jacobi = 1, l1GS = 2, Kaczmarz = 3, l1GStr = 4, lumpedJacobi = 5,
                GS = 6, Chebyshev = 16, Taubin = 1001, FIR = 1002
              };
 
@@ -740,6 +846,53 @@ public:
    virtual void Mult(const HypreParVector &b, HypreParVector &x) const;
    virtual void Mult(const Vector &b, Vector &x) const;
 
+#ifdef HYPRE_DYLAN
+   virtual HYPRE_ParCSRMatrix Get_Pix() const
+   {
+      return 0;
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_Piy() const
+   {
+      return 0;
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_Piz() const
+   {
+      return 0;
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_G() const
+   {
+      return 0;
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_A_Pix() const
+   {
+      return 0;
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_A_Piy() const
+   {
+      return 0;
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_A_Piz() const
+   {
+      return 0;
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_A_G() const
+   {
+      return 0;
+   }
+
+   virtual HYPRE_ParCSRMatrix* Get_Restriction() const
+   {
+      return 0;
+   }
+#endif
+
    /** @brief Set the behavior for treating hypre errors, see the ErrorMode
        enum. The default mode in the base class is ABORT_HYPRE_ERRORS. */
    /** Currently, there are three cases in derived classes where the error flag
@@ -1014,6 +1167,17 @@ public:
    virtual HYPRE_PtrToParSolverFcn SolveFcn() const
    { return (HYPRE_PtrToParSolverFcn) HYPRE_BoomerAMGSolve; }
 
+   /*
+   virtual HYPRE_ParCSRMatrix* Get_Restriction() const
+   {
+     //int m = hypre_ParAMGData_GetMaxLevels(amg_precond);
+     //int n = hypre_ParAMGData_GetNumLevels(amg_precond);
+     //return hypre_AMGGetRestriction(amg_precond);
+     //hypre_ParCSRMatrix** hypre_AMGGetRestriction(void *solver);
+     //return (HYPRE_ParCSRMatrix) hypre_ParAMGDataRArray(amg_precond);  // See hypre/src/parcsr_ls/par_amg.h
+   }
+   */
+
    virtual ~HypreBoomerAMG();
 };
 
@@ -1060,6 +1224,48 @@ public:
    virtual HYPRE_PtrToParSolverFcn SolveFcn() const
    { return (HYPRE_PtrToParSolverFcn) HYPRE_AMSSolve; }
 
+#ifdef HYPRE_DYLAN
+   virtual HYPRE_ParCSRMatrix Get_Pix() const
+   {
+      return hypre_AMSGet_Pix(ams);
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_Piy() const
+   {
+      return hypre_AMSGet_Piy(ams);
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_Piz() const
+   {
+      return hypre_AMSGet_Piz(ams);
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_G() const
+   {
+      return hypre_AMSGet_G(ams);
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_A_Pix() const
+   {
+      return hypre_AMSGetA_Pix(ams);
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_A_Piy() const
+   {
+      return hypre_AMSGetA_Piy(ams);
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_A_Piz() const
+   {
+      return hypre_AMSGetA_Piz(ams);
+   }
+
+   virtual HYPRE_ParCSRMatrix Get_A_G() const
+   {
+      return hypre_AMSGetA_G(ams);
+   }
+#endif
+
    virtual ~HypreAMS();
 };
 
@@ -1317,6 +1523,81 @@ public:
    HypreParVector ** StealEigenvectors();
 };
 
+#ifdef HYPRE_DYLAN
+/// An experimental auxiliary-space indefinite Maxwell Solver, using hypre
+class HypreIAMS : public Solver
+{
+private:
+   Operator* Arow[4];
+   STRUMPACKSolver* strumpack[4];
+   HypreAMS *m_ams;
+   HYPRE_Solver empty_ams;
+   HypreParMatrix m_Pix, m_Piy, m_Piz, m_G;
+   HypreParMatrix *m_A;
+   HypreSmoother smoother;
+   mutable HypreParVector z, w, r, v;
+   STRUMPACKSolver *m_CSL;
+   BlockVector *m_trueBlockX, *m_trueBlockY;
+
+public:
+   HypreIAMS(HypreParMatrix &A, HypreParMatrix *H, STRUMPACKSolver *CSL,
+             BlockVector *trueBlockX, BlockVector *trueBlockY, HypreAMS *ams,
+             int argc, char *argv[]);
+
+   void SetPrintLevel(int print_lvl);
+
+   virtual void Mult(const Vector &x, Vector &y) const;
+   //virtual void Mult(const HypreParVector &b, HypreParVector &x) const;
+   virtual void SetOperator(const Operator &op);
+
+   void MultAdditive(const Vector &x, Vector &y) const;
+   void MultMultiplicative(const Vector &x, Vector &y) const;
+   void MultCSL(const mfem::Vector &x, mfem::Vector &y) const;
+
+   void Smooth(const int n, const mfem::Vector &x, mfem::Vector &y) const;
+
+   void CorrectionGradient(const mfem::Vector &x, mfem::Vector &y) const;
+   void CorrectionCSL(const mfem::Vector &x, mfem::Vector &y) const;
+   void CorrectionPix(const mfem::Vector &x, mfem::Vector &y) const;
+   void CorrectionPiy(const mfem::Vector &x, mfem::Vector &y) const;
+   void CorrectionPiz(const mfem::Vector &x, mfem::Vector &y) const;
+
+   /*
+   virtual operator HYPRE_Solver() const { return empty_ams; }
+   //virtual operator HYPRE_Solver() const { return m_ams->HYPRE_Solver(); }
+
+   virtual HYPRE_PtrToParSolverFcn SetupFcn() const
+   { return (HYPRE_PtrToParSolverFcn) HYPRE_AMSSetup; }
+   virtual HYPRE_PtrToParSolverFcn SolveFcn() const
+   { return (HYPRE_PtrToParSolverFcn) HYPRE_AMSSolve; }
+   */
+
+   virtual ~HypreIAMS();
+};
+
+class HypreAMSG : public Solver
+{
+private:
+   Operator* Arow;
+   STRUMPACKSolver* strumpack;
+   HypreParMatrix m_G;
+   HypreParMatrix *m_A;
+   HypreSmoother smoother;
+   mutable HypreParVector z, w, v;
+
+public:
+   HypreAMSG(HypreAMS *ams, int argc, char *argv[]);
+
+   void SetPrintLevel(int print_lvl);
+
+   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void SetOperator(const Operator &op);
+
+   virtual ~HypreAMSG();
+};
+#endif
+
+
 }
 
 #endif // MFEM_USE_MPI

linalg/hypre_parcsr.cpp

@@ -665,7 +665,7 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
       if (col_mark[i]) { Ae_offd_ncols++; }
    }
 
-   Ae_col_map_offd  = mfem_hypre_CTAlloc(HYPRE_Int, Ae_offd_ncols);
+   Ae_col_map_offd  = mfem_hypre_CTAlloc_host(HYPRE_Int, Ae_offd_ncols);
 
    Ae_offd_ncols = 0;
    for (i = 0; i < A_offd_ncols; i++)
@@ -948,8 +948,8 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
          hypre_CSRMatrix *block_offd = hypre_ParCSRMatrixOffd(block);
          HYPRE_Int block_offd_cols = hypre_CSRMatrixNumCols(block_offd);
 
-         HYPRE_Int *block_col_map = mfem_hypre_TAlloc(HYPRE_Int,
-                                                      block_offd_cols);
+         HYPRE_Int *block_col_map = mfem_hypre_TAlloc_host(HYPRE_Int,
+                                                           block_offd_cols);
          for (i = j = 0; i < offd_cols; i++)
          {
             HYPRE_Int bn = offd_col_block_num[i];
@@ -1359,7 +1359,7 @@ hypre_ParCSRCommHandleCreate_bool(HYPRE_Int            job,
    HYPRE_Int                  ip, vec_start, vec_len;
 
    num_requests = num_sends + num_recvs;
-   requests = mfem_hypre_CTAlloc(hypre_MPI_Request, num_requests);
+   requests = mfem_hypre_CTAlloc_host(hypre_MPI_Request, num_requests);
 
    hypre_MPI_Comm_size(comm, &num_procs);
    hypre_MPI_Comm_rank(comm, &my_id);
@@ -1416,7 +1416,7 @@ hypre_ParCSRCommHandleCreate_bool(HYPRE_Int            job,
     * set up comm_handle and return
     *--------------------------------------------------------------------*/
 
-   comm_handle = mfem_hypre_CTAlloc(hypre_ParCSRCommHandle, 1);
+   comm_handle = mfem_hypre_CTAlloc_host(hypre_ParCSRCommHandle, 1);
 
    hypre_ParCSRCommHandleCommPkg(comm_handle)     = comm_pkg;
    hypre_ParCSRCommHandleSendData(comm_handle)    = send_data;
@@ -1810,7 +1810,7 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A,
          return NULL; /* error: A_offd and B_offd have different dimensions */
       }
       /* copy A_cmap -> C_cmap */
-      C_cmap = mfem_hypre_TAlloc(HYPRE_Int, A_cmap_size);
+      C_cmap = mfem_hypre_TAlloc_host(HYPRE_Int, A_cmap_size);
       for (im = 0; im < A_cmap_size; im++)
       {
          C_cmap[im] = A_cmap[im];

linalg/hypre_parcsr.hpp

@@ -31,13 +31,23 @@
 #define mfem_hypre_CTAlloc(type, size) hypre_CTAlloc(type, size)
 #define mfem_hypre_TFree(ptr) hypre_TFree(ptr)
 
+#define mfem_hypre_TAlloc_host(type, size) hypre_TAlloc(type, size)
+#define mfem_hypre_CTAlloc_host(type, size) hypre_CTAlloc(type, size)
+#define mfem_hypre_TFree_host(ptr) hypre_TFree(ptr)
+
 #else // MFEM_HYPRE_VERSION >= 21400
 
 #define mfem_hypre_TAlloc(type, size) \
-   hypre_TAlloc(type, size, HYPRE_MEMORY_HOST)
+   hypre_TAlloc(type, size, HYPRE_MEMORY_DEVICE)
 #define mfem_hypre_CTAlloc(type, size) \
+   hypre_CTAlloc(type, size, HYPRE_MEMORY_DEVICE)
+#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, HYPRE_MEMORY_DEVICE)
+
+#define mfem_hypre_TAlloc_host(type, size) \
+   hypre_TAlloc(type, size, HYPRE_MEMORY_HOST)
+#define mfem_hypre_CTAlloc_host(type, size) \
    hypre_CTAlloc(type, size, HYPRE_MEMORY_HOST)
-#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, HYPRE_MEMORY_HOST)
+#define mfem_hypre_TFree_host(ptr) hypre_TFree(ptr, HYPRE_MEMORY_HOST)
 
 // Notes regarding allocation and deallocation of hypre objects in 2.14.0
 //-----------------------------------------------------------------------

linalg/linalg.hpp

@@ -28,6 +28,7 @@
 #include "solvers.hpp"
 #include "handle.hpp"
 #include "invariants.hpp"
+#include "auxiliary.hpp"
 
 #ifdef MFEM_USE_AMGX
 #include "amgxsolver.hpp"

linalg/operator.cpp

@@ -307,6 +307,8 @@ ProductOperator::ProductOperator(const Operator *A, const Operator *B,
                "incompatible Operators: A->Width() = " << A->Width()
                << ", B->Height() = " << B->Height());
 
+   z.UseDevice(true);
+
    {
       const Solver* SolverB = dynamic_cast<const Solver*>(B);
       if (SolverB)
@@ -323,6 +325,26 @@ ProductOperator::~ProductOperator()
    if (ownB) { delete B; }
 }
 
+SumOperator::SumOperator(const Operator *A_, const Operator *B_,
+                         bool ownA_, bool ownB_, double cA_, double cB_)
+   : Operator(A_->Height(), B_->Width()),
+     A(A_), B(B_), ownA(ownA_), ownB(ownB_), z(A_->Height()), w(A_->Width()),
+     cA(cA_), cB(cB_)
+{
+   MFEM_VERIFY(A->Width() == B->Width() && A->Height() == B->Height(),
+               "incompatible Operators: A->Width() = " << A->Width()
+               << ", B->Height() = " << B->Height());
+
+   z.UseDevice(true);
+   w.UseDevice(true);
+}
+
+SumOperator::~SumOperator()
+{
+   if (ownA) { delete A; }
+   if (ownB) { delete B; }
+}
+
 
 RAPOperator::RAPOperator(const Operator &Rt_, const Operator &A_,
                          const Operator &P_)
@@ -392,6 +414,9 @@ TripleProductOperator::TripleProductOperator(
    MemoryType mem_type = GetMemoryType(mem_class*B->GetMemoryClass());
    t1.SetSize(C->Height(), mem_type);
    t2.SetSize(B->Height(), mem_type);
+
+   t1.UseDevice(true);
+   t2.UseDevice(true);
 }
 
 TripleProductOperator::~TripleProductOperator()

linalg/operator.hpp

@@ -37,7 +37,8 @@ protected:
       const Array<int> &test_tdof_list,
       RectangularConstrainedOperator* &Aout);
 
-   /// Returns RAP Operator of this, taking in input/output Prolongation matrices
+   /** @brief Returns RAP Operator of this, using input/output Prolongation matrices
+       @a Pi corresponds to "P", @a Po corresponds to "Rt" */
    Operator *SetupRAP(const Operator *Pi, const Operator *Po);
 
 public:
@@ -112,6 +113,10 @@ public:
    {
       return GetProlongation(); // Assume square unless specialized
    }
+   /** @brief Prolongation operator from linear algebra (linear system) vectors,
+       to output vectors for the operator, including only processor-local portion.
+       `NULL` means identity. */
+   virtual const Operator *GetLocalOutputProlongation() const { return NULL; }
    /** @brief Restriction operator from output vectors for the operator to linear
        algebra (linear system) vectors. `NULL` means identity. */
    virtual const Operator *GetOutputRestriction() const
@@ -737,6 +742,26 @@ public:
    virtual ~ProductOperator();
 };
 
+/// General product operator: x -> A(x)+B(x)
+class SumOperator : public Operator
+{
+   const Operator *A, *B;
+   bool ownA, ownB;
+   mutable Vector z, w;
+   double cA, cB;
+
+public:
+   SumOperator(const Operator *A_, const Operator *B_, bool ownA_, bool ownB_,
+               double cA_, double cB_);
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   { B->Mult(x, z); A->Mult(x, y); y *= cA; z *= cB; y += z;}
+
+   virtual void MultTranspose(const Vector &x, Vector &y) const
+   { B->MultTranspose(x, w); A->MultTranspose(x, y); y *= cA; w *= cB; y += w;}
+
+   virtual ~SumOperator();
+};
 
 /// The operator x -> R*A*P*x constructed through the actions of R^T, A and P
 class RAPOperator : public Operator

linalg/solvers.cpp

@@ -527,9 +527,17 @@ void SLI(const Operator &A, Solver &B, const Vector &b, Vector &x,
 
 void CGSolver::UpdateVectors()
 {
-   r.SetSize(width);
-   d.SetSize(width);
-   z.SetSize(width);
+   MemoryType mt = GetMemoryType(oper->GetMemoryClass());
+
+   r.SetSize(width, mt);
+   d.SetSize(width, mt);
+   z.SetSize(width, mt);
+
+   /*
+   r.UseDevice(true);
+   d.UseDevice(true);
+   z.UseDevice(true);
+   */
 }
 
 void CGSolver::Mult(const Vector &b, Vector &x) const
@@ -537,6 +545,9 @@ void CGSolver::Mult(const Vector &b, Vector &x) const
    int i;
    double r0, den, nom, nom0, betanom, alpha, beta;
 
+   int myid = -1;
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
    if (iterative_mode)
    {
       oper->Mult(x, r);
@@ -584,6 +595,7 @@ void CGSolver::Mult(const Vector &b, Vector &x) const
       converged = 1;
       final_iter = 0;
       final_norm = sqrt(nom);
+      //cout << myid << ": PCG iter " << final_iter << '\n' << std::flush;
       return;
    }
 
@@ -602,6 +614,7 @@ void CGSolver::Mult(const Vector &b, Vector &x) const
          converged = 0;
          final_iter = 0;
          final_norm = sqrt(nom);
+         //cout << myid << ": PCG iter " << final_iter << '\n' << std::flush;
          return;
       }
    }
@@ -712,6 +725,7 @@ void CGSolver::Mult(const Vector &b, Vector &x) const
       mfem::out << "Average reduction factor = "
                 << pow (betanom/nom0, 0.5/final_iter) << '\n';
    }
+   //cout << myid << ": PCG iter " << final_iter << '\n' << std::flush;
    final_norm = sqrt(betanom);
 
    Monitor(final_iter, final_norm, r, x, true);
@@ -807,7 +821,10 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
    Vector r(n), w(n);
    Array<Vector *> v;
 
-   double resid;
+   r.UseDevice(true);
+   w.UseDevice(true);
+
+   double resid, l2resid;
    int i, j, k;
 
    if (iterative_mode)
@@ -824,10 +841,12 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
       if (iterative_mode)
       {
          subtract(b, r, w);
+         l2resid = Norm(w);
          prec->Mult(w, r);    // r = M (b - A x)
       }
       else
       {
+         l2resid = Norm(b);
          prec->Mult(b, r);
       }
    }
@@ -841,8 +860,11 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
       {
          r = b;
       }
+      l2resid = Norm(r);
    }
+
    double beta = Norm(r);  // beta = ||r||
+   const bool betaFinite = IsFinite(beta);
    MFEM_ASSERT(IsFinite(beta), "beta = " << beta);
 
    final_norm = std::max(rel_tol*beta, abs_tol);
@@ -860,6 +882,7 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
       mfem::out << "   Pass : " << setw(2) << 1
                 << "   Iteration : " << setw(3) << 0
                 << "  ||B r|| = " << beta << (print_level == 3 ? " ...\n" : "\n");
+      mfem::out << "   ||r|| = " << l2resid << '\n';
    }
 
    Monitor(0, beta, r, x);
@@ -868,7 +891,11 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
 
    for (j = 1; j <= max_iter; )
    {
-      if (v[0] == NULL) { v[0] = new Vector(n); }
+      if (v[0] == NULL)
+      {
+         v[0] = new Vector(n);
+         v[0]->UseDevice(true);
+      }
       v[0]->Set(1.0/beta, r);
       s = 0.0; s(0) = beta;
 
@@ -892,7 +919,11 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
 
          H(i+1,i) = Norm(w);           // H(i+1,i) = ||w||
          MFEM_ASSERT(IsFinite(H(i+1,i)), "Norm(w) = " << H(i+1,i));
-         if (v[i+1] == NULL) { v[i+1] = new Vector(n); }
+         if (v[i+1] == NULL)
+         {
+            v[i+1] = new Vector(n);
+            v[i+1]->UseDevice(true);
+         }
          v[i+1]->Set(1.0/H(i+1,i), w); // v[i+1] = w / H(i+1,i)
 
          for (k = 0; k < i; k++)
@@ -921,6 +952,9 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
             mfem::out << "   Pass : " << setw(2) << (j-1)/m+1
                       << "   Iteration : " << setw(3) << j
                       << "  ||B r|| = " << resid << '\n';
+
+            cout << "Pass " << (j-1)/m+1 << ", iteration " << j << ", ||B r|| = " << resid
+                 << endl;
          }
 
          Monitor(j, resid, r, x);
@@ -937,12 +971,17 @@ void GMRESSolver::Mult(const Vector &b, Vector &x) const
       if (prec)
       {
          subtract(b, r, w);
+         l2resid = Norm(w);
          prec->Mult(w, r);    // r = M (b - A x)
       }
       else
       {
          subtract(b, r, r);
+         l2resid = Norm(r);
       }
+
+      if (print_level == 1) { mfem::out << "   ||r|| = " << l2resid << '\n'; }
+
       beta = Norm(r);         // beta = ||r||
       MFEM_ASSERT(IsFinite(beta), "beta = " << beta);
       if (beta <= final_norm)
@@ -971,7 +1010,7 @@ finish:
    }
    if (print_level >= 0 && !converged)
    {
-      mfem::out << "GMRES: No convergence!\n";
+      mfem::out << "GMRES: " << name << ": No convergence!\n";
    }
 
    Monitor(final_iter, final_norm, r, x, true);
@@ -2171,6 +2210,530 @@ slbqp_done:
    }
 }
 
+#ifdef MFEM_USE_MPI
+
+GMGSolver::GMGSolver(HypreParMatrix * Af_,
+                     std::vector<HypreParMatrix *> P_, CoarseSolver cs)
+   : Solver(Af_->Height(), Af_->Width()), Af(Af_), P(P_)
+{
+
+   NumGrids = P.size();
+   S.resize(NumGrids);
+   A.resize(NumGrids + 1);
+
+   A[NumGrids] = Af;
+
+   for (int i = NumGrids ; i > 0; i--)
+   {
+      A[i - 1] = RAP(A[i], P[i - 1]);
+   }
+   // Set up coarse solve operator
+   switch (cs)
+   {
+      case PETSC:
+#ifndef MFEM_USE_PETSC
+         MFEM_ABORT("Invalid choice of CoarseSolver. MFEM is not linked with PETSC");
+#else
+         petsc = new PetscLinearSolver(MPI_COMM_WORLD, "direct");
+         // Convert to PetscParMatrix
+         petsc->SetOperator(PetscParMatrix(A[0], Operator::PETSC_MATAIJ));
+         invAc = petsc;
+#endif
+         break;
+      case SUPERLU:
+#ifndef MFEM_USE_SUPERLU
+         MFEM_ABORT("Invalid choice of CoarseSolver. MFEM is not linked with SUPERLU");
+#else
+         SluA = new SuperLURowLocMatrix(*A[0]);
+         superlu = new SuperLUSolver(*SluA);
+         superlu->SetPrintStatistics(false);
+         superlu->SetSymmetricPattern(true);
+         superlu->SetColumnPermutation(superlu::PARMETIS);
+         // superlu->SetColumnPermutation(superlu::NATURAL); // Sometimes parmetis crashes for multiple processos.
+         superlu->SetOperator(*SluA);
+         invAc = superlu;
+#endif
+         break;
+      case STRUMPACK:
+#ifndef MFEM_USE_STRUMPACK
+         MFEM_ABORT("Invalid choice of CoarseSolver. MFEM is not linked with STRUMPACK");
+#else
+         StpA = new STRUMPACKRowLocMatrix(*A[0]);
+         strumpack = new STRUMPACKSolver(*StpA);
+         strumpack->SetPrintFactorStatistics(false);
+         strumpack->SetPrintSolveStatistics(false);
+         strumpack->SetOperator(*StpA);
+         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+         strumpack->DisableMatching();
+         invAc = strumpack;
+#endif
+         break;
+   }
+   // Check if direct solver is found
+   if (!invAc) { MFEM_ABORT("Direct Solver of coarse solve not found"); }
+   // construct smoothers
+   for (int i = NumGrids - 1; i >= 0 ; i--)
+   {
+      S[i] = new HypreSmoother;
+      S[i]->SetType(HypreSmoother::Jacobi);
+      S[i]->SetOperator(*A[i+1]);
+   }
+}
+
+void GMGSolver::SetSmootherType(const HypreSmoother::Type type) const
+{
+   for (int i = NumGrids - 1; i >= 0 ; i--)
+   {
+      S[i]->SetType(type);
+      S[i]->SetOperator(*A[i+1]);
+   }
+}
+
+void GMGSolver::Mult(const Vector &r, Vector &z) const
+{
+   // Residual vectors
+   std::vector<Vector> rv(NumGrids + 1);
+   // correction vectors
+   std::vector<Vector> zv(NumGrids + 1);
+   // allocation
+   for (int i = 0; i <= NumGrids ; i++)
+   {
+      int n = A[i]->Width();
+      rv[i].SetSize(n);
+      zv[i].SetSize(n);
+   }
+   // Initial residual
+   rv[NumGrids] = r;
+   // smooth and update residuals down to the coarsest level
+   for (int i = NumGrids; i > 0 ; i--)
+   {
+      // Pre smooth
+      S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+      // compute residual
+      Vector w(A[i]->Height());
+      A[i]->Mult(zv[i], w);
+      rv[i] -= w;
+      // Restrict
+      P[i - 1]->MultTranspose(rv[i], rv[i - 1]);
+   }
+
+   // Coarse grid Solve
+   invAc->Mult(rv[0], zv[0]);
+   //
+   for (int i = 1; i <= NumGrids ; i++)
+   {
+      // Prolong correction
+      Vector u(P[i - 1]->Height());
+      P[i - 1]->Mult(zv[i - 1], u);
+      // Update correction
+      zv[i] += u;
+      // Update residual
+      Vector v(A[i]->Height());
+      A[i]->Mult(u, v); rv[i] -= v;
+      // Post smooth
+      S[i - 1]->Mult(rv[i], v); v *= theta;
+      // Update correction
+      zv[i] += v;
+   }
+   z = zv[NumGrids];
+}
+
+GMGSolver::~GMGSolver()
+{
+   int n = S.size();
+   for (int i = n - 1; i >= 0 ; i--)
+   {
+      delete S[i];
+      delete A[i];
+   }
+   S.clear();
+   A.clear();
+#ifdef MFEM_USE_PETSC
+   // delete petsc;
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   delete StpA;
+   delete strumpack;
+#endif
+#ifdef MFEM_USE_SUPERLU
+   // delete SluA;
+   // delete superlu;
+#endif
+   //delete invAc;
+}
+
+ComplexGMGSolver::ComplexGMGSolver(ComplexHypreParMatrix * Af_,
+                                   std::vector<HypreParMatrix *> P_, CoarseSolver cs, bool printCoarse)
+   : Solver(Af_->Height(), Af_->Width()), Af(Af_), P(P_)
+{
+
+   NumGrids = P.size();
+   S.resize(NumGrids);
+   A.resize(NumGrids + 1);
+   block_OffsetsI.SetSize(3);
+   block_OffsetsJ.SetSize(3);
+   block_OffsetsI[0]=0;
+   block_OffsetsJ[0]=0;
+
+   A[NumGrids] = Af;
+
+   for (int i = NumGrids ; i > 0; i--)
+   {
+      A[i - 1] = new ComplexHypreParMatrix(RAP(&A[i]->real(), P[i - 1]),
+                                           A[i]->hasImagPart() ? RAP(&A[i]->imag(), P[i - 1]) : NULL,
+                                           false, false, ComplexOperator::HERMITIAN);
+   }
+   // Set up coarse solve operator
+   switch (cs)
+   {
+      case PETSC:
+#ifndef MFEM_USE_PETSC
+         MFEM_ABORT("Invalid choice of CoarseSolver. MFEM is not linked with PETSC");
+#else
+         petsc = new PetscLinearSolver(MPI_COMM_WORLD, "direct");
+         // Convert to PetscParMatrix
+         petsc->SetOperator(PetscParMatrix(A[0]->GetSystemMatrix(),
+                                           Operator::PETSC_MATAIJ));
+         invAc = petsc;
+#endif
+         break;
+      case SUPERLU:
+#ifndef MFEM_USE_SUPERLU
+         MFEM_ABORT("Invalid choice of CoarseSolver. MFEM is not linked with SUPERLU");
+#else
+         SluA = new SuperLURowLocMatrix(*A[0]->GetSystemMatrix());
+         superlu = new SuperLUSolver(*SluA);
+         superlu->SetPrintStatistics(false);
+         superlu->SetSymmetricPattern(true);
+         superlu->SetColumnPermutation(superlu::PARMETIS);
+         // superlu->SetColumnPermutation(superlu::NATURAL);
+         superlu->SetOperator(*SluA);
+         invAc = superlu;
+#endif
+         break;
+      case STRUMPACK:
+#ifndef MFEM_USE_STRUMPACK
+         MFEM_ABORT("Invalid choice of CoarseSolver. MFEM is not linked with STRUMPACK");
+#else
+         {
+            HypreParMatrix *Ahyp0 = A[0]->GetSystemMatrix();
+            if (printCoarse)
+            {
+               Ahyp0->Print("A0FA.txt");
+            }
+
+            StpA = new STRUMPACKRowLocMatrix(*Ahyp0);
+            // StpA = new STRUMPACKRowLocMatrix(*A[0]->GetSystemMatrix());
+            strumpack = new STRUMPACKSolver(*StpA);
+            strumpack->SetPrintFactorStatistics(false);
+            strumpack->SetPrintSolveStatistics(false);
+            strumpack->SetOperator(*StpA);
+            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+            strumpack->DisableMatching();
+            invAc = strumpack;
+         }
+#endif
+         break;
+      case UMFPACK:
+      {
+#ifdef MFEM_USE_SUITESPARSE
+         HypreParMatrix *Ahyp0 = A[0]->GetSystemMatrix();
+         {
+            int nprocs = 0;
+            MPI_Comm_size(Ahyp0->GetComm(), &nprocs);
+            MFEM_VERIFY(nprocs == 1, "");
+         }
+         SparseMatrix *Asp = new SparseMatrix();
+         Ahyp0->GetDiag(*Asp);  // Asp does not own the data
+         UMFPackSolver *umf_solver = new UMFPackSolver();
+         umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+         umf_solver->SetOperator(*Asp);
+         invAc = umf_solver;
+#endif
+      }
+      break;
+   }
+   // Check if direct solver is found
+   if (!invAc) { MFEM_ABORT("Direct Solver of coarse solve not found"); }
+
+   // construct smoothers
+   for (int i = NumGrids - 1; i >= 0 ; i--)
+   {
+      S[i] = new HypreSmoother;
+      S[i]->SetType(HypreSmoother::Jacobi);
+      S[i]->SetOperator(*A[i+1]->GetSystemMatrix());
+   }
+}
+
+void ComplexGMGSolver::SetSmootherType(const HypreSmoother::Type type) const
+{
+   for (int i = NumGrids - 1; i >= 0 ; i--)
+   {
+      S[i]->SetType(type);
+      S[i]->SetOperator(*A[i+1]->GetSystemMatrix());
+   }
+}
+void ComplexGMGSolver::Mult(const Vector &r, Vector &z) const
+{
+   // Residual vectors
+   std::vector<Vector> rv(NumGrids + 1);
+   // correction vectors
+   std::vector<Vector> zv(NumGrids + 1);
+   // allocation
+
+   // TODO: make the block operators members of the class, so they do not reallocate their data (e.g. tmp) on every Mult() call.
+
+   for (int i = 0; i <= NumGrids ; i++)
+   {
+      int n = A[i]->Width();
+      rv[i].SetSize(n);
+      zv[i].SetSize(n);
+   }
+   // Initial residual
+   rv[NumGrids] = r;
+   // smooth and update residuals down to the coarsest level
+   for (int i = NumGrids; i > 0 ; i--)
+   {
+      // Pre smooth
+      S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+      // compute residual
+      Vector w(A[i]->Height());
+      A[i]->Mult(zv[i], w);
+      rv[i] -= w;
+      // Restrict
+      block_OffsetsI[1] = P[i - 1]->GetNumRows();
+      block_OffsetsI[2] = P[i - 1]->GetNumRows();
+      block_OffsetsI.PartialSum();
+      block_OffsetsJ[1] = P[i - 1]->GetNumCols();
+      block_OffsetsJ[2] = P[i - 1]->GetNumCols();
+      block_OffsetsJ.PartialSum();
+      BlockOperator BlkP(block_OffsetsI,block_OffsetsJ);
+      BlkP.SetBlock(0,0,P[i-1]);
+      BlkP.SetBlock(1,1,P[i-1]);
+      BlkP.MultTranspose(rv[i], rv[i - 1]);
+   }
+   // Coarse grid Stiffness matrix
+   invAc->Mult(rv[0], zv[0]);
+   //
+   for (int i = 1; i <= NumGrids ; i++)
+   {
+      // Prolong correction
+      block_OffsetsI[1] = P[i - 1]->GetNumRows();
+      block_OffsetsI[2] = P[i - 1]->GetNumRows();
+      block_OffsetsI.PartialSum();
+      block_OffsetsJ[1] = P[i - 1]->GetNumCols();
+      block_OffsetsJ[2] = P[i - 1]->GetNumCols();
+      block_OffsetsJ.PartialSum();
+      BlockOperator BlkP(block_OffsetsI,block_OffsetsJ);
+      BlkP.SetBlock(0,0,P[i-1]);
+      BlkP.SetBlock(1,1,P[i-1]);
+      Vector u(BlkP.Height());
+      BlkP.Mult(zv[i - 1], u);
+      // Update correction
+      zv[i] += u;
+      // Update residual
+      Vector v(A[i]->Height());
+      A[i]->Mult(u, v); rv[i] -= v;
+      // Post smooth
+      S[i - 1]->Mult(rv[i], v); v *= theta;
+      // Update correction
+      zv[i] += v;
+   }
+   z = zv[NumGrids];
+}
+ComplexGMGSolver::~ComplexGMGSolver()
+{
+   int n = S.size();
+   for (int i = n - 1; i >= 0 ; i--)
+   {
+      delete S[i];
+   }
+#ifdef MFEM_USE_PETSC
+   // delete petsc;
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   delete StpA;
+   delete strumpack;
+#endif
+#ifdef MFEM_USE_SUPERLU
+   // delete SluA;
+   // delete superlu;
+#endif
+   // delete invAc;
+}
+
+ComplexGMGPASolver::ComplexGMGPASolver(MPI_Comm comm, Operator * Af_Re,
+                                       Operator * Af_Im,
+                                       Vector& diagRe_,
+                                       Array<int>& ess_tdof_list,
+                                       std::vector<HypreParMatrix *> P_,
+                                       HypreParMatrix * Ac_Re, HypreParMatrix * Ac_Im, bool printCoarse)
+   : Solver(2*Af_Re->Height(), 2*Af_Re->Width()), P(P_), diagRe(diagRe_),
+     Jacobi(diagRe, ess_tdof_list, 1.0)
+{
+   NumGrids = P.size();
+   Pt.resize(NumGrids);
+   S.resize(NumGrids);
+   AO.resize(NumGrids + 1);
+   AO_Re.resize(NumGrids + 1);
+   AO_Im.resize(NumGrids + 1);
+   block_OffsetsI.SetSize(3);
+   block_OffsetsJ.SetSize(3);
+   block_OffsetsI[0]=0;
+   block_OffsetsJ[0]=0;
+
+   AO_Re[NumGrids] = Af_Re;
+   AO_Im[NumGrids] = Af_Im;
+
+   AO[NumGrids] = new ComplexOperator(AO_Re[NumGrids], AO_Im[NumGrids],
+                                      false, false, ComplexOperator::HERMITIAN);
+
+   Ac = new ComplexHypreParMatrix(Ac_Re, Ac_Im, false, false,
+                                  ComplexOperator::HERMITIAN);
+
+#ifdef MFEM_USE_STRUMPACK
+   HypreParMatrix *Ahyp0 = Ac->GetSystemMatrix();
+   if (printCoarse)
+   {
+      Ahyp0->Print("A0PA.txt");
+   }
+
+   //StpA = new STRUMPACKRowLocMatrix(*Ac->GetSystemMatrix());
+   StpA = new STRUMPACKRowLocMatrix(*Ahyp0);
+   strumpack = new STRUMPACKSolver(*StpA);
+   strumpack->SetPrintFactorStatistics(false);
+   strumpack->SetPrintSolveStatistics(false);
+   strumpack->SetOperator(*StpA);
+   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+   strumpack->DisableMatching();
+   invAc = strumpack;
+#endif
+
+   for (int i = NumGrids ; i > 0; i--)
+   {
+      Pt[i - 1] = new TransposeOperator(P[i - 1]);
+      AO_Re[i - 1] = new TripleProductOperator(Pt[i - 1], AO_Re[i], P[i - 1], false,
+                                               false, false);
+      AO_Im[i - 1] = (AO_Im[i] == NULL) ? NULL : new TripleProductOperator(Pt[i - 1],
+                                                                           AO_Im[i], P[i - 1], false, false, false);
+      AO[i - 1] = new ComplexOperator(AO_Re[i - 1], AO_Im[i - 1],
+                                      false, false, ComplexOperator::HERMITIAN);
+   }
+
+   // Check if direct solver is found
+   if (!invAc) { MFEM_ABORT("Direct Solver of coarse solve not found"); }
+
+   if (NumGrids > 0)
+   {
+      S[NumGrids - 1] = &Jacobi;
+   }
+
+   // construct smoothers
+   for (int i = NumGrids - 2; i >= 0 ; i--)
+   {
+      S[i] = new TripleProductOperator(Pt[i+1], S[i+1], P[i+1], false, false, false);
+   }
+}
+
+void ComplexGMGPASolver::Mult(const Vector &r, Vector &z) const
+{
+   // Residual vectors
+   std::vector<Vector> rv(NumGrids + 1);
+   // correction vectors
+   std::vector<Vector> zv(NumGrids + 1);
+   // allocation
+
+   // TODO: make the block operators members of the class, so they do not reallocate their data (e.g. tmp) on every Mult() call.
+
+   for (int i = 0; i <= NumGrids ; i++)
+   {
+      int n = AO[i]->Width();
+      rv[i].SetSize(n);
+      zv[i].SetSize(n);
+   }
+   // Initial residual
+   rv[NumGrids] = r;
+   // smooth and update residuals down to the coarsest level
+   for (int i = NumGrids; i > 0 ; i--)
+   {
+      // Pre smooth
+      block_OffsetsI[1] = S[i - 1]->Height();
+      block_OffsetsI[2] = S[i - 1]->Height();
+      block_OffsetsI.PartialSum();
+      BlockOperator BlkS(block_OffsetsI);
+      BlkS.SetBlock(0,0,S[i-1]);
+      BlkS.SetBlock(1,1,S[i-1]);
+      BlkS.Mult(rv[i], zv[i]);
+      zv[i] *= theta;
+      //S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
+
+      // compute residual
+      Vector w(AO[i]->Height());
+      AO[i]->Mult(zv[i], w);
+      rv[i] -= w;
+      // Restrict
+      block_OffsetsI[1] = P[i - 1]->GetNumRows();
+      block_OffsetsI[2] = P[i - 1]->GetNumRows();
+      block_OffsetsI.PartialSum();
+      block_OffsetsJ[1] = P[i - 1]->GetNumCols();
+      block_OffsetsJ[2] = P[i - 1]->GetNumCols();
+      block_OffsetsJ.PartialSum();
+      BlockOperator BlkP(block_OffsetsI,block_OffsetsJ);
+      BlkP.SetBlock(0,0,P[i-1]);
+      BlkP.SetBlock(1,1,P[i-1]);
+      BlkP.MultTranspose(rv[i], rv[i - 1]);
+   }
+   // Coarse grid Stiffness matrix
+   invAc->Mult(rv[0], zv[0]);
+   //
+   for (int i = 1; i <= NumGrids ; i++)
+   {
+      // Prolong correction
+      block_OffsetsI[1] = P[i - 1]->GetNumRows();
+      block_OffsetsI[2] = P[i - 1]->GetNumRows();
+      block_OffsetsI.PartialSum();
+      block_OffsetsJ[1] = P[i - 1]->GetNumCols();
+      block_OffsetsJ[2] = P[i - 1]->GetNumCols();
+      block_OffsetsJ.PartialSum();
+      BlockOperator BlkP(block_OffsetsI,block_OffsetsJ);
+      BlkP.SetBlock(0,0,P[i-1]);
+      BlkP.SetBlock(1,1,P[i-1]);
+      Vector u(BlkP.Height());
+      BlkP.Mult(zv[i - 1], u);
+      // Update correction
+      zv[i] += u;
+      // Update residual
+      Vector v(AO[i]->Height());
+      AO[i]->Mult(u, v); rv[i] -= v;
+      // Post smooth
+      block_OffsetsI[1] = S[i - 1]->Height();
+      block_OffsetsI[2] = S[i - 1]->Height();
+      block_OffsetsI.PartialSum();
+      BlockOperator BlkS(block_OffsetsI);
+      BlkS.SetBlock(0,0,S[i-1]);
+      BlkS.SetBlock(1,1,S[i-1]);
+      BlkS.Mult(rv[i], v);
+      v *= theta;
+      //S[i - 1]->Mult(rv[i], v); v *= theta;
+      // Update correction
+      zv[i] += v;
+   }
+   z = zv[NumGrids];
+}
+
+ComplexGMGPASolver::~ComplexGMGPASolver()
+{
+   delete invAc;
+#ifdef MFEM_USE_STRUMPACK
+   delete StpA;
+#endif
+   delete Ac;
+}
+#endif
+
 struct WeightMinHeap
 {
    const std::vector<double> &w;
@@ -2825,7 +3388,7 @@ void UMFPackSolver::Mult(const Vector &b, Vector &x) const
    {
       int status =
          umfpack_di_solve(UMFPACK_At, mat->GetI(), mat->GetJ(),
-                          mat->GetData(), x, b, Numeric, Control, Info);
+                          mat->GetData(), x.HostWrite(), b.HostRead(), Numeric, Control, Info);
       umfpack_di_report_info(Control, Info);
       if (status < 0)
       {
@@ -2836,7 +3399,8 @@ void UMFPackSolver::Mult(const Vector &b, Vector &x) const
    else
    {
       SuiteSparse_long status =
-         umfpack_dl_solve(UMFPACK_At, AI, AJ, mat->GetData(), x, b,
+         umfpack_dl_solve(UMFPACK_At, AI, AJ, mat->GetData(), x.HostWrite(),
+                          b.HostRead(),
                           Numeric, Control, Info);
       umfpack_dl_report_info(Control, Info);
       if (status < 0)
@@ -2972,4 +3536,137 @@ KLUSolver::~KLUSolver()
 
 #endif // MFEM_USE_SUITESPARSE
 
+void OrthominSolver::UpdateVectors()
+{
+   p.SetSize(width);
+   Ap.SetSize(width);
+   r.SetSize(width);
+   omk = width;
+}
+
+void OrthominSolver::Mult(const Vector &b, Vector &x) const
+{
+   pprev.clear();
+   Apprev.clear();
+   Ap2prev.clear();
+
+   if (iterative_mode)
+   {
+      oper->Mult(x, r);
+      subtract(b, r, r); // r = b - A x
+   }
+   else
+   {
+      x = 0.0;
+      r = b;
+   }
+
+   double resid = Norm(r);
+   MFEM_ASSERT(IsFinite(resid), "resid = " << resid);
+   if (print_level >= 0)
+      mfem::out << "   Iteration : " << setw(3) << 0
+                << "   ||r|| = " << resid << '\n';
+
+   const double tol_goal = std::max(resid*rel_tol, abs_tol);
+
+   if (resid <= tol_goal)
+   {
+      final_norm = resid;
+      final_iter = 0;
+      converged = 1;
+      return;
+   }
+
+   p = r;
+   oper->Mult(p, Ap);
+
+   for (int i = 1; i <= max_iter; i++)
+   {
+      // TODO: can this be optimized more?
+      const double Ap2 = Dot(Ap, Ap);
+      const double ai = Dot(r, Ap) / Ap2;
+
+      x.Add(ai, p);   // x += a_i * p
+      r.Add(-ai, Ap); // r -= a_i * Ap
+
+      resid = Norm(r);
+      MFEM_ASSERT(IsFinite(resid), "resid = " << resid);
+      if (resid < tol_goal)
+      {
+         if (print_level >= 0)
+            mfem::out << "   Iteration : " << setw(3) << i
+                      << "   ||r|| = " << resid << '\n';
+         final_norm = resid;
+         final_iter = i;
+         converged = 1;
+         return;
+      }
+      if (print_level >= 0)
+         mfem::out << "   Iteration : " << setw(3) << i
+                   << "   ||r|| = " << resid << '\n';
+
+      pprev.push_back(p);
+      Apprev.push_back(Ap);
+      Ap2prev.push_back(Ap2);
+
+      if (pprev.size() > omk) // orthomin(k)
+      {
+         //cout << "Orthomin erasing" << endl;
+         pprev.erase(pprev.begin());
+         Apprev.erase(Apprev.begin());
+         Ap2prev.erase(Ap2prev.begin());
+      }
+
+      // Compute next p
+      p = r;
+      oper->Mult(p, Ap);  // Ap = Ar
+
+      const int np = pprev.size();
+
+      //cout << "Orthomin using " << np << " p vectors" << endl;
+
+      MFEM_VERIFY(np == Apprev.size() && np == Ap2prev.size(), "");
+
+      for (int j=0; j<np; ++j)
+      {
+         p.Add(-Dot(Ap, Apprev[j]) / Ap2prev[j], pprev[j]);
+      }
+
+      oper->Mult(p, Ap);
+   }
+}
+
+
+IncompleteCholesky::IncompleteCholesky(SparseMatrix &A_) : A(&A_)
+{
+#ifdef MFEM_USE_CUDA
+   A->IncompleteCholeskySetup();
+#endif
+}
+
+void IncompleteCholesky::Mult(const Vector &b, Vector &x) const
+{
+#ifdef MFEM_USE_CUDA
+   A->IncompleteCholeskyMult(b, x);
+#else
+   x = b;
+#endif
+}
+
+ILUcusparse::ILUcusparse(SparseMatrix &A_) : A(&A_)
+{
+#ifdef MFEM_USE_CUDA
+   A->ILUSetup();
+#endif
+}
+
+void ILUcusparse::Mult(const Vector &b, Vector &x) const
+{
+#ifdef MFEM_USE_CUDA
+   A->ILUMult(b, x);
+#else
+   x = b;
+#endif
+}
+
 }

linalg/solvers.hpp

@@ -291,6 +291,7 @@ class GMRESSolver : public IterativeSolver
 {
 protected:
    int m; // see SetKDim()
+   std::string name;
 
 public:
    GMRESSolver() { m = 50; }
@@ -303,6 +304,11 @@ public:
    void SetKDim(int dim) { m = dim; }
 
    virtual void Mult(const Vector &b, Vector &x) const;
+
+   void SetName(const std::string &s)
+   {
+      name = s;
+   }
 };
 
 /// FGMRES method
@@ -706,6 +712,128 @@ private:
    mutable Array<int> ipiv;
 };
 
+#ifdef MFEM_USE_MPI
+
+class GMGSolver : public Solver
+{
+private:
+   /// The linear system matrix
+   HypreParMatrix * Af;
+   std::vector<HypreParMatrix *> A;
+   std::vector<HypreParMatrix *> P;
+   std::vector<HypreSmoother  *> S;
+   int NumGrids;
+   //
+#ifdef MFEM_USE_PETSC
+   PetscLinearSolver *petsc = nullptr;
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   STRUMPACKRowLocMatrix *StpA = nullptr;
+   STRUMPACKSolver *strumpack = nullptr;
+#endif
+#ifdef MFEM_USE_SUPERLU
+   SuperLURowLocMatrix *SluA = nullptr;
+   SuperLUSolver *superlu = nullptr;
+#endif
+   Solver * invAc=nullptr;
+   double theta = 1.0;
+public:
+   enum CoarseSolver { PETSC, SUPERLU, STRUMPACK};
+
+   GMGSolver(HypreParMatrix * Af_, std::vector<HypreParMatrix *> P_, CoarseSolver);
+
+   virtual void SetOperator(const Operator &op) {}
+
+   virtual void SetSmootherType(const HypreSmoother::Type type) const;
+
+   virtual void SetTheta(const double a) {theta = a;}
+
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~GMGSolver();
+};
+
+class ComplexGMGSolver : public Solver
+{
+private:
+   /// The linear system matrix
+   ComplexHypreParMatrix * Af;
+   std::vector<ComplexHypreParMatrix *> A;
+   std::vector<HypreParMatrix *> P;
+   std::vector<HypreSmoother  *> S;
+   int NumGrids;
+   Solver * invAc=nullptr;
+   //
+#ifdef MFEM_USE_PETSC
+   PetscLinearSolver *petsc = nullptr;
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   STRUMPACKRowLocMatrix *StpA = nullptr;
+   STRUMPACKSolver *strumpack = nullptr;
+#endif
+#ifdef MFEM_USE_SUPERLU
+   SuperLURowLocMatrix *SluA = nullptr;
+   SuperLUSolver *superlu = nullptr;
+#endif
+   double theta = 1.0;
+   mutable Array<int> block_OffsetsI;
+   mutable Array<int> block_OffsetsJ;
+public:
+   enum CoarseSolver { PETSC, SUPERLU, STRUMPACK, UMFPACK };
+
+   ComplexGMGSolver(ComplexHypreParMatrix * Af_, std::vector<HypreParMatrix *> P_,
+                    CoarseSolver cs, bool printCoarse=false);
+   virtual void SetOperator(const Operator &op) {}
+
+   virtual void SetSmootherType(const HypreSmoother::Type type) const;
+
+   virtual void SetTheta(const double a) {theta = a;}
+
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual ~ComplexGMGSolver();
+};
+
+class ComplexGMGPASolver : public Solver
+{
+private:
+   ComplexOperator * AOf;
+   std::vector<ComplexOperator *> AO;
+   std::vector<Operator*> AO_Re;
+   std::vector<Operator*> AO_Im;
+   std::vector<HypreParMatrix *> P;
+   std::vector<Operator*> Pt;
+   int NumGrids;
+   Solver * invAc=nullptr;
+   ComplexHypreParMatrix *Ac;
+   Vector diagRe;
+   OperatorJacobiSmoother Jacobi;
+
+   std::vector<Operator*> S;
+
+#ifdef MFEM_USE_STRUMPACK
+   STRUMPACKRowLocMatrix *StpA = nullptr;
+   STRUMPACKSolver *strumpack = nullptr;
+#endif
+
+   double theta = 1.0;
+   mutable Array<int> block_OffsetsI;
+   mutable Array<int> block_OffsetsJ;
+public:
+   ComplexGMGPASolver(MPI_Comm comm, Operator * Af_Re, Operator * Af_Im,
+                      Vector& diagRe_,
+                      Array<int>& ess_tdof_list,
+                      std::vector<HypreParMatrix *> P_,
+                      HypreParMatrix * Ac_Re, HypreParMatrix * Ac_Im, bool printCoarse=false);
+
+   virtual void SetOperator(const Operator &op) {}
+
+   //virtual void SetSmootherType(const HypreSmoother::Type type) const;
+
+   virtual void SetTheta(const double a) {theta = a;}
+
+   virtual void Mult(const Vector &r, Vector &z) const;
+   ~ComplexGMGPASolver();
+};
+#endif
 
 /// Monitor that checks whether the residual is zero at a given set of dofs.
 /** This monitor is useful for checking if the initial guess, rhs, operator, and
@@ -724,7 +852,6 @@ public:
                         bool final) override;
 };
 
-
 #ifdef MFEM_USE_SUITESPARSE
 
 /// Direct sparse solver using UMFPACK
@@ -799,6 +926,58 @@ public:
 
 #endif // MFEM_USE_SUITESPARSE
 
+class OrthominSolver : public IterativeSolver
+{
+private:
+   int omk;
+
+protected:
+   mutable Vector p, r, Ap, Ar;
+   mutable std::vector<Vector> pprev, Apprev;
+   mutable std::vector<double> Ap2prev;
+
+   void UpdateVectors();
+
+public:
+   OrthominSolver() { }
+
+#ifdef MFEM_USE_MPI
+   OrthominSolver(MPI_Comm _comm) : IterativeSolver(_comm) { }
+#endif
+
+   virtual void SetOperator(const Operator &op)
+   { IterativeSolver::SetOperator(op); UpdateVectors(); }
+
+   virtual void Mult(const Vector &b, Vector &x) const;
+
+   void SetKDim(const int k)
+   {
+      omk = k;
+   }
+};
+
+class IncompleteCholesky : public IterativeSolver
+{
+private:
+   SparseMatrix *A;
+
+public:
+   IncompleteCholesky(SparseMatrix &A_);
+
+   virtual void Mult(const Vector &b, Vector &x) const;
+};
+
+class ILUcusparse : public IterativeSolver
+{
+private:
+   SparseMatrix *A;
+
+public:
+   ILUcusparse(SparseMatrix &A_);
+
+   virtual void Mult(const Vector &b, Vector &x) const;
+};
+
 }
 
 #endif // MFEM_SOLVERS

linalg/sparsemat.cpp

@@ -101,8 +101,19 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
      At(NULL),
      isSorted(issorted)
 {
-   I.Wrap(i, height+1, ownij);
-   J.Wrap(j, I[height], ownij);
+   //I.Wrap(i, height+1, ownij);
+   //J.Wrap(j, I[height], ownij);
+
+   I.New(height+1);
+   for (int q=0; q<height+1; ++q)
+   {
+      I[q] = i[q];
+   }
+   J.New(I[height]);
+   for (int q=0; q<I[height]; ++q)
+   {
+      J[q] = j[q];
+   }
 
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = NULL;
@@ -110,7 +121,12 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
 
    if (data)
    {
-      A.Wrap(data, I[height], owna);
+      //A.Wrap(data, I[height], owna);
+      A.New(I[height]);
+      for (int q=0; q<I[height]; ++q)
+      {
+         A[q] = data[q];
+      }
    }
    else
    {
@@ -288,6 +304,7 @@ void SparseMatrix::SetEmpty()
       cusparseDestroySpMat(matA_descr);
       cusparseDestroyDnVec(vecX_descr);
       cusparseDestroyDnVec(vecY_descr);
+      cusparseDestroyDnVec(vecZ_descr);
       initBuffers = false;
    }
 #endif
@@ -3161,6 +3178,7 @@ void SparseMatrix::Destroy()
       cusparseDestroySpMat(matA_descr);
       cusparseDestroyDnVec(vecX_descr);
       cusparseDestroyDnVec(vecY_descr);
+      //cusparseDestroyDnVec(vecZ_descr);  // TODO: destroy only if allocated
       initBuffers = false;
    }
 #endif
@@ -3899,4 +3917,323 @@ void SparseMatrix::Swap(SparseMatrix &other)
    mfem::Swap(isSorted, other.isSorted);
 }
 
+#ifdef MFEM_USE_CUDA
+void SparseMatrix::IncompleteCholeskyMult(const Vector &x, Vector &y) const
+{
+   if (!(Device::Allows(Backend::CUDA_MASK) && useCuSparse))
+   {
+      y = x;
+      return;
+   }
+
+   MFEM_VERIFY(initCholesky, "Setup not done");
+
+   const double alpha = 1.0;
+
+   auto d_x = x.Read();
+   auto d_y = y.ReadWrite();
+   auto d_z = vecZ.ReadWrite();
+
+   const int height = this->height;
+   const int nnz = J.Capacity();
+   int64_t m = height;
+
+   auto d_csrRowPtr = Read(I, height+1);
+   auto d_csrColInd = Read(J, nnz);
+   auto d_csrVal = Read(A, nnz);
+
+   cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
+   cusparseDnVecSetValues(vecY_descr, d_y);
+   cusparseDnVecSetValues(vecZ_descr, d_z);
+
+   const cusparseOperation_t trans_L  = CUSPARSE_OPERATION_NON_TRANSPOSE;
+   const cusparseOperation_t trans_Lt = CUSPARSE_OPERATION_TRANSPOSE;
+
+   const cusparseSolvePolicy_t policy_L  = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
+   const cusparseSolvePolicy_t policy_Lt = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
+
+   // Solve L*z = x
+   cusparseDcsrsv2_solve(handle, trans_L, m, nnz, &alpha, descr_L,
+                         d_csrVal, d_csrRowPtr, d_csrColInd, info_L,
+                         d_x, d_z, policy_L, pBuffer);
+
+   // Solve L'*y = z
+   cusparseDcsrsv2_solve(handle, trans_Lt, m, nnz, &alpha, descr_L,
+                         d_csrVal, d_csrRowPtr, d_csrColInd, info_Lt,
+                         d_z, d_y, policy_Lt, pBuffer);
+}
+
+void SparseMatrix::IncompleteCholeskySetup()
+{
+   if (!(Device::Allows(Backend::CUDA_MASK) && useCuSparse))
+   {
+      return;
+   }
+
+   MFEM_VERIFY(!initILU && !initCholesky, "");
+
+   const int height = this->height;
+   const int nnz = J.Capacity();
+
+   auto d_csrRowPtr = Read(I, height+1);
+   auto d_csrColInd = Read(J, nnz);
+   auto d_csrVal = Read(A, nnz);
+
+   //MFEM_VERIFY(I[0] == 0, "cusparse thinks this is not zero based");
+
+   csric02Info_t info_M  = 0;
+   int bufferSize_M;
+   int pBufferSize_L;
+   int pBufferSize_Lt;
+   int pBufferSize;
+   int structural_zero;
+   int numerical_zero;
+
+   const cusparseSolvePolicy_t policy_M  = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
+   const cusparseSolvePolicy_t policy_L  = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
+   const cusparseSolvePolicy_t policy_Lt = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
+   const cusparseOperation_t trans_L  = CUSPARSE_OPERATION_NON_TRANSPOSE;
+   const cusparseOperation_t trans_Lt = CUSPARSE_OPERATION_TRANSPOSE;
+
+   // step 1: create a descriptor which contains
+   // - matrix M is base-0
+   // - matrix L is base-0
+   // - matrix L is lower triangular
+   // - matrix L has non-unit diagonal
+   cusparseCreateMatDescr(&descr_M);
+   cusparseSetMatIndexBase(descr_M, CUSPARSE_INDEX_BASE_ZERO);
+   cusparseSetMatType(descr_M, CUSPARSE_MATRIX_TYPE_GENERAL);
+
+   cusparseCreateMatDescr(&descr_L);
+   cusparseSetMatIndexBase(descr_L, CUSPARSE_INDEX_BASE_ZERO);
+   cusparseSetMatType(descr_L, CUSPARSE_MATRIX_TYPE_GENERAL);
+   cusparseSetMatFillMode(descr_L, CUSPARSE_FILL_MODE_LOWER);
+   cusparseSetMatDiagType(descr_L, CUSPARSE_DIAG_TYPE_NON_UNIT);
+
+   // step 2: create a empty info structure
+   // we need one info for csric02 and two info's for csrsv2
+   cusparseCreateCsric02Info(&info_M);
+   cusparseCreateCsrsv2Info(&info_L);
+   cusparseCreateCsrsv2Info(&info_Lt);
+
+   int64_t m = height;
+
+   // step 3: query how much memory used in csric02 and csrsv2, and allocate the buffer
+   cusparseDcsric02_bufferSize(handle, m, nnz,
+                               descr_M, const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
+                               const_cast<int *>(d_csrColInd), info_M, &bufferSize_M);
+   cusparseDcsrsv2_bufferSize(handle, trans_L, m, nnz,
+                              descr_L, const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
+                              const_cast<int *>(d_csrColInd), info_L, &pBufferSize_L);
+   cusparseDcsrsv2_bufferSize(handle, trans_Lt, m, nnz,
+                              descr_L, const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
+                              const_cast<int *>(d_csrColInd), info_Lt,&pBufferSize_Lt);
+
+   pBufferSize = max(bufferSize_M, max(pBufferSize_L, pBufferSize_Lt));
+
+   // pBuffer returned by cudaMalloc is automatically aligned to 128 bytes.
+   cudaMalloc((void**)&pBuffer, pBufferSize);
+
+   // step 4: perform analysis of incomplete Cholesky on M
+   //         perform analysis of triangular solve on L
+   //         perform analysis of triangular solve on L'
+   // The lower triangular part of M has the same sparsity pattern as L, so
+   // we can do analysis of csric02 and csrsv2 simultaneously.
+
+   cusparseDcsric02_analysis(handle, m, nnz, descr_M,
+                             d_csrVal, d_csrRowPtr, d_csrColInd, info_M,
+                             policy_M, pBuffer);
+   status = cusparseXcsric02_zeroPivot(handle, info_M, &structural_zero);
+   if (CUSPARSE_STATUS_ZERO_PIVOT == status)
+   {
+      printf("A(%d,%d) is missing\n", structural_zero, structural_zero);
+   }
+
+   cusparseDcsrsv2_analysis(handle, trans_L, m, nnz, descr_L,
+                            d_csrVal, d_csrRowPtr, d_csrColInd,
+                            info_L, policy_L, pBuffer);
+
+   cusparseDcsrsv2_analysis(handle, trans_Lt, m, nnz, descr_L,
+                            d_csrVal, d_csrRowPtr, d_csrColInd,
+                            info_Lt, policy_Lt, pBuffer);
+
+   // step 5: M = L * L'
+   cusparseDcsric02(handle, m, nnz, descr_M,
+                    const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
+                    const_cast<int *>(d_csrColInd), info_M, policy_M, pBuffer);
+   status = cusparseXcsric02_zeroPivot(handle, info_M, &numerical_zero);
+   if (CUSPARSE_STATUS_ZERO_PIVOT == status)
+   {
+      printf("L(%d,%d) is zero\n", numerical_zero, numerical_zero);
+   }
+
+   vecZ.SetSize(height);
+   vecZ = 0.0;
+   auto d_z = vecZ.ReadWrite();
+   cusparseCreateDnVec(&vecZ_descr, vecZ.Size(), d_z, CUDA_R_64F);
+
+   initCholesky = true;
+}
+
+void SparseMatrix::ILUMult(const Vector &x, Vector &y) const
+{
+   MFEM_VERIFY(initILU, "Setup not done");
+
+   const double alpha = 1.0;
+
+   auto d_x = x.Read();
+   auto d_y = y.ReadWrite();
+   auto d_z = vecZ.ReadWrite();
+
+   const int height = this->height;
+   const int nnz = J.Capacity();
+   int64_t m = height;
+
+   auto d_csrRowPtr = Read(I, height+1);
+   auto d_csrColInd = Read(J, nnz);
+   auto d_csrVal = Read(A, nnz);
+
+   cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
+   cusparseDnVecSetValues(vecY_descr, d_y);
+   cusparseDnVecSetValues(vecZ_descr, d_z);
+
+   const cusparseSolvePolicy_t policy_L = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
+   const cusparseSolvePolicy_t policy_U = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
+   const cusparseOperation_t trans_L  = CUSPARSE_OPERATION_NON_TRANSPOSE;
+   const cusparseOperation_t trans_U  = CUSPARSE_OPERATION_NON_TRANSPOSE;
+
+   // Solve L*z = x
+   cusparseDcsrsv2_solve(handle, trans_L, m, nnz, &alpha, descr_L,
+                         d_csrVal, d_csrRowPtr, d_csrColInd, info_L,
+                         d_x, d_z, policy_L, pBuffer);
+
+   // Solve U*y = z
+   cusparseDcsrsv2_solve(handle, trans_U, m, nnz, &alpha, descr_U,
+                         d_csrVal, d_csrRowPtr, d_csrColInd, info_U,
+                         d_z, d_y, policy_U, pBuffer);
+
+   // TODO: destructor
+}
+
+void SparseMatrix::ILUSetup()
+{
+   if (!(Device::Allows(Backend::CUDA_MASK) && useCuSparse))
+   {
+      return;
+   }
+
+   MFEM_VERIFY(!initILU && !initCholesky, "");
+
+   const int height = this->height;
+   const int nnz = J.Capacity();
+
+   auto d_csrRowPtr = Read(I, height+1);
+   auto d_csrColInd = Read(J, nnz);
+   auto d_csrVal = Read(A, nnz);
+
+   csrilu02Info_t info_M  = 0;
+   int pBufferSize_M;
+   int pBufferSize_L;
+   int pBufferSize_U;
+   int pBufferSize;
+   int structural_zero;
+   int numerical_zero;
+   const cusparseSolvePolicy_t policy_M = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
+   const cusparseSolvePolicy_t policy_L = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
+   const cusparseSolvePolicy_t policy_U = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
+   const cusparseOperation_t trans_L  = CUSPARSE_OPERATION_NON_TRANSPOSE;
+   const cusparseOperation_t trans_U  = CUSPARSE_OPERATION_NON_TRANSPOSE;
+
+   // step 1: create a descriptor which contains
+   // - matrix M is base-0
+   // - matrix L is base-0
+   // - matrix L is lower triangular
+   // - matrix L has unit diagonal
+   // - matrix U is base-0
+   // - matrix U is upper triangular
+   // - matrix U has non-unit diagonal
+   cusparseCreateMatDescr(&descr_M);
+   cusparseSetMatIndexBase(descr_M, CUSPARSE_INDEX_BASE_ZERO);
+   cusparseSetMatType(descr_M, CUSPARSE_MATRIX_TYPE_GENERAL);
+
+   cusparseCreateMatDescr(&descr_L);
+   cusparseSetMatIndexBase(descr_L, CUSPARSE_INDEX_BASE_ZERO);
+   cusparseSetMatType(descr_L, CUSPARSE_MATRIX_TYPE_GENERAL);
+   cusparseSetMatFillMode(descr_L, CUSPARSE_FILL_MODE_LOWER);
+   cusparseSetMatDiagType(descr_L, CUSPARSE_DIAG_TYPE_UNIT);
+
+   cusparseCreateMatDescr(&descr_U);
+   cusparseSetMatIndexBase(descr_U, CUSPARSE_INDEX_BASE_ZERO);
+   cusparseSetMatType(descr_U, CUSPARSE_MATRIX_TYPE_GENERAL);
+   cusparseSetMatFillMode(descr_U, CUSPARSE_FILL_MODE_UPPER);
+   cusparseSetMatDiagType(descr_U, CUSPARSE_DIAG_TYPE_NON_UNIT);
+
+   // step 2: create a empty info structure
+   // we need one info for csrilu02 and two info's for csrsv2
+   cusparseCreateCsrilu02Info(&info_M);
+   cusparseCreateCsrsv2Info(&info_L);
+   cusparseCreateCsrsv2Info(&info_U);
+
+   // step 3: query how much memory used in csrilu02 and csrsv2, and allocate the buffer
+   int64_t m = height;
+
+   cusparseDcsrilu02_bufferSize(handle, m, nnz, descr_M,
+                                const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
+                                const_cast<int *>(d_csrColInd), info_M, &pBufferSize_M);
+   cusparseDcsrsv2_bufferSize(handle, trans_L, m, nnz, descr_L,
+                              const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
+                              const_cast<int *>(d_csrColInd), info_L, &pBufferSize_L);
+   cusparseDcsrsv2_bufferSize(handle, trans_U, m, nnz, descr_U,
+                              const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
+                              const_cast<int *>(d_csrColInd), info_U, &pBufferSize_U);
+
+   pBufferSize = max(pBufferSize_M, max(pBufferSize_L, pBufferSize_U));
+
+   // pBuffer returned by cudaMalloc is automatically aligned to 128 bytes.
+   cudaMalloc((void**)&pBuffer, pBufferSize);
+
+   // step 4: perform analysis of incomplete Cholesky on M
+   //         perform analysis of triangular solve on L
+   //         perform analysis of triangular solve on U
+   // The lower(upper) triangular part of M has the same sparsity pattern as L(U),
+   // we can do analysis of csrilu0 and csrsv2 simultaneously.
+
+   cusparseDcsrilu02_analysis(handle, m, nnz, descr_M,
+                              d_csrVal, d_csrRowPtr, d_csrColInd, info_M,
+                              policy_M, pBuffer);
+   status = cusparseXcsrilu02_zeroPivot(handle, info_M, &structural_zero);
+   if (CUSPARSE_STATUS_ZERO_PIVOT == status)
+   {
+      printf("A(%d,%d) is missing\n", structural_zero, structural_zero);
+   }
+
+   cusparseDcsrsv2_analysis(handle, trans_L, m, nnz, descr_L,
+                            d_csrVal, d_csrRowPtr, d_csrColInd,
+                            info_L, policy_L, pBuffer);
+
+   cusparseDcsrsv2_analysis(handle, trans_U, m, nnz, descr_U,
+                            d_csrVal, d_csrRowPtr, d_csrColInd,
+                            info_U, policy_U, pBuffer);  // bug?
+
+   // step 5: M = L * U
+   cusparseDcsrilu02(handle, m, nnz, descr_M,
+                     const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
+                     const_cast<int *>(d_csrColInd), info_M, policy_M, pBuffer);
+   status = cusparseXcsrilu02_zeroPivot(handle, info_M, &numerical_zero);
+   if (CUSPARSE_STATUS_ZERO_PIVOT == status)
+   {
+      printf("U(%d,%d) is zero\n", numerical_zero, numerical_zero);
+   }
+
+   vecZ.SetSize(height);
+   vecZ = 0.0;
+   auto d_z = vecZ.ReadWrite();
+   cusparseCreateDnVec(&vecZ_descr, vecZ.Size(), d_z, CUDA_R_64F);
+
+   initILU = true;
+
+   // TODO: destructor
+}
+#endif // MFEM_USE_CUDA
+
 }

linalg/sparsemat.hpp

@@ -103,6 +103,22 @@ protected:
    mutable cusparseSpMatDescr_t matA_descr;
    mutable cusparseDnVecDescr_t vecX_descr;
    mutable cusparseDnVecDescr_t vecY_descr;
+
+   mutable cusparseDnVecDescr_t vecZ_descr;
+   mutable Vector vecZ;
+
+   cusparseMatDescr_t descr_M = 0;
+   cusparseMatDescr_t descr_L = 0;
+   cusparseMatDescr_t descr_U = 0;
+
+   csrsv2Info_t  info_L  = 0;
+   csrsv2Info_t  info_Lt = 0;
+   csrsv2Info_t  info_U  = 0;
+
+   void *pBuffer = 0;
+
+   bool initILU = false;
+   bool initCholesky = false;
 #endif
 
 public:
@@ -110,7 +126,6 @@ public:
    SparseMatrix()
    {
       SetEmpty();
-
       InitCuSparse();
    }
 
@@ -610,6 +625,14 @@ public:
 
    void Swap(SparseMatrix &other);
 
+#ifdef MFEM_USE_CUDA
+   void IncompleteCholeskySetup();
+   void IncompleteCholeskyMult(const Vector &x, Vector &y) const;
+
+   void ILUSetup();
+   void ILUMult(const Vector &x, Vector &y) const;
+#endif
+
    /// Destroys sparse matrix.
    virtual ~SparseMatrix()
    {

linalg/strumpack.hpp

@@ -26,6 +26,8 @@
 namespace mfem
 {
 
+class HypreParMatrix;
+
 class STRUMPACKRowLocMatrix : public Operator
 {
 public:

linalg/vector.cpp

@@ -211,6 +211,21 @@ Vector &Vector::Add(const double a, const Vector &Va)
    return *this;
 }
 
+Vector &Vector::AddOffset(const double a, const Vector &Va, const int offset)
+{
+   MFEM_ASSERT(size >= Va.size + offset, "incompatible Vectors!");
+
+   if (a != 0.0)
+   {
+      const int N = Va.size;
+      const bool use_dev = UseDevice() || Va.UseDevice();
+      auto y = ReadWrite(use_dev);
+      auto x = Va.Read(use_dev);
+      MFEM_FORALL_SWITCH(use_dev, i, N, y[offset + i] += a * x[i];);
+   }
+   return *this;
+}
+
 Vector &Vector::Set(const double a, const Vector &Va)
 {
    MFEM_ASSERT(size == Va.size, "incompatible Vectors!");
@@ -223,6 +238,21 @@ Vector &Vector::Set(const double a, const Vector &Va)
    return *this;
 }
 
+Vector &Vector::SetOffset(const double a, const Vector &Va, const int offset,
+                          const int myoffset, const int L)
+{
+   const int N = L > 0 ? L : size;
+
+   MFEM_ASSERT(N + offset <= Va.size &&
+               N + myoffset <= size, "incompatible Vectors!");
+
+   const bool use_dev = UseDevice() || Va.UseDevice();
+   auto x = Va.Read(use_dev);
+   auto y = Write(use_dev);
+   MFEM_FORALL_SWITCH(use_dev, i, N, y[myoffset + i] = a * x[offset + i];);
+   return *this;
+}
+
 void Vector::SetVector(const Vector &v, int offset)
 {
    MFEM_ASSERT(v.Size() + offset <= size, "invalid sub-vector");

linalg/vector.hpp

@@ -249,9 +249,16 @@ public:
    /// (*this) += a * Va
    Vector &Add(const double a, const Vector &Va);
 
+   /// (*this) += a * Va, with an offset for this
+   Vector &AddOffset(const double a, const Vector &Va, const int offset);
+
    /// (*this) = a * x
    Vector &Set(const double a, const Vector &x);
 
+   /// (*this) = a * x, with an offset for x
+   Vector &SetOffset(const double a, const Vector &Va, const int offset,
+                     const int myoffset=0, const int L=0);
+
    void SetVector(const Vector &v, int offset);
 
    /// (*this) = -(*this)

mesh/mesh.cpp

@@ -5727,6 +5727,59 @@ int *Mesh::CartesianPartitioning(int nxyz[])
    return partitioning;
 }
 
+int *Mesh::CartesianPartitioningXY(int nxyz[], const int X, const int Y)
+{
+   int *partitioning;
+   double pmin[3] = { infinity(), infinity(), infinity() };
+   double pmax[3] = { -infinity(), -infinity(), -infinity() };
+   // find a bounding box using the vertices
+   for (int vi = 0; vi < NumOfVertices; vi++)
+   {
+      const double *p = vertices[vi]();
+      for (int i = 0; i < spaceDim; i++)
+      {
+         if (p[i] < pmin[i]) { pmin[i] = p[i]; }
+         if (p[i] > pmax[i]) { pmax[i] = p[i]; }
+      }
+   }
+
+   partitioning = new int[NumOfElements];
+
+   // determine the partitioning using the centers of the elements
+   double ppt[3];
+   Vector pt(ppt, spaceDim);
+   int pvec[3];
+   for (int el = 0; el < NumOfElements; el++)
+   {
+      GetElementTransformation(el)->Transform(
+         Geometries.GetCenter(GetElementBaseGeometry(el)), pt);
+      int part = 0;
+      for (int i = spaceDim-1; i >= 0; i--)
+      {
+         int idx = (int)floor(nxyz[i]*((pt(i) - pmin[i])/(pmax[i] - pmin[i])));
+         if (idx < 0) { idx = 0; }
+         if (idx >= nxyz[i]) { idx = nxyz[i]-1; }
+         part = part * nxyz[i] + idx;
+         pvec[i] = idx;
+      }
+
+      {
+         const int idX = pvec[0] / X;
+         const int idY = pvec[1] / Y;
+
+         const int idbX = pvec[0] - (idX * X);
+         const int idbY = pvec[1] - (idY * Y);
+
+         part = (pvec[2] * nxyz[1] * nxyz[0]) + (((idY * nxyz[0] / X) + idX) * X * Y) +
+                (idbY * X) + idbX;
+      }
+
+      partitioning[el] = part;
+   }
+
+   return partitioning;
+}
+
 int *Mesh::GeneratePartitioning(int nparts, int part_method)
 {
 #ifdef MFEM_USE_METIS

mesh/mesh.hpp

@@ -260,6 +260,8 @@ protected:
        reference element at the center of the element. */
    void GetElementJacobian(int i, DenseMatrix &J);
 
+   //void GetElementCenter(int i, Vector &c);
+
    void MarkForRefinement();
    void MarkTriMeshForRefinement();
    void GetEdgeOrdering(DSTable &v_to_v, Array<int> &order);
@@ -506,6 +508,8 @@ public:
       InitMesh(_Dim, _spaceDim, NVert, NElem, NBdrElem);
    }
 
+   void GetElementCenter(int i, Vector &c);
+
    /** @name Methods for Mesh construction.
 
        These methods are intended to be used with the @ref mfem_Mesh_init_ctor
@@ -1077,6 +1081,8 @@ public:
    virtual void ReorientTetMesh();
 
    int *CartesianPartitioning(int nxyz[]);
+   int *CartesianPartitioningXY(int nxyz[], const int X, const int Y);
+
    int *GeneratePartitioning(int nparts, int part_method = 1);
    void CheckPartitioning(int *partitioning);
 
@@ -1300,8 +1306,6 @@ public:
 
    double GetElementVolume(int i);
 
-   void GetElementCenter(int i, Vector &center);
-
    /// Returns the minimum and maximum corners of the mesh bounding box.
    /** For high-order meshes, the geometry is first refined @a ref times. */
    void GetBoundingBox(Vector &min, Vector &max, int ref = 2);

tests/unit/fem/test_pa_coeff.cpp

@@ -258,6 +258,7 @@ TEST_CASE("Hcurl/Hdiv pa_coeff",
       {
          Coefficient* coeff = nullptr;
          Coefficient* coeff2 = nullptr;
+
          VectorCoefficient* vcoeff = nullptr;
          MatrixCoefficient* mcoeff = nullptr;
          MatrixCoefficient* smcoeff = nullptr;
@@ -558,6 +559,7 @@ TEST_CASE("Hcurl/Hdiv pa_coeff",
          }
 
          delete coeff;
+         delete vcoeff;
          delete coeff2;
          delete vcoeff;
          delete mcoeff;
@@ -787,4 +789,138 @@ TEST_CASE("Hcurl/Hdiv mixed pa_coeff",
    }
 }
 
+TEST_CASE("Hcurl L2 mixed pa_coeff")  // TODO: merge this with the other Hcurl mixed test in rtpa
+{
+   for (dimension = 3; dimension < 4; ++dimension)
+   {
+      Mesh* mesh;
+      const int ne = 2;
+      if (dimension == 3)
+      {
+         mesh = new Mesh(ne, ne, ne, Element::HEXAHEDRON, 1, 1.0, 1.0, 1.0);
+      }
+
+      for (int coeffType = 0; coeffType < 3; ++coeffType)
+      {
+         Coefficient* coeff = nullptr;
+         VectorCoefficient* vcoeff = nullptr;
+         if (coeffType == 0)
+         {
+            coeff = new ConstantCoefficient(12.34);
+         }
+         else if (coeffType == 1)
+         {
+            coeff = new FunctionCoefficient(&coeffFunction);
+         }
+         else if (coeffType == 2)
+         {
+            vcoeff = new VectorFunctionCoefficient(3, &vectorCoeffFunction);
+         }
+
+         for (int integrator = 0; integrator < 1; ++integrator)
+         {
+            std::cout << "Testing " << dimension << "D ND L2 mixed partial assembly with "
+                      << "coeffType " << coeffType << " and "
+                      << "integrator " << integrator << std::endl;
+            for (int order = 1; order < 4; ++order)
+            {
+               FiniteElementCollection* ND_fec =
+                  new ND_FECollection(order, dimension);
+               FiniteElementSpace ND_fespace(mesh, ND_fec);
+
+               Array<int> ess_tdof_list;
+
+               MixedBilinearForm paform(&ND_fespace, &ND_fespace);
+               paform.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+               if (integrator == 0)
+               {
+                  if (coeffType == 2)
+                  {
+                     paform.AddDomainIntegrator(new MixedVectorCurlIntegrator(*vcoeff));
+                  }
+                  else
+                  {
+                     paform.AddDomainIntegrator(new MixedVectorCurlIntegrator(*coeff));
+                  }
+               }
+               else
+               {
+                  if (coeffType == 2)
+                  {
+                     paform.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(*vcoeff));
+                  }
+                  else
+                  {
+                     paform.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(*coeff));
+                  }
+               }
+
+               paform.Assemble();
+
+               MixedBilinearForm assemblyform(&ND_fespace, &ND_fespace);
+               if (integrator == 0)
+               {
+                  if (coeffType == 2)
+                  {
+                     assemblyform.AddDomainIntegrator(new MixedVectorCurlIntegrator(*vcoeff));
+                  }
+                  else
+                  {
+                     assemblyform.AddDomainIntegrator(new MixedVectorCurlIntegrator(*coeff));
+                  }
+               }
+               else
+               {
+                  if (coeffType == 2)
+                  {
+                     assemblyform.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(*vcoeff));
+                  }
+                  else
+                  {
+                     assemblyform.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(*coeff));
+                  }
+               }
+
+               assemblyform.Assemble();
+               assemblyform.Finalize();
+               const SparseMatrix& A_explicit = assemblyform.SpMat();
+
+               Vector xin(ND_fespace.GetTrueVSize());
+               xin.Randomize();
+               Vector y_mat(ND_fespace.GetTrueVSize());
+               y_mat = 0.0;
+               Vector y_assembly(ND_fespace.GetTrueVSize());
+               y_assembly = 0.0;
+               Vector y_pa(ND_fespace.GetTrueVSize());
+               y_pa = 0.0;
+
+               paform.Mult(xin, y_pa);
+               assemblyform.Mult(xin, y_assembly);
+               A_explicit.Mult(xin, y_mat);
+
+               y_pa -= y_mat;
+               double pa_error = y_pa.Norml2();
+               std::cout << "  order: " << order
+                         << ", pa error norm: " << pa_error << std::endl;
+               REQUIRE(pa_error < 1.e-12);
+
+               y_assembly -= y_mat;
+               double assembly_error = y_assembly.Norml2();
+               std::cout << "  order: " << order
+                         << ", assembly error norm: " << assembly_error
+                         << std::endl;
+               REQUIRE(assembly_error < 1.e-12);
+
+               delete ND_fec;
+            }
+         }
+
+         delete coeff;
+         delete vcoeff;
+      }
+
+      delete mesh;
+   }
+}
+
 } // namespace pa_coeff

tests/unit/fem/test_pa_grad.cpp

@@ -0,0 +1,254 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "catch.hpp"
+#include "mfem.hpp"
+
+using namespace mfem;
+
+double compare_pa_assembly(int dim, int num_elements, int order, bool transpose)
+{
+   Mesh * mesh;
+   if (num_elements == 0)
+   {
+      if (dim == 2)
+      {
+         mesh = new Mesh("../../data/star.mesh", order);
+      }
+      else
+      {
+         mesh = new Mesh("../../data/beam-hex.mesh", order);
+      }
+   }
+   else
+   {
+      if (dim == 2)
+      {
+         mesh = new Mesh(num_elements, num_elements, Element::QUADRILATERAL, true);
+      }
+      else
+      {
+         mesh = new Mesh(num_elements, num_elements, num_elements,
+                         Element::HEXAHEDRON, true);
+      }
+   }
+   FiniteElementCollection *h1_fec = new H1_FECollection(order, dim);
+   FiniteElementCollection *nd_fec = new ND_FECollection(order, dim);
+   FiniteElementSpace h1_fespace(mesh, h1_fec);
+   FiniteElementSpace nd_fespace(mesh, nd_fec);
+
+   DiscreteLinearOperator assembled_grad(&h1_fespace, &nd_fespace);
+   assembled_grad.AddDomainInterpolator(new GradientInterpolator);
+   const int skip_zeros = 1;
+   assembled_grad.Assemble(skip_zeros);
+   assembled_grad.Finalize(skip_zeros);
+   const SparseMatrix& assembled_grad_mat = assembled_grad.SpMat();
+
+   DiscreteLinearOperator pa_grad(&h1_fespace, &nd_fespace);
+   pa_grad.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   pa_grad.AddDomainInterpolator(new GradientInterpolator);
+   pa_grad.Assemble();
+   pa_grad.Finalize();
+
+   int insize, outsize;
+   if (transpose)
+   {
+      insize = nd_fespace.GetVSize();
+      outsize = h1_fespace.GetVSize();
+   }
+   else
+   {
+      insize = h1_fespace.GetVSize();
+      outsize = nd_fespace.GetVSize();
+   }
+   Vector xv(insize);
+   Vector assembled_y(outsize);
+   Vector pa_y(outsize);
+
+   xv.Randomize();
+   if (transpose)
+   {
+      assembled_grad_mat.BuildTranspose();
+      assembled_grad_mat.MultTranspose(xv, assembled_y);
+      pa_grad.MultTranspose(xv, pa_y);
+   }
+   else
+   {
+      assembled_grad_mat.Mult(xv, assembled_y);
+      pa_grad.Mult(xv, pa_y);
+   }
+
+   if (false)
+   {
+      std::cout << "true   \tpa\n";
+      for (int i = 0; i < assembled_y.Size(); ++i)
+      {
+         std::cout << i << " : " << assembled_y(i) << "\t" << pa_y(i) << std::endl;
+      }
+   }
+
+   pa_y -= assembled_y;
+   double error = pa_y.Norml2() / assembled_y.Norml2();
+   std::cout << "dim " << dim << " ne " << num_elements << " order "
+             << order;
+   if (transpose)
+   {
+      std::cout << " T";
+   }
+   std::cout << ": error in PA gradient: " << error << std::endl;
+
+   delete h1_fec;
+   delete nd_fec;
+   delete mesh;
+
+   return error;
+}
+
+TEST_CASE("PAGradient", "[CUDA]")
+{
+   for (bool transpose : {false, true})
+   {
+      for (int dim = 2; dim < 4; ++dim)
+      {
+         for (int num_elements = 0; num_elements < 5; ++num_elements)
+         {
+            for (int order = 1; order < 5; ++order)
+            {
+               double error = compare_pa_assembly(dim, num_elements, order, transpose);
+               REQUIRE(error < 1.e-14);
+            }
+         }
+      }
+   }
+}
+
+#ifdef MFEM_USE_MPI
+
+double par_compare_pa_assembly(int dim, int num_elements, int order,
+                               bool transpose)
+{
+   int rank;
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+   int size;
+   MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+   Mesh * smesh;
+   if (dim == 2)
+   {
+      smesh = new Mesh(num_elements, num_elements, Element::QUADRILATERAL, true);
+   }
+   else
+   {
+      smesh = new Mesh(num_elements, num_elements, num_elements,
+                       Element::HEXAHEDRON, true);
+   }
+   ParMesh * mesh = new ParMesh(MPI_COMM_WORLD, *smesh);
+   delete smesh;
+   FiniteElementCollection *h1_fec = new H1_FECollection(order, dim);
+   FiniteElementCollection *nd_fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace h1_fespace(mesh, h1_fec);
+   ParFiniteElementSpace nd_fespace(mesh, nd_fec);
+
+   ParDiscreteLinearOperator assembled_grad(&h1_fespace, &nd_fespace);
+   assembled_grad.AddDomainInterpolator(new GradientInterpolator);
+   const int skip_zeros = 1;
+   assembled_grad.Assemble(skip_zeros);
+   assembled_grad.Finalize(skip_zeros);
+   HypreParMatrix * assembled_grad_mat = assembled_grad.ParallelAssemble();
+
+   ParDiscreteLinearOperator pa_grad(&h1_fespace, &nd_fespace);
+   pa_grad.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   pa_grad.AddDomainInterpolator(new GradientInterpolator);
+   pa_grad.Assemble();
+   OperatorPtr pa_grad_oper;
+   pa_grad.FormRectangularSystemMatrix(pa_grad_oper);
+
+   int insize, outsize;
+   if (transpose)
+   {
+      insize = assembled_grad_mat->Height();
+      outsize = assembled_grad_mat->Width();
+   }
+   else
+   {
+      insize = assembled_grad_mat->Width();
+      outsize = assembled_grad_mat->Height();
+   }
+   Vector xv(insize);
+   Vector assembled_y(outsize);
+   Vector pa_y(outsize);
+   assembled_y = 0.0;
+   pa_y = 0.0;
+
+   xv.Randomize();
+   if (transpose)
+   {
+      assembled_grad_mat->MultTranspose(xv, assembled_y);
+      pa_grad_oper->MultTranspose(xv, pa_y);
+   }
+   else
+   {
+      assembled_grad_mat->Mult(xv, assembled_y);
+      pa_grad_oper->Mult(xv, pa_y);
+   }
+
+   Vector error_vec(pa_y);
+   error_vec -= assembled_y;
+   // serial norms and serial error; we are enforcing equality on each processor
+   // in the test
+   double error = error_vec.Norml2() / assembled_y.Norml2();
+
+   for (int p = 0; p < size; ++p)
+   {
+      if (rank == p)
+      {
+         std::cout << "[" << rank << "]";
+         // std::cout << "pa_y.Norml2() = " << pa_y.Norml2() << std::endl;
+         // std::cout << "assembled_y.Norml2() = " << assembled_y.Norml2() << std::endl;
+         std::cout << "[par] dim " << dim << " ne " << num_elements << " order "
+                   << order;
+         if (transpose)
+         {
+            std::cout << " T";
+         }
+         std::cout << ": error in PA gradient: " << error << std::endl;
+         std::cout.flush();
+      }
+      MPI_Barrier(MPI_COMM_WORLD);
+   }
+
+   delete h1_fec;
+   delete nd_fec;
+   delete assembled_grad_mat;
+   delete mesh;
+
+   return error;
+}
+
+TEST_CASE("ParallelPAGradient", "[Parallel], [ParallelPAGradient]")
+{
+   for (bool transpose : {false, true})
+   {
+      for (int dim = 2; dim < 4; ++dim)
+      {
+         for (int num_elements = 4; num_elements < 6; ++num_elements)
+         {
+            for (int order = 1; order < 5; ++order)
+            {
+               double error = par_compare_pa_assembly(dim, num_elements, order, transpose);
+               REQUIRE(error < 1.e-14);
+            }
+         }
+      }
+   }
+}
+
+#endif

tests/unit/fem/test_pa_idinterp.cpp

@@ -0,0 +1,134 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "catch.hpp"
+#include "mfem.hpp"
+
+using namespace mfem;
+
+double compare_pa_id_assembly(int dim, int num_elements, int order,
+                              bool transpose)
+{
+   Mesh * mesh;
+   if (num_elements == 0)
+   {
+      if (dim == 2)
+      {
+         mesh = new Mesh("../../data/star.mesh", order);
+      }
+      else
+      {
+         mesh = new Mesh("../../data/beam-hex.mesh", order);
+
+         // Transform mesh vertices to test without alignment with coordinate axes.
+         for (int i=0; i<mesh->GetNV(); ++i)
+         {
+            double *v = mesh->GetVertex(i);
+            const double yscale = 1.0 + v[1];
+            const double zscale = 1.0 + v[2];
+            v[0] *= zscale;
+            v[1] *= zscale;
+            v[2] *= yscale;
+         }
+      }
+   }
+   else
+   {
+      if (dim == 2)
+      {
+         mesh = new Mesh(num_elements, num_elements, Element::QUADRILATERAL, true);
+      }
+      else
+      {
+         mesh = new Mesh(num_elements, num_elements, num_elements,
+                         Element::HEXAHEDRON, true);
+      }
+   }
+   FiniteElementCollection *h1_fec = new H1_FECollection(order, dim);
+   FiniteElementCollection *nd_fec = new ND_FECollection(order, dim);
+   FiniteElementSpace h1_fespace(mesh, h1_fec, dim);
+   FiniteElementSpace nd_fespace(mesh, nd_fec);
+
+   DiscreteLinearOperator assembled_id(&h1_fespace, &nd_fespace);
+   assembled_id.AddDomainInterpolator(new IdentityInterpolator);
+   const int skip_zeros = 1;
+   assembled_id.Assemble(skip_zeros);
+   assembled_id.Finalize(skip_zeros);
+   const SparseMatrix& assembled_id_mat = assembled_id.SpMat();
+
+   DiscreteLinearOperator pa_id(&h1_fespace, &nd_fespace);
+   pa_id.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   pa_id.AddDomainInterpolator(new IdentityInterpolator);
+   pa_id.Assemble();
+   pa_id.Finalize();
+
+   int insize, outsize;
+   if (transpose)
+   {
+      insize = nd_fespace.GetVSize();
+      outsize = h1_fespace.GetVSize();
+   }
+   else
+   {
+      insize = h1_fespace.GetVSize();
+      outsize = nd_fespace.GetVSize();
+   }
+   Vector x(insize);
+   Vector assembled_y(outsize);
+   Vector pa_y(outsize);
+
+   x.Randomize();
+   if (transpose)
+   {
+      assembled_id_mat.BuildTranspose();
+      assembled_id_mat.MultTranspose(x, assembled_y);
+      pa_id.MultTranspose(x, pa_y);
+   }
+   else
+   {
+      assembled_id.Mult(x, assembled_y);
+      pa_id.Mult(x, pa_y);
+   }
+
+   pa_y -= assembled_y;
+   double error = pa_y.Norml2() / assembled_y.Norml2();
+   std::cout << "dim " << dim << " ne " << num_elements << " order "
+             << order;
+   if (transpose)
+   {
+      std::cout << " T";
+   }
+   std::cout << ": error in PA identity: " << error << std::endl;
+
+   delete h1_fec;
+   delete nd_fec;
+   delete mesh;
+
+   return error;
+}
+
+TEST_CASE("PAIdentityInterp", "[CUDA]")
+{
+   for (bool transpose : {false, true})
+   {
+      for (int dim = 2; dim < 4; ++dim)
+      {
+         for (int num_elements = 0; num_elements < 5; ++num_elements)
+         {
+            for (int order = 1; order < 5; ++order)
+            {
+               double error = compare_pa_id_assembly(dim, num_elements, order, transpose);
+               REQUIRE(error < 1.0e-14);
+            }
+         }
+      }
+   }
+}