testing dpg residual with cholesky factors

examples/dpg_tests/acoustics/fosls.cpp

@@ -0,0 +1,294 @@
+//                 MFEM FOSLS acoustics Example
+//
+// Compile with: make fosls
+//
+// Definite/Indefinite Helmholtz
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// FOSLS:
+//       minimize  1/2(||∇p - ω u||^2 + ||-∇⋅u ± ω p - f||^2)
+
+// (p,u) ∈ H^1(Ω) × H(div,Ω)
+// -------------------------------------------------------------------
+// |   |            p           |            u             |   RHS    | 
+// -------------------------------------------------------------------
+// | q |  (∇ p,∇ q) + ω^2(p,q)  | ∓ ω (∇⋅u,q) - ω (u, ∇ q) | ± ω(f,q) |
+// |   |                        |                          |          |
+// | v | ∓ ω (p,∇⋅v) - ω (∇ p,v)|  (∇⋅u,∇⋅v) + ω^2 (u,v)   | -(f,∇⋅v) |
+
+// where (q,v) ∈ H^1(Ω) × H(div,Ω)
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int sr = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
+   FiniteElementCollection *RTfec = new RT_FECollection(order-1, dim);
+   FiniteElementSpace * H1fes = new FiniteElementSpace(&mesh, H1fec);
+   FiniteElementSpace * RTfes = new FiniteElementSpace(&mesh, RTfec);
+
+   Array<FiniteElementSpace *> fespaces(2);
+   fespaces[0] = H1fes;
+   fespaces[1] = RTfes;
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   BlockBilinearForm a(fespaces);
+   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
+   cout << "H1 fespace = " << H1fes->GetTrueVSize() << endl;
+   cout << "RT fespace = " << RTfes->GetTrueVSize() << endl;
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg2(omega*omega);
+
+
+   Array2D<BilinearFormIntegrator * > blfi(2,2);
+
+   // blfi(0,0) = (∇ p,∇ q) + ω^2(p,q)
+   SumIntegrator * integ00 = new SumIntegrator();
+   integ00->AddIntegrator(new DiffusionIntegrator(one));
+   integ00->AddIntegrator(new MassIntegrator(omeg2));
+   blfi(0,0) = integ00;
+
+   // blfi(0,1) = ∓ ω (∇⋅u,q) - ω (u, ∇ q)
+   SumIntegrator * integ01 = new SumIntegrator();
+#ifdef DEFINITE
+   // -ω (∇⋅u,q)
+   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+#else   
+   // ω (∇⋅u,q)
+   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+#endif
+   // - ω (u, ∇ q)
+   integ01->AddIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   blfi(0,1) = integ01;
+
+   // blfi(1,0) = ∓ ω (p,∇⋅v) - ω (∇ p,v)
+   SumIntegrator * integ10 = new SumIntegrator();
+#ifdef DEFINITE
+   // - ω (p,∇⋅v)
+   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+#else
+   // ω (p,∇⋅v)
+   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+#endif
+   // - ω (∇ p,v)
+   integ10->AddIntegrator(new MixedVectorGradientIntegrator(negomeg));
+   blfi(1,0) = integ10;
+
+   // blfi(1,1) = (∇⋅u,∇⋅v) + ω^2 (u,v)
+   SumIntegrator * integ11 = new SumIntegrator();
+   integ11->AddIntegrator(new DivDivIntegrator(one));
+   integ11->AddIntegrator(new VectorFEMassIntegrator(omeg2));
+   blfi(1,1) = integ11;
+
+
+   BlockLinearForm b(fespaces);
+   Array<LinearFormIntegrator * > lfi(2);
+   // ± ω (f,q)
+   FunctionCoefficient f_rhs(rhs_func);
+#ifdef DEFINITE  
+   ProductCoefficient w_f(omeg,f_rhs);
+#else   
+   ProductCoefficient w_f(negomeg,f_rhs);
+#endif   
+   // lfi[0] = new DomainLFIntegrator(w_f);
+   lfi[0] = new DomainLFIntegrator(w_f);
+
+
+   //  -(f,∇⋅v)
+   ProductCoefficient neg_f(negone,f_rhs);
+   // lfi[1] = new VectorFEDomainLFDivIntegrator(f_rhs);
+   lfi[1] = new VectorFEDomainLFDivIntegrator(neg_f);
+
+   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
+   integ->SetIntegrators(blfi);
+   a.AddDomainIntegrator(integ);
+   a.Assemble();
+
+   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
+   lininteg->SetIntegrators(lfi);
+   b.AddDomainIntegrator(lininteg);
+   b.Assemble();
+
+   int size = 0;
+   for (int i = 0; i<fespaces.Size(); i++)
+   {
+      size += fespaces[i]->GetVSize();
+   }
+
+   Vector x(size);
+   x = 0.0;
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   GridFunction p_gf, u_gf;   
+   GridFunction pex_gf(H1fes);   
+
+   p_gf.MakeRef(H1fes,x,0);
+   // p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+   p_gf.ProjectCoefficient(p_ex);
+   pex_gf.ProjectCoefficient(p_ex);
+
+   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
+   u_gf = 0.;
+
+
+   OperatorPtr A;
+   Vector X,B;
+   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
+
+   GSSmoother M((SparseMatrix&)(*A));
+   CGSolver cg;
+   cg.SetRelTol(1e-10);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X,b,x);
+
+   p_gf.MakeRef(H1fes,x,0);
+   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << p_gf <<
+                "window_title 'Numerical p' "
+                << flush;
+      // socketstream sols_sock(vishost, visport);
+      // sols_sock.precision(8);
+      // sols_sock << "solution\n" << mesh << u_gf <<
+               //  "window_title 'Numerical sigma' "
+               //  << flush;
+
+      socketstream solex_sock(vishost, visport);
+      solex_sock.precision(8);
+      solex_sock << "solution\n" << mesh << pex_gf <<
+                    "window_title 'Exact p' "
+                    << flush;          
+   }
+
+   return 0;
+}
+
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/dpg_tests/acoustics/makefile

@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = fosls uw_dpg  strong_dpg
+PAR_EXAMPLES = uw_dpgp
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:

examples/dpg_tests/acoustics/strong_dpg.cpp

@@ -0,0 +1,271 @@
+//                 MFEM DPG_strong acoustics Example
+//
+// Compile with: make strong_dpg
+//
+// Definite/Indefinite Helmholtz
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// Strong DPG formulation
+// (p,u) ∈ H^1(Ω) × H(div,Ω)
+// 
+//   (∇ p, v) - ω (u,v) = 0,     in Ω, ∀ v ∈ (L^2)^dim
+//  -(∇⋅u, q) ± ω (p,q) = (f,q), in Ω, ∀ q ∈  L^2 
+//                    p = p_0, in ∂Ω
+// 
+// ------------------------------------
+// |   |      p    |     u    |  RHS  | 
+// ------------------------------------
+// | q | ± ω (p,q) | -(∇⋅u,q) | (f,q) |
+// |   |           |          |       |
+// | v | (∇ p, v)  | -ω (u,v) |       |
+
+// where (q,v) ∈ L^2  × (L^2)^dim
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");                  
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh.UniformRefinement();
+   }
+
+   // Define spaces
+   // H1 space for p
+   FiniteElementCollection *p_fec = new H1_FECollection(order, dim);
+   FiniteElementSpace * p_fes = new FiniteElementSpace(&mesh, p_fec);
+
+   // H(div) for u
+   FiniteElementCollection *u_fec = new RT_FECollection(order-1, dim);
+   FiniteElementSpace * u_fes = new FiniteElementSpace(&mesh, u_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new L2_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new L2_FECollection(test_order-1, dim);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->SetTestFECollVdim(1,dim);
+   
+   a->StoreMatrices(true);
+
+//  ± ω (p, q)
+#ifdef DEFINITE
+//  ω (p, q)
+   a->AddTrialIntegrator(new MassIntegrator(omeg),0,0);
+#else
+// -ω (p, q)
+   a->AddTrialIntegrator(new MassIntegrator(negomeg),0,0);
+#endif
+
+// -(∇⋅u, q) 
+   a->AddTrialIntegrator(new MixedScalarDivergenceIntegrator(negone),1,0);
+
+// -ω (u,v)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(negomeg),1,1);
+
+// (∇ p, v)
+   a->AddTrialIntegrator(new GradientIntegrator(one),0,1);
+
+// (v,δv)
+   a->AddTestIntegrator(new VectorMassIntegrator(one),1,1);
+    
+// (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   GridFunction p_gf, u_gf;   
+   GridFunction pex_gf(p_fes);   
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = p_fes->GetVSize();
+   offsets[2] = u_fes->GetVSize();
+   offsets.PartialSum();
+   BlockVector x(offsets);
+   x = 0.0;
+
+   p_gf.MakeRef(p_fes,x.GetBlock(0));
+   p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+
+   u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = Ah.As<BlockMatrix>();
+
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream p_out;
+      socketstream u_out;
+      p_out.open(vishost, visport);
+      u_out.open(vishost, visport);
+      p_out.precision(8);
+      p_out << "solution\n" << mesh << p_gf <<
+               "window_title 'Numerical p' "
+               << flush;
+
+      u_out.precision(8);
+      u_out << "solution\n" << mesh << u_gf <<
+            "window_title 'Numerical flux' "
+            << flush;
+   }
+
+   return 0;
+}
+
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}

examples/dpg_tests/acoustics/uw_dpg.cpp

@@ -0,0 +1,546 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make uw_dpg
+//
+// ./uw_dpg -m ../../../data/inline-quad.mesh -rnum 40 -theta 0.7 -prob 1 -graph-norm -ref 40 -o 3
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂ = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := -u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p);
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+double divu_exact(const Vector &x);
+double hatp_exact(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");    
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");                                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+
+   // ± ω (p,q)
+#ifdef DEFINITE
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
+#else
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
+#endif   
+
+// (u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
+
+// - ω (u,v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+// test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -ω (∇q,δv)
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
+      // -ω (v,δq)
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
+
+#ifdef DEFINITE
+      // - ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // - ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
+#else
+      // ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
+      // ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+#endif
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+
+   FunctionCoefficient hatpex(hatp_exact);
+   FunctionCoefficient pex(p_exact);
+   VectorFunctionCoefficient uex(dim,u_exact);
+   Array<int> elements_to_refine;
+   GridFunction hatp_gf;
+
+
+   socketstream p_out;
+   // socketstream u_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out.open(vishost, visport);
+      // u_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+   // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
+      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-8);
+      cg.SetMaxIter(20000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+      
+      GridFunction p_gf;
+      p_gf.MakeRef(p_fes,x.GetBlock(0));
+
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+      GridFunction pex_gf(p_fes);
+      GridFunction uex_gf(u_fes);
+      pex_gf.ProjectCoefficient(pex);
+      uex_gf.ProjectCoefficient(uex);
+
+
+      // Error
+      int dofs = X.Size();
+      double p_err = p_gf.ComputeL2Error(pex);
+      double p_norm = uex_gf.ComputeL2Error(zero);
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = u_gf.ComputeL2Error(vzero);
+
+      double L2Error = sqrt(p_err*p_err + u_err*u_err);
+      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::endl;
+
+      if (visualization)
+      {
+         p_out.precision(8);
+         p_out << "solution\n" << mesh << p_gf <<
+                  "window_title 'Numerical presure' "
+                  << flush;
+
+         // u_out.precision(8);
+         // u_out << "solution\n" << mesh << u_gf <<
+         //       "window_title 'Numerical velocity' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   double p, d2p;
+   Vector dp;
+   acoustics_solution(x,p,dp,d2p);
+   return p;
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   double p, d2p;
+   acoustics_solution(x,p,u,d2p);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   double p, d2p;
+   Vector dp;
+   acoustics_solution(x,p,dp,d2p);
+   return d2p/omega;
+}
+
+double hatp_exact(const Vector & X)
+{
+   return p_exact(X);
+}
+
+void hatu_exact(const Vector & X, Vector & hatu)
+{
+   u_exact(X,hatu);
+   hatu *= -1.;
+}
+
+void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p)
+{
+   dp.SetSize(X.Size());
+   switch (prob)
+   {
+   case plane_wave:
+   {
+      p = sin(omega*X.Sum());
+      dp = omega * cos(omega * X.Sum());
+      d2p = -dim * omega * omega * sin(omega*X.Sum());
+   }
+      break;
+   default:
+   {
+      double rk = omega;
+      double alpha = 45 * M_PI/180.;
+      double sina = sin(alpha); 
+      double cosa = cos(alpha);
+      // shift the origin
+      double xprim=X(0) + 0.1; 
+      double yprim=X(1) + 0.1;
+
+      double  x = xprim*sina - yprim*cosa;
+      double  y = xprim*cosa + yprim*sina;
+      double  dxdxprim = sina, dxdyprim = -cosa;
+      double  dydxprim = cosa, dydyprim =  sina;
+      //wavelength
+      double rl = 2.*M_PI/rk;
+
+      // beam waist radius
+      double w0 = 0.05;
+
+      // function w
+      double fact = rl/M_PI/(w0*w0);
+      double aux = 1. + (fact*y)*(fact*y);
+
+      double w = w0*sqrt(aux);
+      double dwdy = w0*fact*fact*y/sqrt(aux);
+      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+      double phi0 = atan(fact*y);
+      double dphi0dy = cos(phi0)*cos(phi0)*fact;
+      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+      double r = y + 1./y/(fact*fact);
+      double drdy = 1. - 1./(y*y)/(fact*fact);
+      double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+      // pressure
+      complex<double> zi = complex<double>(0., 1.);
+      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
+
+      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
+      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+      complex<double> zd2edydx = zd2edxdy;
+      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+      double pf = pow(2.0/M_PI/(w*w),0.25);
+      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+      complex<double> zp = pf*exp(ze);
+      complex<double> zdpdx = zp*zdedx;
+      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+      p = zp.real();
+      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim).real();
+      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim).real();
+
+      d2p = (  (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+            + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim ).real();
+   }
+   break;
+   }
+
+}

examples/dpg_tests/acoustics/uw_dpgp.cpp

@@ -0,0 +1,507 @@
+//                   MFEM Ultraweak DPG MPI acoustics (Helmholtz) example
+//
+// Compile with: make uw_dpgp
+//
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+//
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+// 
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂ = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := -u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+double hatp_exact(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");  
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   Array<ParFiniteElementSpace * > trial_fes; 
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   Array<FiniteElementCollection * > test_fec; 
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+
+   // Integrators
+
+   // ± ω (p,q)
+#ifdef DEFINITE
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
+#else
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
+#endif   
+
+// (u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
+
+// - ω (u,v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   // test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -ω (∇q,δv)
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
+      // -ω (v,δq)
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
+
+#ifdef DEFINITE
+      // - ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // - ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
+#else
+      // ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
+      // ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+#endif
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
+   }   
+
+   // RHS
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+
+   FunctionCoefficient hatpex(hatp_exact);
+   FunctionCoefficient pex(p_exact);
+   VectorFunctionCoefficient uex(dim,u_exact);
+   Array<int> elements_to_refine;
+   ParGridFunction hatp_gf;
+
+
+
+
+   socketstream p_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out.open(vishost, visport);
+   }
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << " Refinement |" 
+               << "    Dofs    |" 
+               << "  L2 Error  |" 
+               << " Relative % |" 
+               << "  Rate  |" 
+               << "  Residual  |" 
+               << "  Rate  |" << endl;
+      mfem::out << " --------------------"      
+               <<  "-------------------"    
+               <<  "-------------------"    
+               <<  "-------------------" << endl;   
+   }
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
+      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+
+      HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(2,2));
+      amg0->SetPrintLevel(0);
+      amg1->SetPrintLevel(0);
+      amg2->SetPrintLevel(0);
+      amg0->SetRelaxType(16);
+      amg1->SetRelaxType(16);
+      amg2->SetRelaxType(16);
+
+      M->SetDiagonalBlock(0,amg0);
+      M->SetDiagonalBlock(1,amg1);
+      M->SetDiagonalBlock(2,amg2);
+      // for (int i = 0; i < 3; i++)
+      // {
+      //    MUMPSSolver * mumps = new MUMPSSolver;
+      //    mumps->SetOperator(A->GetBlock(i,i));
+      //    M->SetDiagonalBlock(i,mumps);
+      // }
+
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(3,3), hatu_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(3,3), hatu_fes);
+      }
+      M->SetDiagonalBlock(3,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-10);
+      cg.SetMaxIter(20000);
+      cg.SetPrintLevel(-1);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+
+      ParGridFunction p_gf;
+      p_gf.MakeRef(p_fes,x.GetBlock(0));
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+
+      ParGridFunction pex_gf(p_fes);
+      ParGridFunction uex_gf(u_fes);
+      pex_gf.ProjectCoefficient(pex);
+      uex_gf.ProjectCoefficient(uex);
+
+      int dofs = p_fes->GlobalTrueVSize()
+               + u_fes->GlobalTrueVSize()
+               + hatp_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize();
+
+      double p_err = p_gf.ComputeL2Error(pex);
+      double p_norm = pex_gf.ComputeL2Error(zero);
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(vzero);
+
+      double L2Error = sqrt(p_err*p_err + u_err*u_err);
+      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+         std::ios oldState(nullptr);
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(11) << i << " | " 
+                   << std::setw(10) <<  dof0 << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::scientific <<  err0 << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                   << std::setprecision(2) 
+                   << std::setw(6) << std::fixed << rate_err << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::scientific <<  res0 << " | " 
+                   << std::setprecision(2) 
+                   << std::setw(6) << std::fixed << rate_res << " | " 
+                   << std::setprecision(5) 
+                   << std::scientific 
+                   << std::endl;
+      }   
+
+      if (visualization)
+      {
+         p_out << "parallel " << num_procs << " " << myid << "\n";
+         p_out.precision(8);
+         p_out << "solution\n" << pmesh << p_gf <<
+                  "window_title 'Numerical pressure' "
+                  << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}
+
+double hatp_exact(const Vector & X)
+{
+   return p_exact(X);
+}
+
+void hatu_exact(const Vector & X, Vector & hatu)
+{
+   u_exact(X,hatu);
+   hatu *= -1.;
+}

examples/dpg_tests/convection-diffusion/makefile

@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/convection-diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = uw_dpg
+PAR_EXAMPLES = uw_dpgp
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:

examples/dpg_tests/convection-diffusion/uw_dpg.cpp

@@ -0,0 +1,649 @@
+//                     MFEM Ultraweak DPG example
+//
+// Compile with: make uw_dpg
+//
+// sample runs 
+// ./uw_dpg  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
+//     - εΔu + ∇⋅(βu) = f,   in Ω
+//                  u = u_0, on ∂Ω
+
+// First Order System
+
+//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
+//        1/ε σ - ∇u  = 0,   in Ω
+//                  u = u_0, on ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
+//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
+//                                        û = u_0  on ∂Ω 
+
+// Note: 
+// f̂ := βu - σ
+// û := -u
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
+// -------------------------------------------------------------
+// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
+// |   |           |           |           |          |         |
+// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
+
+// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   polynomial,
+   EJ,   
+   general  
+};
+
+enum test_norm_type
+{
+   standard,   
+   adjoint_graph,
+   robust  
+};
+
+prob_type prob;
+test_norm_type test_norm;
+Vector beta;
+double epsilon;
+// Function returns the solution u, and gradient du and the Laplacian d2u
+void solution(const Vector & x, double & u, Vector & du, double & d2u);
+double exact_u(const Vector & X);
+void exact_sigma(const Vector & X, Vector & sigma);
+double exact_hatu(const Vector & X);
+void exact_hatf(const Vector & X, Vector & hatf);
+double f_exact(const Vector & X);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool visualization = true;
+   int iprob = 0;
+   int itest_norm = 0;
+   double theta = 0.7;
+   epsilon = 1e0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&epsilon, "-eps", "--epsilon",
+                  "Epsilon coefficient");               
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");        
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                             
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: polynomial, 1: EJ ,2: General");           
+   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
+                  " 0: Standard, 1: Adjoint Graph, 2: Robust");       
+   args.AddOption(&beta, "-beta", "--beta",
+                  "Vector Coefficient beta");                                              
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 2) { iprob = 2; }
+   prob = (prob_type)iprob;
+   test_norm = (test_norm_type)itest_norm;
+
+   if (prob == prob_type::EJ)
+   {
+      mesh_file = "../../../data/inline-quad.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (beta.Size() == 0)
+   {
+      beta.SetSize(dim);
+      beta[0] = 1.;
+      beta[1] = 0.;
+   }
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatf_fes = new FiniteElementSpace(&mesh,hatf_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient eps(epsilon);
+   ConstantCoefficient eps1(1./epsilon);
+   ConstantCoefficient negeps1(-1./epsilon);
+   ConstantCoefficient eps2(1/(epsilon*epsilon));
+
+   ConstantCoefficient negeps(-epsilon);
+   VectorConstantCoefficient betacoeff(beta);
+   Vector negbeta = beta;
+   negbeta.Neg();
+
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+
+
+   DenseMatrix bbt(beta.Size());
+   MultVVt(beta, bbt); 
+   MatrixConstantCoefficient bbtcoeff(bbt);
+
+
+   VectorConstantCoefficient negbetacoeff(negbeta);
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatf_fes);
+   test_fec.Append(v_fec);
+   test_fec.Append(tau_fec);
+
+
+   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
+   FiniteElementSpace *coeff_fes = new FiniteElementSpace(&mesh,coeff_fec);
+   GridFunction c1_gf, c2_gf;
+   GridFunctionCoefficient c1_coeff(&c1_gf);
+   GridFunctionCoefficient c2_coeff(&c2_gf);
+
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //-(βu , ∇v)
+   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
+   
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // (u ,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
+
+   // 1/ε (σ,τ)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // <f̂ ,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   switch (test_norm)
+   {
+      case standard:
+      {   
+         // (∇v,∇δv)
+         mfem::out << "\n Test norm: Standard" << endl;
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+      }
+      break;
+      case adjoint_graph:
+      {
+         mfem::out << "\n Test norm: Adjoint Graph" << endl;
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+         // 1/ε^2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
+         // 1/ε (∇v, δτ)
+         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
+         // - (β ⋅ ∇v,∇⋅δτ)  
+         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
+         // 1/ε (τ,∇δv) 
+         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
+         // -(β ∇⋅τ ,∇⋅δv)  
+         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
+      }
+      break;
+      default:
+      {
+         mfem::out << "\n Test norm: Robust" << endl;
+         c1_gf.SetSpace(coeff_fes);
+         c2_gf.SetSpace(coeff_fes);
+         Array<int> dofs;
+         for (int i =0; i < mesh.GetNE(); i++)
+         {
+            double volume = mesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            // double c2 = 1.;
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+         // c1 (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
+         // ε (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // c2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+      }
+      break;
+   }
+
+
+   FunctionCoefficient f(f_exact);
+   // if (prob != prob_type::EJ)
+   // {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+   // }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   VectorFunctionCoefficient hatfex(dim,exact_hatf);
+   Array<int> elements_to_refine;
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
+   GridFunction hatu_gf;
+   GridFunction hatf_gf;
+
+   // socketstream uex_out;
+   socketstream u_out;
+   // socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // uex_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+   for (int i = 0; i<=ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list_uhat;
+      Array<int> ess_tdof_list_fhat;
+      Array<int> ess_bdr_uhat;
+      Array<int> ess_bdr_fhat;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr_uhat.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr_fhat.SetSize(mesh.bdr_attributes.Max());
+         // ess_bdr_uhat = 1;
+         // ess_bdr_fhat = 0;
+         ess_bdr_uhat = 0;
+         ess_bdr_fhat = 1;
+         ess_bdr_uhat[1] = 1;
+         ess_bdr_fhat[1] = 0;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
+         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
+      }
+
+      // shift the ess_tdofs
+      int n = ess_tdof_list_uhat.Size();
+      int m = ess_tdof_list_fhat.Size();
+      Array<int> ess_tdof_list(n+m);
+      for (int i = 0; i < n; i++)
+      {
+         ess_tdof_list[i] = ess_tdof_list_uhat[i] 
+                          + u_fes->GetTrueVSize() 
+                          + sigma_fes->GetTrueVSize();
+      }
+      for (int i = 0; i < m; i++)
+      {
+         ess_tdof_list[i+n] = ess_tdof_list_fhat[i] 
+                            + u_fes->GetTrueVSize() 
+                            + sigma_fes->GetTrueVSize()
+                            + hatu_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatf_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+
+      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
+
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
+      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(200000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }   
+
+      GridFunction uex_gf(u_fes);
+      uex_gf.ProjectCoefficient(uex);
+
+      GridFunction sigmaex_gf(sigma_fes);
+      sigmaex_gf.ProjectCoefficient(sigmaex);
+
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      GridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      int dofs = X.Size();
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(zero);
+      // mfem::out << "u_err = " << u_err << endl;
+      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
+      double sigma_norm = sigmaex_gf.ComputeL2Error(vzero);
+      // mfem::out << "sigma_err = " << sigma_err << endl;
+      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
+      double L2norm = sqrt(u_norm * u_norm + sigma_norm * sigma_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::endl;
+
+
+      if (visualization)
+      {
+         // uex_out.precision(8);
+         // uex_out << "solution\n" << mesh << uex_gf <<
+         //          "window_title 'Exact u' "
+         //          << flush;
+         u_out.precision(8);
+         u_out << "solution\n" << mesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << mesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+      if (test_norm == test_norm_type::robust)
+      {
+         coeff_fes->Update();
+         c1_gf.Update();
+         c2_gf.Update();
+         Array<int> dofs;
+         for (int i = 0; i < mesh.GetNE(); i++)
+         {
+            double volume = mesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            // double c2 = 1.;
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+      }
+
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatf_fes;
+   delete hatf_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case polynomial:
+      {
+         int n=2;
+         int m=2;
+         u = pow(x,n)*pow(y,m);
+         du[0] = n * pow(x,n-1) * pow(y,m);
+         du[1] = m * pow(x,n) * pow(y,m-1);
+         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
+             + m * (m-1) * pow(x,n) * pow(y,m-2);
+      }
+      break;
+      case EJ:
+      {
+         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
+         double r1 = (1. + alpha) / (2.*epsilon);
+         double r2 = (1. - alpha) / (2.*epsilon);
+         double denom = exp(-r2) - exp(-r1);
+
+
+         double g1 = exp(r2*(x-1.));
+         double g1_x = r2*g1;
+         double g1_xx = r2*g1_x;
+         double g2 = exp(r1*(x-1.));
+         double g2_x = r1*g2;
+         double g2_xx = r1*g2_x;
+         double g = g1-g2;
+         double g_x = g1_x - g2_x;
+         double g_xx = g1_xx - g2_xx;
+
+
+         u = g * cos(M_PI * y)/denom;
+         double u_x = g_x * cos(M_PI * y)/denom;
+         double u_xx = g_xx * cos(M_PI * y)/denom;
+         double u_y = -M_PI * g * sin(M_PI*y)/denom;
+         double u_yy = -M_PI * M_PI * u;
+         du[0] = u_x;
+         du[1] = u_y;
+         d2u = u_xx + u_yy;
+
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ε ∇ u
+   sigma = du;
+   sigma *= epsilon;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return -exact_u(X);
+}
+
+void exact_hatf(const Vector & X, Vector & hatf)
+{
+   Vector sigma;
+   exact_sigma(X,sigma);
+   double u = exact_u(X);
+   hatf.SetSize(X.Size());
+   for (int i = 0; i<hatf.Size(); i++)
+   {
+      hatf[i] = beta[i] * u - sigma[i];
+   }
+}
+
+double f_exact(const Vector & X)
+{
+   // f = - εΔu + ∇⋅(βu)
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+
+   double s = 0;
+   for (int i = 0; i<du.Size(); i++)
+   {
+      s += beta[i] * du[i];
+   }
+   return -epsilon * d2u + s;
+}

examples/dpg_tests/convection-diffusion/uw_dpgp.cpp

@@ -0,0 +1,698 @@
+//                                MFEM Ultraweak DPG example
+//
+// Compile with: make  uw_dpgp
+//
+// sample runs 
+// mpirun -np 6 ./uw_dpgp  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
+//     - εΔu + ∇⋅(βu) = f,   in Ω
+//                  u = u_0, on ∂Ω
+
+// First Order System
+
+//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
+//        1/ε σ - ∇u  = 0,   in Ω
+//                  u = u_0, on ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, f̂ ∈ H^-1/2  
+// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
+//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
+//                                        û = u_0  on ∂Ω 
+
+// Note: 
+// f̂ := βu - σ
+// û := -u
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
+// -------------------------------------------------------------
+// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
+// |   |           |           |           |          |         |
+// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
+
+// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   polynomial,
+   EJ,   
+   general  
+};
+
+enum test_norm_type
+{
+   standard,   
+   adjoint_graph,
+   robust  
+};
+
+prob_type prob;
+test_norm_type test_norm;
+Vector beta;
+double epsilon;
+// Function returns the solution u, and gradient du and the Laplacian d2u
+void solution(const Vector & x, double & u, Vector & du, double & d2u);
+double exact_u(const Vector & X);
+void exact_sigma(const Vector & X, Vector & sigma);
+double exact_hatu(const Vector & X);
+void exact_hatf(const Vector & X, Vector & hatf);
+double f_exact(const Vector & X);
+
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool visualization = true;
+   int iprob = 0;
+   int itest_norm = 0;
+   double theta = 0.7;
+   bool static_cond = false;
+   epsilon = 1e0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&epsilon, "-eps", "--epsilon",
+                  "Epsilon coefficient");               
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                           
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");                         
+   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
+                  " 0: Standard, 1: Adjoint Graph, 2: Robust");     
+   args.AddOption(&beta, "-beta", "--beta",
+                  "Vector Coefficient beta");          
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                            
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (iprob > 2) { iprob = 2; }
+   prob = (prob_type)iprob;
+
+   test_norm = (test_norm_type)itest_norm;
+
+   if (prob == prob_type::EJ)
+   {
+      mesh_file = "../../../data/inline-quad.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (beta.Size() == 0)
+   {
+      beta.SetSize(dim);
+      beta[0] = 1.;
+      beta[1] = 0.;
+   }
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatf_fes = new ParFiniteElementSpace(&pmesh,hatf_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient eps(epsilon);
+   ConstantCoefficient eps1(1./epsilon);
+   ConstantCoefficient negeps1(-1./epsilon);
+   ConstantCoefficient eps2(1/(epsilon*epsilon));
+
+   ConstantCoefficient negeps(-epsilon);
+   VectorConstantCoefficient betacoeff(beta);
+   Vector negbeta = beta;
+   negbeta.Neg();
+
+   DenseMatrix bbt(beta.Size());
+   MultVVt(beta, bbt); 
+   MatrixConstantCoefficient bbtcoeff(bbt);
+
+
+   VectorConstantCoefficient negbetacoeff(negbeta);
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatf_fes);
+   test_fec.Append(v_fec);
+   test_fec.Append(tau_fec);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //-(βu , ∇v)
+   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
+   
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // (u ,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
+
+   // 1/ε (σ,τ)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // <f̂ ,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
+   ParFiniteElementSpace *coeff_fes = new ParFiniteElementSpace(&pmesh,coeff_fec);
+   ParGridFunction c1_gf, c2_gf;
+   GridFunctionCoefficient c1_coeff(&c1_gf);
+   GridFunctionCoefficient c2_coeff(&c2_gf);
+
+   switch (test_norm)
+   {
+      case standard:
+      {   
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Standard" << endl;
+         }
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+      }
+      break;
+      case adjoint_graph:
+      {
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Adjoint Graph" << endl;
+         }
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+         // 1/ε^2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
+         // 1/ε (∇v, δτ)
+         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
+         // - (β ⋅ ∇v,∇⋅δτ)  
+         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
+         // 1/ε (τ,∇δv) 
+         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
+         // -(β ∇⋅τ ,∇⋅δv)  
+         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
+      }
+      break;
+      default:
+      {
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Robust" << endl;
+         }
+         c1_gf.SetSpace(coeff_fes);
+         c2_gf.SetSpace(coeff_fes);
+         Array<int> dofs;
+         for (int i =0; i < pmesh.GetNE(); i++)
+         {
+            double volume = pmesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+         // c1 (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
+         // ε (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // c2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+      }
+      break;
+   }
+
+
+   FunctionCoefficient f(f_exact);
+   // if (prob != prob_type::EJ)
+   // {
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+   // }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   VectorFunctionCoefficient hatfex(dim,exact_hatf);
+   Array<int> elements_to_refine;
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
+
+   ParGridFunction hatu_gf;
+   ParGridFunction hatf_gf;
+
+   // socketstream uex_out;
+   socketstream u_out;
+   // socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // uex_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << " Refinement |" 
+               << "    Dofs    |" 
+               << "  L2 Error  |" 
+               << "  Rate  |" 
+               << "  Residual  |" 
+               << "  Rate  |" << endl;
+      mfem::out << " --------------------"      
+               <<  "-------------------"    
+               <<  "-------------------"    
+               <<  "-------------------" << endl;   
+   }
+
+
+   for (int i = 0; i<=ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list_uhat;
+      Array<int> ess_tdof_list_fhat;
+      Array<int> ess_bdr_uhat;
+      Array<int> ess_bdr_fhat;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr_uhat.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr_fhat.SetSize(pmesh.bdr_attributes.Max());
+         // ess_bdr_uhat = 1;
+         // ess_bdr_fhat = 0;
+         ess_bdr_uhat = 0;
+         ess_bdr_fhat = 1;
+         ess_bdr_uhat[1] = 1;
+         ess_bdr_fhat[1] = 0;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
+         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
+      }
+
+      // shift the ess_tdofs
+      int n = ess_tdof_list_uhat.Size();
+      int m = ess_tdof_list_fhat.Size();
+      Array<int> ess_tdof_list(n+m);
+      for (int i = 0; i < n; i++)
+      {
+         ess_tdof_list[i] = ess_tdof_list_uhat[i] 
+                          + u_fes->GetTrueVSize() 
+                          + sigma_fes->GetTrueVSize();
+      }
+      for (int i = 0; i < m; i++)
+      {
+         ess_tdof_list[i+n] = ess_tdof_list_fhat[i] 
+                            + u_fes->GetTrueVSize() 
+                            + sigma_fes->GetTrueVSize()
+                            + hatu_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatf_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
+
+      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
+      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip = 2;
+      }
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      M->SetDiagonalBlock(skip,amg2);
+
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-6);
+      cg.SetMaxIter(200000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max(); 
+
+      double gresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&gresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      gresidual = sqrt(gresidual);
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      ParGridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      int dofs = u_fes->GlobalTrueVSize()
+               + sigma_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize()
+               + hatf_fes->GlobalTrueVSize();
+
+      double u_err = u_gf.ComputeL2Error(uex);
+      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
+      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/gresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = gresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(11) << i << " | " 
+                  << std::setw(10) <<  dof0 << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  err0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_err << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  res0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_res << " | " 
+                  << std::resetiosflags(std::ios::showbase)
+                  << std::endl;
+      }
+
+
+      if (visualization)
+      {
+         // uex_out.precision(8);
+         // uex_out << "parallel " << num_procs << " " << myid << "\n";
+         // uex_out << "solution\n" << pmesh << uex_gf <<
+         //          "window_title 'Exact u' "
+         //          << flush;
+
+         u_out << "parallel " << num_procs << " " << myid << "\n";
+         u_out.precision(8);
+         u_out << "solution\n" << pmesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+
+         // sigma_out << "parallel " << num_procs << " " << myid << "\n";
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << pmesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+      if (test_norm == test_norm_type::robust)
+      {
+         coeff_fes->Update();
+         c1_gf.Update();
+         c2_gf.Update();
+         Array<int> edofs;
+         for (int i = 0; i < pmesh.GetNE(); i++)
+         {
+            double volume = pmesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            coeff_fes->GetElementDofs(i,edofs);
+            c1_gf.SetSubVector(edofs,c1);
+            c2_gf.SetSubVector(edofs,c2);
+         }
+      }
+   }
+
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatf_fes;
+   delete hatf_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case polynomial:
+      {
+         int n=2;
+         int m=2;
+         u = pow(x,n)*pow(y,m);
+         du[0] = n * pow(x,n-1) * pow(y,m);
+         du[1] = m * pow(x,n) * pow(y,m-1);
+         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
+             + m * (m-1) * pow(x,n) * pow(y,m-2);
+      }
+      break;
+      case EJ:
+      {
+         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
+         double r1 = (1. + alpha) / (2.*epsilon);
+         double r2 = (1. - alpha) / (2.*epsilon);
+         double denom = exp(-r2) - exp(-r1);
+
+
+         double g1 = exp(r2*(x-1.));
+         double g1_x = r2*g1;
+         double g1_xx = r2*g1_x;
+         double g2 = exp(r1*(x-1.));
+         double g2_x = r1*g2;
+         double g2_xx = r1*g2_x;
+         double g = g1-g2;
+         double g_x = g1_x - g2_x;
+         double g_xx = g1_xx - g2_xx;
+
+
+         u = g * cos(M_PI * y)/denom;
+         double u_x = g_x * cos(M_PI * y)/denom;
+         double u_xx = g_xx * cos(M_PI * y)/denom;
+         double u_y = -M_PI * g * sin(M_PI*y)/denom;
+         double u_yy = -M_PI * M_PI * u;
+         du[0] = u_x;
+         du[1] = u_y;
+         d2u = u_xx + u_yy;
+
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ε ∇ u
+   sigma = du;
+   sigma *= epsilon;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return -exact_u(X);
+}
+
+void exact_hatf(const Vector & X, Vector & hatf)
+{
+   Vector sigma;
+   exact_sigma(X,sigma);
+   double u = exact_u(X);
+   hatf.SetSize(X.Size());
+   for (int i = 0; i<hatf.Size(); i++)
+   {
+      hatf[i] = beta[i] * u - sigma[i];
+   }
+}
+
+double f_exact(const Vector & X)
+{
+   // f = - εΔu + ∇⋅(βu)
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+
+   double s = 0;
+   for (int i = 0; i<du.Size(); i++)
+   {
+      s += beta[i] * du[i];
+   }
+   return -epsilon * d2u + s;
+}

examples/dpg_tests/diffusion/blkfosls.cpp

@@ -0,0 +1,203 @@
+//                 MFEM Fosls 1
+//
+// Compile with: make blkfosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// FOSLS:
+//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
+
+
+// -------------------------------------------------
+// |   |    u    |         σ           |    RHS    |
+// -------------------------------------------------
+// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
+// |   |         |                     |           |
+// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
+
+// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec0 = new H1_FECollection(order, dim);
+   FiniteElementCollection *fec1 = new RT_FECollection(order-1, dim);
+   FiniteElementSpace fespace0(&mesh, fec0);
+   FiniteElementSpace fespace1(&mesh, fec1);
+
+   Array<FiniteElementSpace *> fespaces(2);
+   fespaces[0] = &fespace0;
+   fespaces[1] = &fespace1;
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   BlockBilinearForm a(fespaces);
+   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
+
+   cout << "H1 fespace = " << fespace0.GetVSize() << endl;
+   cout << "RT fespace = " << fespace1.GetVSize() << endl;
+
+   FiniteElementCollection *fec2 = new RT_Trace_FECollection(order-1, dim);
+   FiniteElementSpace RT_trace_fes(&mesh, fec2);
+   cout << "RT trace = " << RT_trace_fes.GetVSize() << endl;
+
+   // for (int i = 0; i<mesh.GetNE(); i++)
+   // {
+   //    // const FiniteElement * fe = fespace1.GetFE(i);
+   //    // fespace1.GetTraceElement()
+   //    Array<int> faces, ori;
+   //    mesh.GetElementEdges(i, faces, ori);
+   //    for (int f = 0; f<faces.Size(); f++)
+   //    {
+   //       const FiniteElement * fe_trace = RT_trace_fes.GetFaceElement(faces[f]);
+   //       cout << fe_trace->GetDof() << endl;
+   //       Array<int> face_dofs;
+   //       RT_trace_fes.GetFaceDofs(faces[f],face_dofs);
+   //       cout << "face dofs = " << endl;
+   //       face_dofs.Print();
+   //    }
+
+   //    // cout << fe->GetGeomType() << endl;
+   //    Array<int> vdofs;
+   //    RT_trace_fes.GetElementVDofs(i, vdofs);
+   //    cout << "trace dofs = " << endl;
+   //    vdofs.Print();
+   //    fespace1.GetElementVDofs(i, vdofs);
+   //    cout << "elem dofs = " << endl;
+   //    vdofs.Print();
+   //    cin.get();
+   // }
+
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   Array2D<BilinearFormIntegrator * > blfi(2,2);
+   blfi(0,0) = new DiffusionIntegrator(one);
+   blfi(0,1) = new MixedVectorWeakDivergenceIntegrator(one);
+   blfi(1,0) = new MixedVectorGradientIntegrator(negone);
+
+   BilinearFormIntegrator * divdiv = new DivDivIntegrator(one);
+   BilinearFormIntegrator * mass = new VectorFEMassIntegrator(one);
+   SumIntegrator * suminteg = new SumIntegrator();
+   suminteg->AddIntegrator(divdiv);
+   suminteg->AddIntegrator(mass);
+   blfi(1,1) = suminteg;
+
+   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
+   integ->SetIntegrators(blfi);
+   a.AddDomainIntegrator(integ);
+   a.Assemble();
+
+
+   BlockLinearForm b(fespaces);
+
+   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
+   Array<LinearFormIntegrator * > lfi(2);
+   lfi[0] = nullptr;
+   lfi[1] = new VectorFEDomainLFDivIntegrator(negone);
+   lininteg->SetIntegrators(lfi);
+   b.AddDomainIntegrator(lininteg);
+   b.Assemble();
+
+
+   // need to implement blkgridfunction later but for now Vector would do
+   int size = 0;
+   for (int i = 0; i<fespaces.Size(); i++)
+   {
+      size += fespaces[i]->GetVSize();
+   }
+
+   Vector x(size);
+   x = 0.0;
+
+   OperatorPtr A;
+   Vector X,B;
+   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
+
+   GSSmoother M((SparseMatrix&)(*A));
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(200);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X,b,x);
+
+   GridFunction u_gf, sigma_gf;
+   double *data = x.GetData();
+   u_gf.MakeRef(fespaces[0],&data[0]);
+   sigma_gf.MakeRef(fespaces[1],&data[fespaces[0]->GetVSize()]);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << u_gf <<
+                "window_title 'Numerical u' "
+                << flush;
+      socketstream sols_sock(vishost, visport);
+      sols_sock.precision(8);
+      sols_sock << "solution\n" << mesh << sigma_gf <<
+                "window_title 'Numerical sigma' "
+                << flush;
+   }
+
+   delete fec0;
+
+   return 0;
+}

examples/dpg_tests/diffusion/fosls.cpp

@@ -0,0 +1,223 @@
+//                   MFEM Fosls example
+//
+// Compile with: make fosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// FOSLS:
+//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
+
+
+// -------------------------------------------------
+// |   |    u    |         σ           |    RHS    |
+// -------------------------------------------------
+// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
+// |   |         |                     |           |
+// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
+
+// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   FiniteElementCollection *H1fec = new H1_FECollection(order,dim);
+   FiniteElementSpace *H1fes = new FiniteElementSpace(&mesh, H1fec);
+
+   FiniteElementCollection *RTfec = new RT_FECollection(order-1,dim);
+   FiniteElementSpace *RTfes = new FiniteElementSpace(&mesh, RTfec);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Linear forms
+   LinearForm b_0(H1fes);
+   // (f,∇⋅τ )
+   LinearForm b_1(RTfes);
+   b_1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(negone));
+
+   // Bilinear forms
+   // (∇u,∇v)
+   BilinearForm a_00(H1fes);
+   a_00.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   //   -(σ,∇v)
+   MixedBilinearForm a_01(RTfes, H1fes);
+   a_01.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(
+                               one)); // (-1 is included)
+
+   //    // -(∇u,τ)
+   // MixedBilinearForm()
+   MixedBilinearForm a_10(H1fes, RTfes);
+   a_10.AddDomainIntegrator(new MixedVectorGradientIntegrator(negone));
+
+   // (∇⋅σ, ∇⋅τ) + (σ,τ)
+
+   BilinearForm a_11(RTfes);
+   a_11.AddDomainIntegrator(new DivDivIntegrator(one));
+   a_11.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Array<int> block_Toffsets(3);
+   block_Toffsets[0] = 0;
+   block_Toffsets[1] = H1fes->GetTrueVSize();
+   block_Toffsets[2] = RTfes->GetTrueVSize();
+   block_Toffsets.PartialSum();
+
+   Vector rhs_H1(H1fes->GetVSize()); rhs_H1 = 0.;
+   Vector rhs_RT(RTfes->GetVSize()); rhs_RT = 0.;
+
+   Vector x_H1(H1fes->GetVSize()); x_H1 = 0.;
+   Vector x_RT(RTfes->GetVSize()); x_RT = 0.;
+
+
+   Vector RHS_H1(H1fes->GetTrueVSize()); RHS_H1 = 0.0;
+   Vector RHS_RT(RTfes->GetTrueVSize()); RHS_RT = 0.0;
+
+   Vector X_H1(H1fes->GetTrueVSize()); X_H1 = 0.0;
+   Vector X_RT(RTfes->GetTrueVSize()); X_RT = 0.0;
+
+
+   b_0.Update(H1fes,rhs_H1,0);
+   b_0.Assemble();
+
+   b_1.Update(RTfes,rhs_RT,0);
+   b_1.Assemble();
+
+
+   // Assembly and BC
+   a_00.Assemble();
+   SparseMatrix A_00;
+   a_00.FormLinearSystem(ess_tdof_list,x_H1,rhs_H1,
+                         A_00,X_H1,RHS_H1);
+
+   a_01.Assemble();
+   SparseMatrix A_01;
+   Array<int> empty;
+   a_01.FormRectangularSystemMatrix(empty, ess_tdof_list,A_01);
+
+
+   a_10.Assemble();
+   SparseMatrix A_10;
+
+   a_10.FormRectangularLinearSystem(ess_tdof_list,empty,x_H1,rhs_RT,
+                                    A_10,X_H1,RHS_RT);
+
+   a_11.Assemble();
+   SparseMatrix A_11;
+   a_11.FormSystemMatrix(empty,A_11);
+
+
+   BlockMatrix BlockA(block_Toffsets);
+   BlockA.SetBlock(0,0,&A_00);
+   BlockA.SetBlock(0,1,&A_01);
+   BlockA.SetBlock(1,0,&A_10);
+   BlockA.SetBlock(1,1,&A_11);
+
+
+   BlockVector RHS(block_Toffsets);
+   RHS.GetBlock(0) = RHS_H1;
+   RHS.GetBlock(1) = RHS_RT;
+
+   BlockVector X(block_Toffsets);
+   X.GetBlock(0) = X_H1;
+   X.GetBlock(1) = X_RT;
+
+
+   SparseMatrix * A = BlockA.CreateMonolithic();
+
+   GSSmoother M(*A);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(RHS, X);
+
+   GridFunction u_gf(H1fes), sigma_gf(RTfes);
+   u_gf = 0.;
+   sigma_gf = 0.;
+
+   const SparseMatrix * P = H1fes->GetConformingProlongation();
+   if (P)
+   {
+      a_00.RecoverFEMSolution(X.GetBlock(0),rhs_H1,u_gf);
+      a_11.RecoverFEMSolution(X.GetBlock(1),rhs_RT,sigma_gf);
+   }
+   else
+   {
+      u_gf.MakeRef(X.GetBlock(0),0);
+      sigma_gf.MakeRef(X.GetBlock(1),0);
+   }
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << u_gf <<
+                "window_title 'Numerical u' "
+                << flush;
+      socketstream sols_sock(vishost, visport);
+      sols_sock.precision(8);
+      sols_sock << "solution\n" << mesh << sigma_gf <<
+                "window_title 'Numerical sigma' "
+                << flush;
+   }
+
+   return 0;
+}

examples/dpg_tests/diffusion/makefile

@@ -0,0 +1,61 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = blkfosls fosls primal_dpg \
+	 				uw_dpg 
+PAR_EXAMPLES = uw_dpgp 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm -rf ParaView
+
+clean-exec:

examples/dpg_tests/diffusion/primal_dpg.cpp

@@ -0,0 +1,179 @@
+//                     MFEM primal_dpg example
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");   
+   args.ParseCheck();
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+   // mesh.UniformRefinement();
+
+   // 3. Define a finite element space on the mesh. Here we use H1 continuous
+   //    high-order Lagrange finite elements of the given order.
+   H1_FECollection fec(order, mesh.Dimension());
+   FiniteElementSpace H1fes(&mesh, &fec);
+
+   RT_Trace_FECollection trace_fec(order-1, mesh.Dimension());
+   FiniteElementSpace RTtrace_fes(&mesh, &trace_fec);
+
+   int dim = mesh.Dimension();
+   int test_order = order;
+   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
+   {
+      test_order++;
+   }
+
+   test_order++;
+
+   H1_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+
+   trial_fes.Append(&H1fes);
+   trial_fes.Append(&RTtrace_fes);
+   test_fecs.Append(&test_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+
+
+   ConstantCoefficient one(1.0);
+   a->AddTrialIntegrator(new DiffusionIntegrator(one),0,0);
+   a->AddTrialIntegrator(new TraceIntegrator,1,0);
+
+   BilinearFormIntegrator * diffusion = new DiffusionIntegrator(one);
+   BilinearFormIntegrator * mass = new MassIntegrator(one);
+   a->AddTestIntegrator(diffusion,0,0);
+   a->AddTestIntegrator(mass,0,0);
+
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(one),0);
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Vector X,B;
+   OperatorPtr Ah;
+
+   int size = H1fes.GetVSize() + RTtrace_fes.GetVSize();
+
+   Vector x(size);
+   x = 0.0;
+
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   delete M;
+
+
+   a->RecoverFEMSolution(X,x);
+
+   GridFunction u_gf;
+   double *data = x.GetData();
+   u_gf.MakeRef(&H1fes,data);
+
+   GridFunction s_gf;
+   s_gf.MakeRef(&RTtrace_fes,&data[H1fes.GetVSize()]);
+
+
+
+   RT_FECollection RTfec(order-1, mesh.Dimension());
+   FiniteElementSpace RTfes(&mesh, &RTfec);
+
+   GridFunction sigma_gf(&RTfes);
+   sigma_gf = 0.0;
+   for (int i = 0; i<mesh.GetNE(); i++)
+   {
+      Array<int> strace_dofs;
+      Array<int> trace_dofs;
+      Vector dofs;
+      RTtrace_fes.GetElementDofs(i,trace_dofs);
+      strace_dofs.SetSize(trace_dofs.Size());
+      // shift dofs;
+      for (int j = 0; j< trace_dofs.Size(); j++)
+      {
+         int offset = trace_dofs[j] < 0 ? -H1fes.GetVSize() : H1fes.GetVSize();
+         strace_dofs[j] = offset + trace_dofs[j];
+      }
+      x.GetSubVector(strace_dofs, dofs);
+      sigma_gf.SetSubVector(trace_dofs,dofs);
+
+   }
+
+
+
+
+
+   ParaViewDataCollection paraview_dc("DPG_example", &mesh);
+   paraview_dc.SetPrefixPath("ParaView");
+   paraview_dc.SetLevelsOfDetail(order);
+   paraview_dc.SetCycle(0);
+   paraview_dc.SetDataFormat(VTKFormat::BINARY);
+   paraview_dc.SetHighOrderOutput(true);
+   paraview_dc.SetTime(0.0); // set the time
+   paraview_dc.RegisterField("field",&u_gf);
+   paraview_dc.RegisterField("flux",&sigma_gf);
+   // paraview_dc.RegisterField("flux",&s_gf);
+   paraview_dc.Save();
+
+
+
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << u_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+
+   socketstream soltrace_sock(vishost, visport);
+   soltrace_sock.precision(8);
+   soltrace_sock << "solution\n" << mesh << sigma_gf <<
+                 "window_title 'Flux sigma_n' "
+                 << flush;
+
+
+
+}

examples/dpg_tests/diffusion/uw_dpg.cpp

@@ -0,0 +1,403 @@
+//                       MFEM Ultraweak DPG example
+//
+// Compile with: make uw_dpg
+//
+// sample runs 
+// ./uw_dpg -m ../lshape2.mesh -o 2 -ref 20 -graph-norm -do 1 -prob 0
+
+//     - Δ u = f,   in Ω
+//         u = u_0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(u , ∇⋅τ) - (σ , τ) + < û, τ⋅n> = 0,      ∀ τ ∈ H(div,Ω)      
+//  (σ , ∇ v) + < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
+//                                û = 0       on ∂Ω 
+
+// Note: 
+// û := u
+// σ̂ := -σ
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
+// -------------------------------------------------------------
+// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
+// |   |           |           |           |         |         |
+// | v |           |  (σ,∇ v)  |           |  <σ̂,v>  |  (f,v)  |  
+
+// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   lshape,   
+   general  
+};
+
+prob_type prob;
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u);
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ∇ u
+   sigma = du;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return exact_u(X);
+}
+
+void exact_hatsigma(const Vector & X, Vector & hatsigma)
+{
+   exact_sigma(X,hatsigma);
+   hatsigma *= -1.;
+}
+
+double f_exact(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return -d2u;
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool adjoint_graph_norm = false;
+   bool visualization = true;
+   int iprob = 0;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");               
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space"); 
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 1; }
+   prob = (prob_type)iprob;
+
+   if (prob == prob_type::lshape)
+   {
+      mesh_file = "../lshape2.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.UniformRefinement();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatsigma_fes = new FiniteElementSpace(&mesh,hatsigma_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatsigma_fes);
+
+   test_fec.Append(tau_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //  -(u,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
+
+   // -(σ,τ) 
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negone)),1,0);
+
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
+
+   //  <σ̂,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,1);
+
+   // test integrators (space-induced norm for H(div) × H1)
+   // (∇⋅τ,∇⋅δτ)
+   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
+   // (τ,δτ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),1,1);
+
+   // additional terms for adjoint graph norm
+   if (adjoint_graph_norm)
+   {
+      // -(∇v,δτ) 
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
+      // -(τ,∇δv) 
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
+      // (τ,δτ)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f(f_exact);
+   if (prob == prob_type::general)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),1);
+   }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   Array<int> elements_to_refine;
+   GridFunction hatu_gf;
+
+
+   socketstream u_out;
+   // socketstream sigma_out;
+   socketstream mesh_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+      mesh_out.open(vishost, visport);
+   }
+
+
+   for (int iref = 0; iref<ref; iref++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+   // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new GSSmoother(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      cout << "Residual = " << residual << endl;
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      double theta = 0.7;
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+      
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      GridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      if (visualization)
+      {
+         u_out.precision(8);
+         string keys = (iref == 0) ? "keys em\n" : "keys";
+         u_out << "solution\n" << mesh << u_gf 
+               << "window_title 'Numerical u' "
+                  << flush;
+
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << mesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+
+         mesh_out.precision(8);
+         mesh_out << "mesh\n" << mesh 
+                  << keys
+                  << "window_title 'Mesh' "
+                  << flush;
+
+      }
+
+      mesh.GeneralRefinement(elements_to_refine);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case lshape:
+      {
+         double r = sqrt(x*x + y*y);
+         double alpha = 2./3.;
+         double theta = atan2(y,x);
+         if (theta < 0) theta += 2*M_PI;
+         u = pow(r,alpha) * sin(alpha * theta);
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}

examples/dpg_tests/diffusion/uw_dpgp.cpp

@@ -0,0 +1,404 @@
+//                 MFEM UW DPG parallel example
+//
+// Compile with: make poisson_fosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(u , ∇⋅τ) + < û, τ⋅n> - (σ , τ) = 0,      ∀ τ ∈ H(div,Ω)      
+//  (σ , ∇ v) - < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
+//            û = 0        on ∂Ω 
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
+// -------------------------------------------------------------
+// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
+// |   |           |           |           |         |         |
+// | v |           |  (σ,∇ v)  |           | -<σ̂,v>  |  (f,v)  |  
+
+// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   lshape,   
+   general  
+};
+
+prob_type prob;
+
+double exact(const Vector & X)
+{
+   double x = X[0];
+   double y = X[1];
+
+   double r = sqrt(x*x + y*y);
+   double alpha = 2./3.;
+   double theta = atan2(y,x);
+   if (theta < 0) theta += 2*M_PI;
+
+   return pow(r,alpha) * sin(alpha * theta);
+}
+
+void gradexact(const Vector & X, Vector & grad)
+{
+   grad.SetSize(2);
+   double x = X[0];
+   double y = X[1];
+
+   double r = sqrt(x*x + y*y);
+   double alpha = 2./3.;
+   double theta = atan2(y,x);
+   if (theta < 0) theta += 2*M_PI;
+
+   double r_x = x/r;
+   double r_y = y/r;
+   double theta_x = - y / (r*r);
+   double theta_y = x / (r*r);
+   double beta = alpha * pow(r,alpha - 1.);
+   grad[0] = beta*(r_x * sin(alpha*theta) + r * theta_x * cos(alpha*theta));
+   grad[1] = beta*(r_y * sin(alpha*theta) + r * theta_y * cos(alpha*theta));
+}
+
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool adjoint_graph_norm = false;
+   bool visualization = true;
+   int iprob = 0;
+   bool static_cond = false;
+   double theta = 0.7;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");    
+   args.AddOption(&theta, "-theta", "--theta_factor",
+                  "Refinement factor");                              
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");   
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (iprob > 1) { iprob = 1; }
+   prob = (prob_type)iprob;
+
+   if (prob == prob_type::lshape)
+   {
+      mesh_file = "../lshape2.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.UniformRefinement();
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,hatsigma_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatsigma_fes);
+
+   test_fec.Append(tau_fec);
+   test_fec.Append(v_fec);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //  -(u,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
+
+   // -(σ,τ) 
+   TransposeIntegrator * mass = new TransposeIntegrator(new VectorFEMassIntegrator(negone));
+   a->AddTrialIntegrator(mass,1,0);
+
+   // (σ,∇ v)
+   TransposeIntegrator * grad = new TransposeIntegrator(new GradientIntegrator(one));
+   a->AddTrialIntegrator(grad,1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
+
+   // -<σ̂,v> (sign is included in σ̂)
+   a->AddTrialIntegrator(new TraceIntegrator,3,1);
+
+   // test integrators (space-induced norm for H(div) × H1)
+   // (∇⋅τ,∇⋅δτ)
+   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
+   // (τ,δτ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),1,1);
+
+   // additional terms for adjoint graph norm
+   if (adjoint_graph_norm)
+   {
+      // -(∇v,δτ) 
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
+      // -(τ,∇δv) 
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
+      // (τ,δτ)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   }
+   // RHS
+   if (prob == prob_type::general)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(one),1);
+   }
+
+   FunctionCoefficient uex(exact);
+   Array<int> elements_to_refine;
+   ParGridFunction hatu_gf;
+
+
+   socketstream u_out;
+   socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      sigma_out.open(vishost, visport);
+   }
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      if (prob == prob_type::lshape)
+      {
+         hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+         hatu_gf.ProjectBdrCoefficient(uex,ess_bdr);
+      }
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip=2;
+      }
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      M->SetDiagonalBlock(skip,amg2);
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      if (myid == 0)
+      {
+         cout << "Global Residual = " << globalresidual << endl;
+      }
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      ParGridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      if (visualization)
+      {
+         u_out << "parallel " << num_procs << " " << myid << "\n";
+         u_out.precision(8);
+         u_out << "solution\n" << pmesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+
+         sigma_out << "parallel " << num_procs << " " << myid << "\n";
+         sigma_out.precision(8);
+         sigma_out << "solution\n" << pmesh << sigma_gf <<
+                  "window_title 'Numerical flux' "
+                  << flush;
+      }
+
+
+      if (i == ref-1)
+      {
+         break;
+      }
+
+      pmesh.GeneralRefinement(elements_to_refine);
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}

examples/dpg_tests/grad-div/makefile

@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/grad-div,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = primal_dpg 
+PAR_EXAMPLES = 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:

examples/dpg_tests/grad-div/primal_dpg.cpp

@@ -0,0 +1,176 @@
+//                                MFEM primal dpg example for grad-dic problem
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, F, and r.h.s., f. See below for implementation.
+void F_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.ParseCheck();
+
+   kappa = freq * M_PI;
+
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+   // mesh.UniformRefinement();
+
+   RT_FECollection fec(order-1, mesh.Dimension());
+   FiniteElementSpace RTfes(&mesh, &fec);
+
+   H1_Trace_FECollection trace_fec(order, mesh.Dimension());
+   FiniteElementSpace H1trace_fes(&mesh, &trace_fec);
+
+   int dim = mesh.Dimension();
+   int test_order = order;
+   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
+   {
+      test_order++;
+   }
+
+   test_order++;
+
+   RT_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace *> trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+   
+   trial_fes.Append(&RTfes);
+   trial_fes.Append(&H1trace_fes);
+   test_fecs.Append(&test_fec);
+
+
+   GridFunction rt_gf(&RTfes);
+   VectorFunctionCoefficient F(dim, F_exact);
+   rt_gf.ProjectCoefficient(F);
+
+   Vector x(RTfes.GetVSize()+H1trace_fes.GetVSize());
+   x = 0.;
+   x.SetVector(rt_gf,0);
+
+
+   ConstantCoefficient alpha(1.0);
+   ConstantCoefficient beta(1.0);
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+   a->AddTrialIntegrator(new DivDivIntegrator(alpha),0,0);
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(beta),0,0);
+   a->AddTrialIntegrator(new NormalTraceIntegrator,1,0);
+   a->AddTestIntegrator(new DivDivIntegrator(alpha),0,0);
+   a->AddTestIntegrator(new VectorFEMassIntegrator(beta),0,0);
+
+
+   VectorFunctionCoefficient f(dim, f_exact);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      RTfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Vector X,B;
+
+   OperatorPtr Ah;
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   // GridFunction u_gf;
+   double *data = x.GetData();
+   rt_gf.MakeRef(&RTfes,data);
+
+   GridFunction exact_gf(&RTfes);
+   exact_gf.ProjectCoefficient(F);
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << rt_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+
+   socketstream soltrace_sock(vishost, visport);
+   soltrace_sock.precision(8);
+   soltrace_sock << "solution\n" << mesh << exact_gf <<
+                 "window_title 'Exact' "
+                 << flush;
+
+}
+
+
+// The exact solution (for non-surface meshes)
+void F_exact(const Vector &p, Vector &F)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+
+   F(0) = cos(kappa*x)*sin(kappa*y);
+   F(1) = cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      F(2) = 0.0;
+   }
+}
+
+// The right hand side
+void f_exact(const Vector &p, Vector &f)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+
+   double temp = 1 + 2*kappa*kappa;
+
+   f(0) = temp*cos(kappa*x)*sin(kappa*y);
+   f(1) = temp*cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      f(2) = 0;
+   }
+}

examples/dpg_tests/lshape2.mesh

@@ -0,0 +1,51 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+3
+1 3 0 1 4 3
+1 3 3 4 7 6
+1 3 1 2 5 4
+
+boundary
+8
+1 1 0 1
+1 1 1 2
+1 1 2 5
+2 1 5 4
+2 1 4 7
+1 1 7 6
+1 1 6 3
+1 1 3 0
+
+vertices
+8
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P1
+VDim: 2
+Ordering: 1
+
+-1 1
+-1 -0
+-1 -1
+0 1
+0 -0
+0 -1
+1 1
+1 -0

examples/ex8.cpp

@@ -72,8 +72,8 @@ int main(int argc, char *argv[])
    //    largest number that gives a final mesh with no more than 10,000
    //    elements.
    {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels = 1;
+      // (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
@@ -147,6 +147,8 @@ int main(int argc, char *argv[])
    F.AddDomainIntegrator(new DomainLFIntegrator(one));
    F.Assemble();
 
+
+
    // 7. Set up the mixed bilinear form for the primal trial unknowns, B0,
    //    the mixed bilinear form for the interfacial unknowns, Bhat,
    //    the inverse stiffness matrix on the discontinuous test space, Sinv,
@@ -187,10 +189,17 @@ int main(int argc, char *argv[])
    // 8. Set up the 1x2 block Least Squares DPG operator, B = [B0  Bhat],
    //    the normal equation operator, A = B^t Sinv B, and
    //    the normal equation right-hand-size, b = B^t Sinv F.
-   BlockOperator B(offsets_test, offsets);
+   // BlockOperator B(offsets_test, offsets);
+
+   BlockMatrix B(offsets_test, offsets);
    B.SetBlock(0,0,&matB0);
    B.SetBlock(0,1,&matBhat);
-   RAPOperator A(B, matSinv, B);
+   SparseMatrix * Bh = B.CreateMonolithic();
+
+   SparseMatrix * A = RAP(*Bh, matSinv, *Bh);
+
+
+   // RAPOperator A(B, matSinv, B);
    {
       Vector SinvF(s_test);
       matSinv.Mult(F,SinvF);
@@ -234,7 +243,20 @@ int main(int argc, char *argv[])
    // 10. Solve the normal equation system using the PCG iterative solver.
    //     Check the weighted norm of residual for the DPG least square problem.
    //     Wrap the primal variable in a GridFunction for visualization purposes.
-   PCG(A, P, b, x, 1, 200, 1e-12, 0.0);
+
+
+   // PCG(*A, P, b, x, 1, 200, 1e-12, 0.0);
+
+   GSSmoother M(*A);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(b, x);
+
 
    {
       Vector LSres(s_test);

fem/bilininteg.cpp

@@ -144,6 +144,14 @@ void BilinearFormIntegrator::AssembleFaceMatrix (
                "   is not implemented for this class.");
 }
 
+void BilinearFormIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                     const FiniteElement &trial_face_fe,  const FiniteElement &test_fe,
+                                                     FaceElementTransformations &Trans, DenseMatrix &elmat)
+{
+   mfem_error ("BilinearFormIntegrator::AssembleTraceFaceMatrix(...)\n"
+               "   is not implemented for this class.");
+}
+
 void BilinearFormIntegrator::AssembleFaceMatrix(
    const FiniteElement &trial_face_fe, const FiniteElement &test_fe1,
    const FiniteElement &test_fe2, FaceElementTransformations &Trans,
@@ -2583,6 +2591,55 @@ void DivDivIntegrator::AssembleElementMatrix(
    }
 }
 
+void DivDivIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   double c;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector divshape(tr_nd);
+   Vector te_divshape(te_nd);
+#else
+   divshape.SetSize(tr_nd);
+   te_divshape.SetSize(te_nd);
+#endif
+   elmat.SetSize(te_nd,tr_nd);
+
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * max(test_fe.GetOrder(),
+                          trial_fe.GetOrder()) - 2; // <--- OK for RTk
+      ir = &IntRules.Get(test_fe.GetGeomType(), order);
+   }
+
+   elmat = 0.0;
+
+   for (int i = 0; i < ir -> GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      trial_fe.CalcDivShape(ip,divshape);
+      test_fe.CalcDivShape(ip,te_divshape);
+
+      Trans.SetIntPoint (&ip);
+      c = ip.weight / Trans.Weight();
+
+      if (Q)
+      {
+         c *= Q -> Eval (Trans, ip);
+      }
+
+      te_divshape *= c;
+      AddMultVWt(te_divshape, divshape, elmat);
+   }
+}
 
 void VectorDiffusionIntegrator::AssembleElementMatrix(
    const FiniteElement &el,
@@ -3774,7 +3831,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
       for (i = 0; i < ndof1; i++)
          for (j = 0; j < face_ndof; j++)
          {
-            elmat(i, j) -= shape1_n(i) * face_shape(j);
+            elmat(i, j) += shape1_n(i) * face_shape(j);
          }
       if (ndof2)
       {
@@ -3782,12 +3839,135 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
          for (i = 0; i < ndof2; i++)
             for (j = 0; j < face_ndof; j++)
             {
-               elmat(ndof1+i, j) += shape2_n(i) * face_shape(j);
+               elmat(ndof1+i, j) -= shape2_n(i) * face_shape(j);
             }
       }
    }
 }
 
+void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                              const FiniteElement &trial_face_fe,
+                                              const FiniteElement &test_fe,
+                                              FaceElementTransformations & Trans,
+                                              DenseMatrix &elmat)
+{
+   int i, j, face_ndof, ndof;
+   int order;
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   face_shape.SetSize(face_ndof);
+   shape.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
+      {
+         order += Trans.OrderW();
+      }
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      trial_face_fe.CalcPhysShape(Trans,face_shape);
+
+      // Finite element shape function
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcPhysShape(*eltrans, shape);
+
+      face_shape *= Trans.Weight()*ip.weight;
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += scale * shape(i) * face_shape(j);
+         }
+      }
+   }
+}
+
+void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                    const FiniteElement &trial_face_fe,
+                                                    const FiniteElement &test_fe,
+                                                    FaceElementTransformations &Trans,
+                                                    DenseMatrix &elmat)
+{
+   int i, j, face_ndof, ndof, dim;
+   int order;
+
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE, "");
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+   dim = test_fe.GetDim();
+
+   face_shape.SetSize(face_ndof);
+   normal.SetSize(dim);
+   shape.SetSize(ndof,dim);
+   shape_n.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      Trans.SetAllIntPoints(&ip);
+      trial_face_fe.CalcPhysShape(Trans, face_shape);
+      CalcOrtho(Trans.Jacobian(),normal);
+      ElementTransformation * etrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcVShape(*etrans, shape);
+      shape.Mult(normal, shape_n);
+      face_shape *= ip.weight;
+
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += scale * shape_n(i) * face_shape(j);
+         }
+      }
+   }
+}
+
 
 void NormalInterpolator::AssembleElementMatrix2(
    const FiniteElement &dom_fe, const FiniteElement &ran_fe,

fem/bilininteg.hpp

@@ -150,6 +150,12 @@ public:
                                    FaceElementTransformations &Trans,
                                    DenseMatrix &elmat);
 
+   virtual void AssembleTraceFaceMatrix(int elem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+
    /** Abstract method used for assembling TraceFaceIntegrators in a
        MixedBilinearForm. */
    virtual void AssembleFaceMatrix(const FiniteElement &trial_face_fe,
@@ -2719,7 +2725,7 @@ protected:
 
 private:
 #ifndef MFEM_THREAD_SAFE
-   Vector divshape;
+   Vector divshape, te_divshape;
 #endif
 
    // PA extension
@@ -2736,6 +2742,10 @@ public:
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
 };
 
 /** Integrator for
@@ -3230,6 +3240,19 @@ public:
                                    DenseMatrix &elmat);
 };
 
+class TraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, shape;
+public:
+   TraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+};
+
 /** Integrator for the form: < v, [w.n] > over all faces (the interface) where
     the trial variable v is defined on the interface and the test variable w is
     in an H(div)-conforming space. */
@@ -3249,6 +3272,23 @@ public:
                                    DenseMatrix &elmat);
 };
 
+class NormalTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, normal, shape_n;
+   DenseMatrix shape;
+
+public:
+   NormalTraceIntegrator() { }
+   virtual void AssembleTraceFaceMatrix(int ielem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+};
+
+
+
 /** Abstract class to serve as a base for local interpolators to be used in the
     DiscreteLinearOperator class. */
 class DiscreteInterpolator : public BilinearFormIntegrator { };

fem/blockbilinearform.cpp

@@ -0,0 +1,512 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+BlockBilinearForm::BlockBilinearForm(Array<FiniteElementSpace *> & fespaces_) :
+   Matrix(0), fespaces(fespaces_)
+{
+   height = 0;
+   nblocks = fespaces.Size();
+   dof_offsets.SetSize(nblocks+1);
+   tdof_offsets.SetSize(nblocks+1);
+   dof_offsets[0] = 0;
+   tdof_offsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      dof_offsets[i+1] = fespaces[i]->GetVSize();
+      tdof_offsets[i+1] = fespaces[i]->GetTrueVSize();
+   }
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+   height = dof_offsets[nblocks];
+   width = height;
+   mat = mat_e = NULL;
+   extern_bfs = 0;
+   element_matrices = NULL;
+   diag_policy = DIAG_KEEP;
+}
+
+
+// Allocate appropriate SparseMatrix and assign it to mat
+void BlockBilinearForm::AllocMat()
+{
+   mat = new SparseMatrix(height);
+}
+
+void BlockBilinearForm::BuildProlongation()
+{
+   P = new BlockMatrix(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   for (int i = 0; i<nblocks; i++)
+   {
+      const SparseMatrix *P_ = fespaces[i]->GetConformingProlongation();
+      const SparseMatrix *R_ = fespaces[i]->GetRestrictionMatrix();
+      P->SetBlock(i,i,const_cast<SparseMatrix*>(P_));
+      R->SetBlock(i,i,const_cast<SparseMatrix*>(R_));
+   }
+}
+
+void BlockBilinearForm::ConformingAssemble()
+{
+   Finalize(0);
+   MFEM_ASSERT(mat, "the BilinearForm is not assembled");
+
+   if (!P) { BuildProlongation(); }
+
+   SparseMatrix * Pm = P->CreateMonolithic();
+
+   SparseMatrix *Pt = Transpose(*Pm);
+
+   SparseMatrix *PtA = mfem::Mult(*Pt, *mat);
+   delete mat;
+   if (mat_e)
+   {
+      SparseMatrix *PtAe = mfem::Mult(*Pt, *mat_e);
+      delete mat_e;
+      mat_e = PtAe;
+   }
+   delete Pt;
+   mat = mfem::Mult(*PtA, *Pm);
+   delete PtA;
+   if (mat_e)
+   {
+      SparseMatrix *PtAeP = mfem::Mult(*mat_e, *Pm);
+      delete mat_e;
+      mat_e = PtAeP;
+   }
+   delete Pm;
+   height = mat->Height();
+   width = mat->Width();
+}
+
+void BlockBilinearForm::Mult(const Vector &x, Vector &y) const
+{
+   // TODO
+}
+
+
+double& BlockBilinearForm::Elem (int i, int j)
+{
+   return mat -> Elem(i,j);
+}
+
+const double& BlockBilinearForm::Elem (int i, int j) const
+{
+   return mat -> Elem(i,j);
+}
+
+MatrixInverse * BlockBilinearForm::Inverse() const
+{
+   return mat -> Inverse();
+}
+
+void BlockBilinearForm::Finalize(int skip_zeros)
+{
+   mat->Finalize(skip_zeros);
+   if (mat_e) { mat_e->Finalize(skip_zeros); }
+}
+
+/// Adds new Block Domain Integrator. Assumes ownership of @a bfi.
+void BlockBilinearForm::AddDomainIntegrator(BlockBilinearFormIntegrator *bfi)
+{
+   domain_integs.Append(bfi);
+}
+
+/// Assembles the form i.e. sums over all domain integrators.
+void BlockBilinearForm::Assemble(int skip_zeros)
+{
+   ElementTransformation *eltrans;
+   DofTransformation * doftrans_j, *doftrans_k;
+   Mesh *mesh = fespaces[0] -> GetMesh();
+   DenseMatrix elmat, *elmat_p;
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   Array<int> vdofs_j, vdofs_k;
+   Array<int> offsetvdofs_j;
+   Array<int> elementblockoffsets(nblocks+1);
+   elementblockoffsets[0] = 0;
+   Array<int> blockoffsets(nblocks+1);
+   blockoffsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      blockoffsets[i+1] = fespaces[i]->GetVSize();
+   }
+   blockoffsets.PartialSum();
+   // mfem::out << "blockoffsets = " ; blockoffsets.Print();
+
+   if (mat == NULL)
+   {
+      AllocMat();
+   }
+
+   if (domain_integs.Size())
+   {
+      // loop through elements
+      for (int i = 0; i < mesh -> GetNE(); i++)
+      {
+         if (element_matrices)
+         {
+            elmat_p = &(*element_matrices)(i);
+         }
+         else
+         {
+            elmat.SetSize(0);
+            for (int k = 0; k < domain_integs.Size(); k++)
+            {
+               for (int j = 0; j<nblocks; j++)
+               {
+                  fe[j] = fespaces[j]->GetFE(i);
+                  elementblockoffsets[j+1] = fe[j]->GetDof();
+               }
+               elementblockoffsets.PartialSum();
+               eltrans = mesh->GetElementTransformation(i);
+               domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+               if (elmat.Size() == 0)
+               {
+                  elmat = elemmat;
+               }
+               else
+               {
+                  elmat += elemmat;
+               }
+            }
+         }
+         if (elmat.Size() == 0)
+         {
+            continue;
+         }
+         else
+         {
+            elmat_p = &elmat;
+         }
+         vdofs.SetSize(0);
+         for (int j = 0; j<nblocks; j++)
+         {
+            doftrans_j = fespaces[j]->GetElementVDofs(i, vdofs_j);
+            int jbeg = elementblockoffsets[j];
+            int jend = elementblockoffsets[j+1]-1;
+            int offset_j = blockoffsets[j];
+            offsetvdofs_j.SetSize(vdofs_j.Size());
+
+            for (int l = 0; l<vdofs_j.Size(); l++)
+            {
+               offsetvdofs_j[l] = vdofs_j[l]<0 ? -offset_j + vdofs_j[l]
+                                  :  offset_j + vdofs_j[l];
+            }
+            vdofs.Append(offsetvdofs_j);
+            for (int k = 0; k<nblocks; k++)
+            {
+               doftrans_k = fespaces[k]->GetElementVDofs(i, vdofs_k);
+               if (doftrans_k || doftrans_j)
+               {
+                  int kbeg = elementblockoffsets[k];
+                  int kend = elementblockoffsets[k+1]-1;
+                  DenseMatrix A;
+                  elmat_p->GetSubMatrix(jbeg,jend,kbeg, kend, A);
+                  TransformDual(doftrans_j, doftrans_k, A);
+                  elmat_p->SetSubMatrix(jbeg,kbeg,A);
+               }
+            }
+         }
+         mat->AddSubMatrix(vdofs,vdofs,*elmat_p, skip_zeros);
+      }
+   }
+
+}
+
+
+
+void BlockBilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                         Vector &x,
+                                         Vector &b, OperatorHandle &A, Vector &X,
+                                         Vector &B, int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+
+   if (!P)
+   {
+      EliminateVDofsInRHS(ess_tdof_list, x, b);
+      X.MakeRef(x, 0, x.Size());
+      B.MakeRef(b, 0, b.Size());
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+   else // non conforming space
+   {
+      B.SetSize(P->Width());
+      P->MultTranspose(b, B);
+      X.SetSize(R->Height());
+
+      mfem::out << "R height, width  = " << R->Height() <<" x "<< R->Width() <<
+                std::endl;
+
+      R->Mult(x, X);
+      EliminateVDofsInRHS(ess_tdof_list, X, B);
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+
+}
+
+void BlockBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                         OperatorHandle &A)
+{
+   if (!mat_e)
+   {
+      const SparseMatrix *P_ = fespaces[0]->GetConformingProlongation();
+      if (P_) { ConformingAssemble(); }
+      EliminateVDofs(ess_tdof_list, diag_policy);
+      const int remove_zeros = 0;
+      Finalize(remove_zeros);
+   }
+   A.Reset(mat, false);
+}
+
+void BlockBilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
+                                           Vector &x)
+{
+   if (!P)
+   {
+      x.SyncMemory(X);
+   }
+   else
+   {
+      // Apply conforming prolongation
+      x.SetSize(P->Height());
+      P->Mult(X, x);
+   }
+}
+
+
+
+void BlockBilinearForm::ComputeElementMatrices()
+{
+   MFEM_ABORT("BlockBilinearForm::ComputeElementMatrices:not implemented yet")
+}
+
+void BlockBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
+{
+   if (element_matrices)
+   {
+      elmat.SetSize(element_matrices->SizeI(), element_matrices->SizeJ());
+      elmat = element_matrices->GetData(i);
+      return;
+   }
+
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   ElementTransformation *eltrans;
+
+   elmat.SetSize(0);
+   if (domain_integs.Size())
+   {
+      for (int j = 0; j<nblocks; j++)
+      {
+         fe[j] = fespaces[j]->GetFE(i);
+      }
+      eltrans = fespaces[0]->GetElementTransformation(i);
+      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
+      {
+         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+         elmat += elemmat;
+      }
+   }
+   else
+   {
+      int matsize = 0;
+      for (int j = 0; j<nblocks; j++)
+      {
+         matsize += fespaces[j]->GetFE(i)->GetDof();
+      }
+      elmat.SetSize(matsize);
+      elmat = 0.0;
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                                             const Vector &sol, Vector &rhs,
+                                             DiagonalPolicy dpolicy)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofs(ess_dofs, sol, rhs, dpolicy);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofs(conf_ess_dofs, sol, rhs, dpolicy);
+   // }
+}
+
+void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                                             DiagonalPolicy dpolicy)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofs(ess_dofs, dpolicy);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofs(conf_ess_dofs, dpolicy);
+   // }
+}
+
+void BlockBilinearForm::EliminateEssentialBCDiag (const Array<int>
+                                                  &bdr_attr_is_ess,
+                                                  double value)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBCDiag: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofsDiag(ess_dofs, value);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofsDiag(conf_ess_dofs, value);
+   // }
+}
+
+void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
+                                       const Vector &sol, Vector &rhs,
+                                       DiagonalPolicy dpolicy)
+{
+   vdofs.HostRead();
+   for (int i = 0; i < vdofs.Size(); i++)
+   {
+      int vdof = vdofs[i];
+      if ( vdof >= 0 )
+      {
+         mat -> EliminateRowCol (vdof, sol(vdof), rhs, dpolicy);
+      }
+      else
+      {
+         mat -> EliminateRowCol (-1-vdof, sol(-1-vdof), rhs, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
+                                       DiagonalPolicy dpolicy)
+{
+   if (mat_e == NULL)
+   {
+      mat_e = new SparseMatrix(height);
+   }
+
+   // mat -> EliminateCols(vdofs, *mat_e,)
+
+   for (int i = 0; i < vdofs.Size(); i++)
+   {
+      int vdof = vdofs[i];
+      if ( vdof >= 0 )
+      {
+         mat -> EliminateRowCol (vdof, *mat_e, dpolicy);
+      }
+      else
+      {
+         mat -> EliminateRowCol (-1-vdof, *mat_e, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofs(
+   const Array<int> &ess_dofs, const Vector &sol, Vector &rhs,
+   DiagonalPolicy dpolicy)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+   MFEM_ASSERT(sol.Size() == height, "incorrect sol Vector size");
+   MFEM_ASSERT(rhs.Size() == height, "incorrect rhs Vector size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowCol (i, sol(i), rhs, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofs (const Array<int>
+                                                      &ess_dofs,
+                                                      DiagonalPolicy dpolicy)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowCol (i, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofsDiag (
+   const Array<int> &ess_dofs,
+   double value)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowColDiag (i, value);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateVDofsInRHS(
+   const Array<int> &vdofs, const Vector &x, Vector &b)
+{
+   mat_e->AddMult(x, b, -1.);
+   mat->PartMult(vdofs, x, b);
+}
+
+
+
+BlockBilinearForm::~BlockBilinearForm()
+{
+   delete mat_e;
+   delete mat;
+   delete element_matrices;
+
+   for (int k=0; k < domain_integs.Size(); k++)
+   {
+      delete domain_integs[k];
+   }
+   for (int k=0; k < trace_integs.Size(); k++)
+   {
+      delete trace_integs[k];
+   }
+   delete P;
+   delete R;
+}
+
+
+
+} // namespace mfem

fem/blockbilinearform.hpp

@@ -0,0 +1,288 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKBILINEARFORM
+#define MFEM_BLOCKBILINEARFORM
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+
+namespace mfem
+{
+
+/** @brief A "square matrix" operator for the associated FE space and
+    BLFIntegrators The sum of all the BLFIntegrators can be used form the matrix
+    M.  */
+class BlockBilinearForm : public Matrix
+{
+
+protected:
+
+   int nblocks;
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+
+   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
+   SparseMatrix *mat;
+
+   /** @brief Sparse Matrix \f$ M_e \f$ used to store the eliminations
+        from the b.c.  Owned.
+       \f$ M + M_e = M_{original} \f$ */
+   SparseMatrix *mat_e;
+
+   /// FE spaces on which the block form lives. Not owned.
+   Array<FiniteElementSpace * > fespaces;
+
+   /** @brief Indicates the Mesh::sequence corresponding to the current state of
+      the BilinearForm. */
+   long sequence;
+
+   /** @brief Indicates the BlockBilinearFormIntegrator%s stored in #domain_integs,
+     are owned by another BlockBilinearForm. */
+   int extern_bfs;
+
+   /// Set of Domain Integrators to be applied.
+   Array<BlockBilinearFormIntegrator * > domain_integs;
+
+   /// Trace integrators.
+   Array<BlockBilinearFormIntegrator * > trace_integs;
+
+   DenseMatrix elemmat;
+   Array<int>  vdofs;
+
+   DenseTensor *element_matrices; ///< Owned.
+
+   BlockMatrix * P = nullptr; // Block Prolongation
+   BlockMatrix * R = nullptr; // Block Restriction
+
+   /** This data member allows one to specify what should be done to the
+    diagonal matrix entries and corresponding RHS values upon elimination of
+    the constrained DoFs. */
+   DiagonalPolicy diag_policy;
+
+   // Allocate appropriate SparseMatrix and assign it to mat
+   void AllocMat();
+
+   void ConformingAssemble();
+
+   void BuildProlongation();
+
+
+private:
+
+public:
+
+   /// Creates bilinear form associated with FE spaces @a *fespaces.
+   BlockBilinearForm(Array<FiniteElementSpace * > & fespaces_);
+
+   /// Get the size of the BilinearForm as a square matrix.
+   int Size() const { return height; }
+
+
+   /// Pre-allocate the internal SparseMatrix before assembly.
+   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }
+
+   /// Returns a reference to: \f$ M_{ij} \f$
+   const double &operator()(int i, int j) { return (*mat)(i,j); }
+
+
+   /// Matrix vector multiplication:  \f$ y = M x \f$
+   virtual void Mult(const Vector &x, Vector &y) const;
+
+   /** @brief Matrix vector multiplication with the original uneliminated
+       matrix.  The original matrix is \f$ M + M_e \f$ so we have:
+       \f$ y = M x + M_e x \f$ */
+   void FullMult(const Vector &x, Vector &y) const
+   { mat->Mult(x, y); mat_e->AddMult(x, y); }
+
+   virtual double &Elem(int i, int j);
+   virtual const double &Elem(int i, int j) const;
+   virtual MatrixInverse *Inverse() const;
+
+   /// Finalizes the matrix initialization.
+   virtual void Finalize(int skip_zeros = 1);
+
+   /// Returns a const reference to the sparse matrix.
+   const SparseMatrix &SpMat() const
+   {
+      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
+      return *mat;
+   }
+
+   /// Returns a reference to the sparse matrix:  \f$ M \f$
+   SparseMatrix &SpMat()
+   {
+      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
+      return *mat;
+   }
+
+   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   const SparseMatrix &SpMatElim() const
+   {
+      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
+      return *mat_e;
+   }
+
+   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   SparseMatrix &SpMatElim()
+   {
+      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
+      return *mat_e;
+   }
+
+   /// Adds new Domain Integrator. Assumes ownership of @a bfi.
+   void AddDomainIntegrator(BlockBilinearFormIntegrator *bfi);
+
+   /// Adds new Trace Integrator. Assumes ownership of @a bfi.
+   void AddTraceIntegrator(BlockBilinearFormIntegrator *bfi);
+
+   /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
+   void operator=(const double a)
+   {
+      if (mat != NULL) { *mat = a; }
+      if (mat_e != NULL) { *mat_e = a; }
+   }
+
+   /// Assembles the form i.e. sums over all domain integrators.
+   void Assemble(int skip_zeros = 1);
+
+
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
+                                 Vector &b, OperatorHandle &A, Vector &X,
+                                 Vector &B, int copy_interior = 0);
+
+   /** @brief Form the linear system A X = B, corresponding to this bilinear
+       form and the linear form @a b(.). */
+   /** Version of the method FormLinearSystem() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   template <typename OpType>
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
+                         OpType &A, Vector &X, Vector &B,
+                         int copy_interior = 0)
+   {
+      OperatorHandle Ah;
+      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
+   /// Form the linear system matrix A, see FormLinearSystem() for details.
+   /** Version of the method FormSystemMatrix() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   template <typename OpType>
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
+   {
+      OperatorHandle Ah;
+      FormSystemMatrix(ess_tdof_list, Ah);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
+
+
+   void ComputeElementMatrices();
+
+   /// Free the memory used by the element matrices.
+   void FreeElementMatrices()
+   { delete element_matrices; element_matrices = NULL; }
+
+   /// Compute the element matrix of the given element
+   /** The element matrix is computed by calling the domain integrators
+       or the one stored internally by a prior call of ComputeElementMatrices()
+       is returned when available.
+   */
+   void ComputeElementMatrix(int i, DenseMatrix &elmat);
+
+   /// Eliminate essential boundary DOFs from the system.
+   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
+       the essential part of the boundary. By default, the diagonal at the
+       essential DOFs is set to 1.0. This behavior is controlled by the argument
+       @a dpolicy. */
+   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                             const Vector &sol, Vector &rhs,
+                             DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /// Eliminate essential boundary DOFs from the system matrix.
+   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                             DiagonalPolicy dpolicy = DIAG_ONE);
+   /// Perform elimination and set the diagonal entry to the given value
+   void EliminateEssentialBCDiag(const Array<int> &bdr_attr_is_ess,
+                                 double value);
+
+   /// Eliminate the given @a vdofs.
+   /** NOTE: here, @a vdofs is a list of DOFs from all the fespaces
+       In this case the eliminations are applied to the internal \f$ M \f$
+       and @a rhs without storing the elimination matrix \f$ M_e \f$. */
+   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
+                       DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /// Eliminate the given @a vdofs (all the fespaces), storing the eliminated part internally in \f$ M_e \f$.
+   /** This method works in conjunction with EliminateVDofsInRHS() and allows
+       elimination of boundary conditions in multiple right-hand sides. In this
+       method, @a vdofs is a list of DOFs. */
+   void EliminateVDofs(const Array<int> &vdofs,
+                       DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /** @brief Similar to
+       EliminateVDofs(const Array<int> &, const Vector &, Vector &, DiagonalPolicy)
+       but here @a ess_dofs is a marker (boolean) array on all vector-dofs
+       (@a ess_dofs[i] < 0 is true). */
+   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, const Vector &sol,
+                                     Vector &rhs, DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /** @brief Similar to EliminateVDofs(const Array<int> &, DiagonalPolicy) but
+       here @a ess_dofs is a marker (boolean) array on all vector-dofs
+       (@a ess_dofs[i] < 0 is true). */
+   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs,
+                                     DiagonalPolicy dpolicy = DIAG_ONE);
+   /// Perform elimination and set the diagonal entry to the given value
+   void EliminateEssentialBCFromDofsDiag(const Array<int> &ess_dofs,
+                                         double value);
+
+   /** @brief Use the stored eliminated part of the matrix (see
+       EliminateVDofs(const Array<int> &, DiagonalPolicy)) to modify the r.h.s.
+       @a b; @a vdofs is a list of DOFs (non-directional, i.e. >= 0). */
+   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x,
+                            Vector &b);
+
+
+   /// Sets diagonal policy used upon construction of the linear system.
+   /** Policies include:
+
+       - DIAG_ZERO (Set the diagonal values to zero)
+       - DIAG_ONE  (Set the diagonal values to one)
+       - DIAG_KEEP (Keep the diagonal values)
+   */
+   void SetDiagonalPolicy(DiagonalPolicy policy)
+   {
+      diag_policy = policy;
+   }
+
+   /// Destroys bilinear form.
+   virtual ~BlockBilinearForm();
+
+};
+
+} // namespace mfem
+
+#endif

fem/blockinteg.cpp

@@ -0,0 +1,136 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void BlockBilinearFormIntegrator::AssembleElementMatrix(
+   const Array<const FiniteElement *> &el,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   mfem_error ("BlockBilinearFormIntegrator::AssembleElementMatrix\n"
+               "   is not implemented for this class.");
+}
+
+void BlockLinearFormIntegrator::AssembleRHSElementVect(
+   const Array<const FiniteElement *> &el,
+   ElementTransformation &Trans,
+   Vector &elvect)
+{
+   mfem_error ("BlockLinearFormIntegrator::AssembleElementVector\n"
+               "   is not implemented for this class.");
+}
+
+/** Given a particular Finite Element computes the element vector */
+void TestBlockBilinearFormIntegrator::AssembleElementMatrix
+(const Array<const FiniteElement *> &el,
+ ElementTransformation &Trans,
+ DenseMatrix &elmat)
+{
+   int nd = 0;
+   int nblocks = el.Size();
+   Array<int> offsets(nblocks+1);
+   offsets[0] = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      nd += el[i]->GetDof();
+      offsets[i+1] = el[i]->GetDof();
+   }
+   offsets.PartialSum();
+   elmat.SetSize(nd);
+   elmat = 0.0;
+   DenseMatrix dmat;
+
+   if (blfis.NumRows())
+   {
+      // Get the matrices directly from the existing BilinearFormIntegrators
+      for (int i = 0; i<nblocks; i++)
+      {
+         // mfem::out << "i = " << i << std::endl;
+         int offset_i = offsets[i];
+         const FiniteElement * fe_i = el[i];
+         for (int j = 0; j<nblocks; j++)
+         {
+            // mfem::out << "j = " << j << std::endl;
+            BilinearFormIntegrator * blfi = blfis(i,j);
+            if (!blfi) { continue; }
+            if (j == i)
+            {
+               blfi->AssembleElementMatrix(*fe_i,Trans,dmat);
+               // mfem::out << "j 1 = " << j << std::endl;
+               elmat.SetSubMatrix(offset_i,dmat);
+            }
+            else
+            {
+               const FiniteElement * fe_j = el[j];
+               blfi->AssembleElementMatrix2(*fe_j,*fe_i,Trans,dmat);
+               // mfem::out << "j 2 = " << j << std::endl;
+               int offset_j = offsets[j];
+               elmat.SetSubMatrix(offset_i,offset_j,dmat);
+            }
+         }
+      }
+      return;
+   }
+
+   // else compute the matrices
+   elmat = 25.0;
+   // TODO
+
+}
+
+/** Given a particular Finite Element computes the element vector */
+void TestBlockLinearFormIntegrator::AssembleRHSElementVect
+(const Array<const FiniteElement *> &el,
+ ElementTransformation &Trans,
+ Vector &elvector)
+{
+   int nd = 0;
+   int nblocks = el.Size();
+   Array<int> offsets(nblocks+1);
+   offsets[0] = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      nd += el[i]->GetDof();
+      offsets[i+1] = el[i]->GetDof();
+   }
+   offsets.PartialSum();
+   elvector.SetSize(nd);
+   elvector = 0.0;
+   Vector subvector;
+
+   if (lfis.Size())
+   {
+      // Get the matrices directly from the existing BilinearFormIntegrators
+      for (int i = 0; i<nblocks; i++)
+      {
+         int offset = offsets[i];
+         const FiniteElement * fe_i = el[i];
+         LinearFormIntegrator * lfi = lfis[i];
+         if (!lfi)
+         {
+            continue;
+         }
+         lfi->AssembleRHSElementVect(*fe_i,Trans,subvector);
+         elvector.SetVector(subvector,offset);
+      }
+      return;
+   }
+
+   // else, compute the block linear form integrator
+   // elvector = 1.0;
+   // TODO
+}
+
+} // namespace mfem

fem/blockinteg.hpp

@@ -0,0 +1,147 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKINTEG
+#define MFEM_BLOCKINTEG
+
+#include "../config/config.hpp"
+#include "fe.hpp"
+#include "coefficient.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+/** The abstract base class BlockBilinearFormIntegrator is
+ a generalization of the BilinearFormIntegrator class suitable
+ for block formulations. */
+class BlockBilinearFormIntegrator
+{
+protected:
+   const IntegrationRule *IntRule;
+   BlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : IntRule(ir) { }
+public:
+
+
+   /// Given a particular Finite Element computes the element matrix elmat.
+   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+
+   virtual ~BlockBilinearFormIntegrator() { }
+};
+
+/** The abstract base class BlockBilinearFormIntegrator is
+ a generalization of the BilinearFormIntegrator class suitable
+ for block formulations. */
+class BlockLinearFormIntegrator
+{
+protected:
+   const IntegrationRule *IntRule;
+   BlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : IntRule(ir) { }
+
+public:
+   /// Given a particular Finite Element computes the element matrix elmat.
+   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
+                                       ElementTransformation &Trans,
+                                       Vector &elvect);
+
+   virtual ~BlockLinearFormIntegrator() { }
+};
+
+
+
+
+class TestBlockBilinearFormIntegrator: public BlockBilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   Array<const FiniteElementSpace * > fespaces;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq, dofs1D, quad1D;
+
+   Array2D<BilinearFormIntegrator *> blfis;
+
+
+public:
+
+   TestBlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : BlockBilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
+
+   /// Construct a mass integrator with coefficient q
+   TestBlockBilinearFormIntegrator(Coefficient &q,
+                                   const IntegrationRule *ir = NULL)
+      : BlockBilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL)   { }
+
+
+   TestBlockBilinearFormIntegrator(Array2D<BilinearFormIntegrator *> blfis_)
+      : BlockBilinearFormIntegrator(NULL), blfis(blfis_) { }
+
+   void SetIntegrators(Array2D<BilinearFormIntegrator *> blfis_)
+   {
+      blfis = blfis_;
+   }
+
+
+   /** Given a particular Finite Element computes the element matrix
+       elmat. */
+   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+
+   virtual ~TestBlockBilinearFormIntegrator() { }
+
+};
+
+/** Class for local vector assembly */
+class TestBlockLinearFormIntegrator: public BlockLinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   Array<const FiniteElementSpace * > fespaces;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq, dofs1D, quad1D;
+   Array<LinearFormIntegrator *> lfis;
+
+public:
+
+   TestBlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : BlockLinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
+
+   /// Construct a test linear integrator with coefficient q
+   TestBlockLinearFormIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
+      : BlockLinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL) { }
+
+
+   TestBlockLinearFormIntegrator(Array<LinearFormIntegrator *> lfis_)
+      : BlockLinearFormIntegrator(NULL), lfis(lfis_) { }
+
+   void SetIntegrators(Array<LinearFormIntegrator *> lfis_)
+   {
+      lfis = lfis_;
+   }
+
+   /** Given a particular Finite Element computes the element vector */
+   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
+                                       ElementTransformation &Trans,
+                                       Vector &elvector);
+
+
+};
+
+
+} // namespace mfem
+
+
+#endif

fem/blocklinearform.cpp

@@ -0,0 +1,123 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+BlockLinearForm::BlockLinearForm(Array<FiniteElementSpace * > & fespaces_) :
+   Vector(0), fespaces(fespaces_)
+{
+   int s = 0;
+   int nblocks = fespaces.Size();
+   for (int i =0; i<nblocks; i++)
+   {
+      s += fespaces[i]->GetVSize();
+   }
+   // mfem::out << "size = " << size << std::endl;
+
+   SetSize(s);
+
+}
+
+
+void BlockLinearForm::AddDomainIntegrator(BlockLinearFormIntegrator *lfi)
+{
+   domain_integs.Append(lfi);
+}
+
+void BlockLinearForm::Assemble()
+{
+   ElementTransformation *eltrans;
+   DofTransformation *doftrans;
+   Mesh *mesh = fespaces[0] -> GetMesh();
+   Vector subvect,elvect, *elvect_p;
+
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   Array<int> offsetvdofs;
+   Array<int> elementblockoffsets(nblocks+1);
+   elementblockoffsets[0] = 0;
+   Array<int> blockoffsets(nblocks+1);
+   blockoffsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      blockoffsets[i+1] = fespaces[i]->GetVSize();
+   }
+   blockoffsets.PartialSum();
+
+   Vector::operator=(0.0);
+
+   if (domain_integs.Size())
+   {
+      // loop through elements
+      for (int i = 0; i < mesh -> GetNE(); i++)
+      {
+         elvect.SetSize(0);
+         for (int k = 0; k < domain_integs.Size(); k++)
+         {
+            for (int j = 0; j<nblocks; j++)
+            {
+               fe[j] = fespaces[j]->GetFE(i);
+               elementblockoffsets[j+1] = fe[j]->GetDof();
+            }
+            elementblockoffsets.PartialSum();
+            eltrans = mesh->GetElementTransformation(i);
+
+            domain_integs[k]->AssembleRHSElementVect(fe, *eltrans, elemvect);
+            if (elvect.Size() == 0)
+            {
+               elvect = elemvect;
+            }
+            else
+            {
+               elvect += elemvect;
+            }
+         }
+         if (elvect.Size() == 0)
+         {
+            continue;
+         }
+         else
+         {
+            elvect_p = &elvect;
+         }
+
+         double *data = elvect_p->GetData();
+
+         for (int j = 0; j<nblocks; j++)
+         {
+            doftrans = fespaces[j]->GetElementVDofs(i, vdofs);
+            int offset = blockoffsets[j];
+            offsetvdofs.SetSize(vdofs.Size());
+            for (int l = 0; l<vdofs.Size(); l++)
+            {
+               offsetvdofs[l] = vdofs[l]<0 ? -offset + vdofs[l]
+                                :  offset + vdofs[l];
+            }
+            int jbeg = elementblockoffsets[j];
+            int jend = elementblockoffsets[j+1]-1;
+            subvect.SetSize(jend-jbeg+1);
+            subvect.SetData(&data[jbeg]);
+
+            if (doftrans)
+            {
+               doftrans->TransformDual(subvect);
+            }
+            AddElementVector(offsetvdofs,subvect);
+         }
+      }
+   }
+}
+
+
+} // name space mfem

fem/blocklinearform.hpp

@@ -0,0 +1,49 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKLINEARFORM
+#define MFEM_BLOCKLINEARFORM
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+
+namespace mfem
+{
+
+
+class BlockLinearForm : public Vector
+{
+protected:
+   /// FE spaces on which the LinearForm lives. Not owned.
+   Array<FiniteElementSpace * > fespaces;
+
+   /// Set of Domain Integrators to be applied.
+   Array<BlockLinearFormIntegrator*> domain_integs;
+
+   Vector elemvect;
+   Array<int> vdofs;
+
+public:
+   BlockLinearForm(Array<FiniteElementSpace * > & fespaces_);
+
+   /// Adds new Domain Integrator. Assumes ownership of @a lfi.
+   void AddDomainIntegrator(BlockLinearFormIntegrator *lfi);
+
+   /// Assembles the block linear form i.e. sums over all domain integrators.
+   void Assemble();
+
+
+};
+
+} // namespace mfem
+
+
+#endif

fem/blockstaticcond.cpp

@@ -0,0 +1,970 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "blockstaticcond.hpp"
+
+namespace mfem
+{
+
+
+BlockStaticCondensation::BlockStaticCondensation(Array<FiniteElementSpace *> &
+                                                 fes_)
+{
+   SetSpaces(fes_);
+
+   Array<int> rvdofs;
+   Array<int> vdofs;
+   Array<int> rdof_edof0;
+   for (int k = 0; k<nblocks; k++)
+   {
+      if (!tr_fes[k]) { continue; }
+      rdof_edof0.SetSize(tr_fes[k]->GetVSize());
+      for (int i = 0; i < mesh->GetNE(); i++)
+      {
+         fes[k]->GetElementVDofs(i, vdofs);
+         tr_fes[k]->GetElementVDofs(i, rvdofs);
+         const int vdim = fes[k]->GetVDim();
+         const int nsd = vdofs.Size()/vdim;
+         const int nsrd = rvdofs.Size()/vdim;
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            for (int j = 0; j < nsrd; j++)
+            {
+               int rvdof = rvdofs[j+nsrd*vd];
+               int vdof = vdofs[j+nsd*vd];
+               if (rvdof < 0)
+               {
+                  rvdof = -1-rvdof;
+                  vdof = -1-vdof;
+               }
+               MFEM_ASSERT(vdof >= 0, "incompatible volume and trace FE spaces");
+               rdof_edof0[rvdof] = vdof + dof_offsets[k];
+            }
+         }
+      }
+      rdof_edof.Append(rdof_edof0);
+   }
+}
+
+void BlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> & fes_)
+{
+#ifdef MFEM_USE_MPI
+   ParMesh *pmesh = nullptr;
+   parallel = false;
+   if (dynamic_cast<ParFiniteElementSpace *>(fes_[0]))
+   {
+      parallel = true;
+   }
+#else
+   parallel = false;
+#endif
+   fes=fes_;
+   nblocks = fes.Size();
+   rblocks = 0;
+   tr_fes.SetSize(nblocks);
+   mesh = fes[0]->GetMesh();
+
+   IsTraceSpace.SetSize(nblocks);
+   const FiniteElementCollection * fec;
+   for (int i = 0; i < nblocks; i++)
+   {
+      fec = fes[i]->FEColl();
+      IsTraceSpace[i] =
+         (dynamic_cast<const H1_Trace_FECollection*>(fec) ||
+          dynamic_cast<const ND_Trace_FECollection*>(fec) ||
+          dynamic_cast<const RT_Trace_FECollection*>(fec));
+#ifdef MFEM_USE_MPI
+      if (parallel)
+      {
+         pmesh = dynamic_cast<ParMesh *>(mesh);
+         tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
+                     nullptr : (IsTraceSpace[i]) ? fes[i] :
+                     new ParFiniteElementSpace(pmesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                                               fes[i]->GetOrdering());
+      }
+      else
+      {
+         tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
+                     nullptr : (IsTraceSpace[i]) ? fes[i] :
+                     new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                                            fes[i]->GetOrdering());
+      }
+#else
+      // skip if it's an L2 space (no trace space to construct)
+      tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
+                  nullptr : (IsTraceSpace[i]) ? fes[i] :
+                  new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                                         fes[i]->GetOrdering());
+#endif
+      if (tr_fes[i]) { rblocks++; }
+   }
+   if (parallel)
+   {
+      ess_tdofs.SetSize(rblocks);
+      for (int i = 0; i<rblocks; i++)
+      {
+         ess_tdofs[i] = new Array<int>();
+      }
+   }
+   Init();
+}
+
+void BlockStaticCondensation::ComputeOffsets()
+{
+   dof_offsets.SetSize(nblocks+1);
+   tdof_offsets.SetSize(nblocks+1);
+   dof_offsets[0] = 0;
+   tdof_offsets[0] = 0;
+
+   rdof_offsets.SetSize(rblocks+1);
+   rtdof_offsets.SetSize(rblocks+1);
+   rdof_offsets[0] = 0;
+   rtdof_offsets[0] = 0;
+
+   int j=0;
+   for (int i =0; i<nblocks; i++)
+   {
+      dof_offsets[i+1] = fes[i]->GetVSize();
+      tdof_offsets[i+1] = fes[i]->GetTrueVSize();
+      if (tr_fes[i])
+      {
+         rdof_offsets[j+1] = tr_fes[i]->GetVSize();
+         rtdof_offsets[j+1] = tr_fes[i]->GetTrueVSize();
+         j++;
+      }
+   }
+   rdof_offsets.PartialSum();
+   rtdof_offsets.PartialSum();
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+}
+
+
+void BlockStaticCondensation::Init()
+{
+   lmat.SetSize(mesh->GetNE());
+   lvec.SetSize(mesh->GetNE());
+   for (int i = 0; i < mesh->GetNE(); i++)
+   {
+      lmat[i] = nullptr;
+      lvec[i] = nullptr;
+   }
+
+   ComputeOffsets();
+
+   S = new BlockMatrix(rdof_offsets);
+   S->owns_blocks = 1;
+
+   for (int i = 0; i<S->NumRowBlocks(); i++)
+   {
+      int h = rdof_offsets[i+1] - rdof_offsets[i];
+      for (int j = 0; j<S->NumColBlocks(); j++)
+      {
+         int w = rdof_offsets[j+1] - rdof_offsets[j];
+         S->SetBlock(i,j,new SparseMatrix(h, w));
+      }
+   }
+   y = new BlockVector(rdof_offsets);
+   *y = 0.;
+}
+
+void BlockStaticCondensation::GetReduceElementIndicesAndOffsets(int el,
+                                                                Array<int> & trace_ldofs,
+                                                                Array<int> & interior_ldofs,
+                                                                Array<int> & offsets) const
+{
+   int dim = mesh->Dimension();
+   offsets.SetSize(tr_fes.Size()+1); offsets = 0;
+   Array<int> dofs;
+   Array<int> faces, ori;
+   if (dim == 1)
+   {
+      mesh->GetElementVertices(el, faces);
+   }
+   if (dim == 2)
+   {
+      mesh->GetElementEdges(el, faces, ori);
+   }
+   else //dim = 3
+   {
+      mesh->GetElementFaces(el,faces,ori);
+   }
+   int numfaces = faces.Size();
+
+   trace_ldofs.SetSize(0);
+   interior_ldofs.SetSize(0);
+   // construct Array of bubble dofs to be extracted
+   int skip=0;
+   Array<int> tr_dofs;
+   Array<int> int_dofs;
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      int td = 0;
+      int ndof;
+      // if it's an L2 space (bubbles)
+      if (!tr_fes[i])
+      {
+         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
+         td = 0;
+      }
+      else if (IsTraceSpace[i])
+      {
+         for (int iface = 0; iface < numfaces; iface++)
+         {
+            td += fes[i]->GetVDim()*fes[i]->GetFaceElement(faces[iface])->GetDof();
+         }
+         ndof = td;
+      }
+      else
+      {
+         Array<int> trace_dofs;
+         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
+         tr_fes[i]->GetElementVDofs(el, trace_dofs);
+         td = trace_dofs.Size(); // number of trace dofs
+      }
+      offsets[i+1] = td;
+      tr_dofs.SetSize(td);
+      int_dofs.SetSize(ndof - td);
+      for (int j = 0; j<td; j++)
+      {
+         tr_dofs[j] = skip + j;
+      }
+      for (int j = 0; j<ndof-td; j++)
+      {
+         int_dofs[j] = skip + td + j;
+      }
+      skip+=ndof;
+
+      trace_ldofs.Append(tr_dofs);
+      interior_ldofs.Append(int_dofs);
+   }
+   offsets.PartialSum();
+}
+
+
+void BlockStaticCondensation::GetReduceElementVDofs(int el,
+                                                    Array<int> & rdofs) const
+{
+   Array<int> faces, ori;
+   int dim = mesh->Dimension();
+   if (dim == 1)
+   {
+      mesh->GetElementVertices(el, faces);
+   }
+   if (dim == 2)
+   {
+      mesh->GetElementEdges(el, faces, ori);
+   }
+   else //dim = 3
+   {
+      mesh->GetElementFaces(el,faces,ori);
+   }
+   int numfaces = faces.Size();
+   rdofs.SetSize(0);
+   int skip = 0;
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      Array<int> vdofs;
+      if (IsTraceSpace[i])
+      {
+         Array<int> face_vdofs;
+         for (int k = 0; k < numfaces; k++)
+         {
+            int iface = faces[k];
+            tr_fes[i]->GetFaceVDofs(iface, face_vdofs);
+            vdofs.Append(face_vdofs);
+         }
+      }
+      else
+      {
+         tr_fes[i]->GetElementVDofs(el, vdofs);
+      }
+      for (int j=0; j<vdofs.Size(); j++)
+      {
+         vdofs[j] = (vdofs[j]>=0) ? vdofs[j]+rdof_offsets[skip] :
+                    vdofs[j]-rdof_offsets[skip];
+      }
+      skip++;
+      rdofs.Append(vdofs);
+   }
+}
+void BlockStaticCondensation::GetElementVDofs(int el, Array<int> & vdofs) const
+{
+   Array<int> faces, ori;
+   int dim = mesh->Dimension();
+   if (dim == 1)
+   {
+      mesh->GetElementVertices(el, faces);
+   }
+   if (dim == 2)
+   {
+      mesh->GetElementEdges(el, faces, ori);
+   }
+   else //dim = 3
+   {
+      mesh->GetElementFaces(el,faces,ori);
+   }
+   int numfaces = faces.Size();
+   vdofs.SetSize(0);
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      Array<int> dofs;
+      if (IsTraceSpace[i])
+      {
+         Array<int> face_vdofs;
+         for (int k = 0; k < numfaces; k++)
+         {
+            int iface = faces[k];
+            fes[i]->GetFaceVDofs(iface, face_vdofs);
+            dofs.Append(face_vdofs);
+         }
+      }
+      else
+      {
+         fes[i]->GetElementVDofs(el, dofs);
+      }
+      for (int j=0; j<dofs.Size(); j++)
+      {
+         dofs[j] = (dofs[j]>=0) ? dofs[j]+dof_offsets[i] :
+                   dofs[j]-dof_offsets[i];
+      }
+      vdofs.Append(dofs);
+   }
+}
+
+
+void BlockStaticCondensation::GetLocalShurComplement(int el,
+                                                     const Array<int> & tr_idx, const Array<int> & int_idx,
+                                                     const DenseMatrix & elmat, const Vector & elvect,
+                                                     DenseMatrix & rmat, Vector & rvect)
+{
+   int rdofs = tr_idx.Size();
+   int idofs = int_idx.Size();
+   MFEM_VERIFY(idofs != 0, "Number of interior dofs is zero");
+   MFEM_VERIFY(rdofs != 0, "Number of interface dofs is zero");
+
+   rmat.SetSize(rdofs);
+   rvect.SetSize(rdofs);
+
+   DenseMatrix A_tt, A_ti, A_it, A_ii;
+   Vector y_t, y_i;
+
+   elmat.GetSubMatrix(tr_idx,A_tt);
+   elmat.GetSubMatrix(tr_idx,int_idx, A_ti);
+   elmat.GetSubMatrix(int_idx, tr_idx, A_it);
+   elmat.GetSubMatrix(int_idx, A_ii);
+
+   elvect.GetSubVector(tr_idx, y_t);
+   elvect.GetSubVector(int_idx, y_i);
+
+   DenseMatrixInverse lu(A_ii);
+   lu.Factor();
+   lmat[el] = new DenseMatrix(idofs,rdofs);
+   lvec[el] = new Vector(idofs);
+
+   lu.Mult(A_it,*lmat[el]);
+   lu.Mult(y_i,*lvec[el]);
+
+   // LHS
+   mfem::Mult(A_ti,*lmat[el],rmat);
+
+   rmat.Neg();
+   rmat.Add(1., A_tt);
+
+   // RHS
+   A_ti.Mult(*lvec[el], rvect);
+   rvect.Neg();
+   rvect.Add(1., y_t);
+}
+
+
+void BlockStaticCondensation::AssembleReducedSystem(int el,
+                                                    DenseMatrix &elmat,
+                                                    Vector & elvect)
+{
+   // Get Shur Complement
+   Array<int> tr_idx, int_idx;
+   Array<int> offsets;
+   // Get local element idx and offsets for global assembly
+   GetReduceElementIndicesAndOffsets(el, tr_idx,int_idx, offsets);
+
+   DenseMatrix rmat, *rmatptr;
+   Vector rvec, *rvecptr;
+   // Extract the reduced matrices based on tr_idx and int_idx
+   if (int_idx.Size()!=0)
+   {
+      GetLocalShurComplement(el,tr_idx,int_idx, elmat, elvect, rmat, rvec);
+      rmatptr = &rmat;
+      rvecptr = &rvec;
+   }
+   else
+   {
+      rmatptr = &elmat;
+      rvecptr = &elvect;
+   }
+
+   // Assemble global mat and rhs
+   DofTransformation * doftrans_i, *doftrans_j;
+
+
+   Array<int> faces, ori;
+   int dim = mesh->Dimension();
+   if (dim == 1)
+   {
+      mesh->GetElementVertices(el, faces);
+   }
+   if (dim == 2)
+   {
+      mesh->GetElementEdges(el, faces, ori);
+   }
+   else //dim = 3
+   {
+      mesh->GetElementFaces(el,faces,ori);
+   }
+   int numfaces = faces.Size();
+
+   int skip_i=0;
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      Array<int> vdofs_i;
+      doftrans_i = nullptr;
+      if (IsTraceSpace[i])
+      {
+         Array<int> face_vdofs;
+         for (int k = 0; k < numfaces; k++)
+         {
+            int iface = faces[k];
+            tr_fes[i]->GetFaceVDofs(iface, face_vdofs);
+            vdofs_i.Append(face_vdofs);
+         }
+      }
+      else
+      {
+         doftrans_i = tr_fes[i]->GetElementVDofs(el, vdofs_i);
+      }
+      int skip_j=0;
+      for (int j = 0; j<tr_fes.Size(); j++)
+      {
+         if (!tr_fes[j]) { continue; }
+         Array<int> vdofs_j;
+         doftrans_j = nullptr;
+
+         if (IsTraceSpace[j])
+         {
+            Array<int> face_vdofs;
+            for (int k = 0; k < numfaces; k++)
+            {
+               int iface = faces[k];
+               tr_fes[j]->GetFaceVDofs(iface, face_vdofs);
+               vdofs_j.Append(face_vdofs);
+            }
+         }
+         else
+         {
+            doftrans_j = tr_fes[j]->GetElementVDofs(el, vdofs_j);
+         }
+
+         DenseMatrix Ae;
+         rmatptr->GetSubMatrix(offsets[i],offsets[i+1],
+                               offsets[j],offsets[j+1], Ae);
+         if (doftrans_i || doftrans_j)
+         {
+            TransformDual(doftrans_i, doftrans_j, Ae);
+         }
+         S->GetBlock(skip_i,skip_j).AddSubMatrix(vdofs_i,vdofs_j, Ae);
+         skip_j++;
+      }
+
+      // assemble rhs
+      double * data = rvecptr->GetData();
+      Vector vec1;
+      // ref subvector
+      vec1.SetDataAndSize(&data[offsets[i]],
+                          offsets[i+1]-offsets[i]);
+      if (doftrans_i)
+      {
+         doftrans_i->TransformDual(vec1);
+      }
+      y->GetBlock(skip_i).AddElementVector(vdofs_i,vec1);
+      skip_i++;
+   }
+}
+
+void BlockStaticCondensation::BuildProlongation()
+{
+   P = new BlockMatrix(rdof_offsets, rtdof_offsets);
+   R = new BlockMatrix(rtdof_offsets, rdof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+   int skip = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      const SparseMatrix *P_ = tr_fes[i]->GetConformingProlongation();
+      if (P_)
+      {
+         const SparseMatrix *R_ = tr_fes[i]->GetRestrictionMatrix();
+         P->SetBlock(skip,skip,const_cast<SparseMatrix*>(P_));
+         R->SetBlock(skip,skip,const_cast<SparseMatrix*>(R_));
+      }
+      skip++;
+   }
+}
+
+#ifdef MFEM_USE_MPI
+void BlockStaticCondensation::BuildParallelProlongation()
+{
+   MFEM_VERIFY(parallel, "BuildParallelProlongation: wrong code path");
+   pP = new BlockOperator(rdof_offsets, rtdof_offsets);
+   R = new BlockMatrix(rtdof_offsets, rdof_offsets);
+   pP->owns_blocks = 0;
+   R->owns_blocks = 0;
+   int skip = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      const HypreParMatrix *P_ =
+         dynamic_cast<ParFiniteElementSpace *>(tr_fes[i])->Dof_TrueDof_Matrix();
+      if (P_)
+      {
+         const SparseMatrix *R_ = tr_fes[i]->GetRestrictionMatrix();
+         pP->SetBlock(skip,skip,const_cast<HypreParMatrix*>(P_));
+         R->SetBlock(skip,skip,const_cast<SparseMatrix*>(R_));
+      }
+      skip++;
+   }
+}
+
+void BlockStaticCondensation::ParallelAssemble(BlockMatrix *m)
+{
+   if (!pP) { BuildParallelProlongation(); }
+
+   pS = new BlockOperator(rtdof_offsets);
+   pS_e = new BlockOperator(rtdof_offsets);
+   pS->owns_blocks = 1;
+   pS_e->owns_blocks = 1;
+   HypreParMatrix * A = nullptr;
+   HypreParMatrix * PtAP = nullptr;
+   int skip_i=0;
+   ParFiniteElementSpace * pfes_i = nullptr;
+   ParFiniteElementSpace * pfes_j = nullptr;
+   for (int i = 0; i<nblocks; i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      pfes_i = dynamic_cast<ParFiniteElementSpace*>(fes[i]);
+      HypreParMatrix * Pi = (HypreParMatrix*)(&pP->GetBlock(skip_i,skip_i));
+      int skip_j=0;
+      for (int j = 0; j<nblocks; j++)
+      {
+         if (!tr_fes[j]) { continue; }
+         if (m->IsZeroBlock(skip_i,skip_j)) { continue; }
+         if (skip_i == skip_j)
+         {
+            // Make block diagonal square hypre matrix
+            A = new HypreParMatrix(pfes_i->GetComm(), pfes_i->GlobalVSize(),
+                                   pfes_i->GetDofOffsets(),&m->GetBlock(skip_i,skip_i));
+            PtAP = RAP(A,Pi);
+            delete A;
+            pS_e->SetBlock(skip_i,skip_i,PtAP->EliminateRowsCols(*ess_tdofs[skip_i]));
+         }
+         else
+         {
+            pfes_j = dynamic_cast<ParFiniteElementSpace*>(fes[j]);
+            HypreParMatrix * Pj = (HypreParMatrix*)(&pP->GetBlock(skip_j,skip_j));
+            A = new HypreParMatrix(pfes_i->GetComm(), pfes_i->GlobalVSize(),
+                                   pfes_j->GlobalVSize(), pfes_i->GetDofOffsets(),
+                                   pfes_j->GetDofOffsets(), &m->GetBlock(skip_i,skip_j));
+            PtAP = RAP(Pi,A,Pj);
+            delete A;
+            pS_e->SetBlock(skip_i,skip_j,PtAP->EliminateCols(*ess_tdofs[skip_j]));
+            PtAP->EliminateRows(*ess_tdofs[skip_i]);
+         }
+         pS->SetBlock(skip_i,skip_j,PtAP);
+         skip_j++;
+      }
+      skip_i++;
+   }
+}
+
+#endif
+
+
+void BlockStaticCondensation::ConformingAssemble(int skip_zeros)
+{
+   Finalize(0);
+   if (!P) { BuildProlongation(); }
+
+   BlockMatrix * Pt = Transpose(*P);
+   BlockMatrix * PtA = mfem::Mult(*Pt, *S);
+   delete S;
+   if (S_e)
+   {
+      BlockMatrix *PtAe = mfem::Mult(*Pt, *S_e);
+      delete S_e;
+      S_e = PtAe;
+   }
+   delete Pt;
+   S = mfem::Mult(*PtA, *P);
+   delete PtA;
+
+   if (S_e)
+   {
+      BlockMatrix *PtAeP = mfem::Mult(*S_e, *P);
+      S_e = PtAeP;
+   }
+   height = S->Height();
+   width = S->Width();
+}
+
+void BlockStaticCondensation::Finalize(int skip_zeros)
+{
+   if (S) { S->Finalize(skip_zeros); }
+   if (S_e) { S_e->Finalize(skip_zeros); }
+}
+
+void BlockStaticCondensation::FormSystemMatrix(Operator::DiagonalPolicy
+                                               diag_policy)
+{
+   if (parallel)
+   {
+      FillEssTdofLists(ess_rtdof_list);
+      if (S)
+      {
+         const int remove_zeros = 0;
+         Finalize(remove_zeros);
+         ParallelAssemble(S);
+         delete S;
+         S=nullptr;
+         delete S_e;
+         S_e = nullptr;
+      }
+   }
+   else
+   {
+      if (!S_e)
+      {
+         bool conforming = true;
+         for (int i = 0; i<nblocks; i++)
+         {
+            if (!tr_fes[i]) { continue; }
+            const SparseMatrix *P_ = tr_fes[i]->GetConformingProlongation();
+            if (P_)
+            {
+               conforming = false;
+               break;
+            }
+         }
+         if (!conforming) { ConformingAssemble(0); }
+         const int remove_zeros = 0;
+         EliminateReducedTrueDofs(ess_rtdof_list, diag_policy);
+         Finalize(remove_zeros);
+      }
+   }
+}
+
+
+void BlockStaticCondensation::ConvertMarkerToReducedTrueDofs(
+   Array<int> & tdof_marker,
+   Array<int> & rtdof_marker)
+{
+   // convert tdof_marker to dof_marker
+   rtdof_marker.SetSize(0);
+   Array<int> tdof_marker0;
+   Array<int> dof_marker0;
+   Array<int> dof_marker;
+   int * data = tdof_marker.GetData();
+   for (int i = 0; i<nblocks; i++)
+   {
+      tdof_marker0.MakeRef(&data[tdof_offsets[i]],tdof_offsets[i+1]-tdof_offsets[i]);
+      const SparseMatrix * R = fes[i]->GetRestrictionMatrix();
+      if (!R)
+      {
+         dof_marker0.MakeRef(tdof_marker0);
+      }
+      else
+      {
+         dof_marker0.SetSize(fes[i]->GetVSize());
+         R->BooleanMultTranspose(tdof_marker0, dof_marker0);
+      }
+      dof_marker.Append(dof_marker0);
+   }
+
+   int rdofs = rdof_edof.Size();
+   Array<int> rdof_marker(rdofs);
+
+   for (int i = 0; i < rdofs; i++)
+   {
+      rdof_marker[i] = dof_marker[rdof_edof[i]];
+   }
+
+   // convert rdof_marker to rtdof_marker
+   Array<int> rtdof_marker0;
+   Array<int> rdof_marker0;
+   int * rdata = rdof_marker.GetData();
+   int k=0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      rdof_marker0.MakeRef(&rdata[rdof_offsets[k]],rdof_offsets[k+1]-rdof_offsets[k]);
+      const SparseMatrix *tr_R = tr_fes[i]->GetRestrictionMatrix();
+      if (!tr_R)
+      {
+         rtdof_marker0.MakeRef(rdof_marker0);
+      }
+      else
+      {
+         rtdof_marker0.SetSize(tr_fes[i]->GetTrueVSize());
+         tr_R->BooleanMult(rdof_marker0, rtdof_marker0);
+      }
+      rtdof_marker.Append(rtdof_marker0);
+      k++;
+   }
+}
+
+void BlockStaticCondensation::FillEssTdofLists(const Array<int> & ess_tdof_list)
+{
+   int j;
+   for (int i = 0; i<ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+      for (j = 0; j < rblocks; j++)
+      {
+         if (rtdof_offsets[j+1] > tdof) { break; }
+      }
+      ess_tdofs[j]->Append(tdof-rtdof_offsets[j]);
+   }
+}
+
+void BlockStaticCondensation::SetEssentialTrueDofs(const Array<int>
+                                                   &ess_tdof_list)
+{
+   Array<int> tdof_marker;
+   Array<int> rtdof_marker;
+   FiniteElementSpace::ListToMarker(ess_tdof_list,tdof_offsets.Last(),tdof_marker);
+   ConvertMarkerToReducedTrueDofs(tdof_marker, rtdof_marker);
+   FiniteElementSpace::MarkerToList(rtdof_marker,ess_rtdof_list);
+}
+
+void BlockStaticCondensation::EliminateReducedTrueDofs(const Array<int>
+                                                       &ess_rtdof_list,
+                                                       Matrix::DiagonalPolicy dpolicy)
+{
+
+   MFEM_VERIFY(!parallel, "EliminateReducedTrueDofs::not implemented yet");
+
+
+   if (S_e == NULL)
+   {
+      Array<int> offsets;
+
+      offsets.MakeRef( (P) ? rtdof_offsets : rdof_offsets);
+
+      S_e = new BlockMatrix(offsets);
+      S_e->owns_blocks = 1;
+      for (int i = 0; i<S_e->NumRowBlocks(); i++)
+      {
+         int h = offsets[i+1] - offsets[i];
+         for (int j = 0; j<S_e->NumColBlocks(); j++)
+         {
+            int w = offsets[j+1] - offsets[j];
+            S_e->SetBlock(i,j,new SparseMatrix(h, w));
+         }
+      }
+   }
+   S->EliminateRowCols(ess_rtdof_list,S_e,dpolicy);
+}
+
+void BlockStaticCondensation::EliminateReducedTrueDofs(Matrix::DiagonalPolicy
+                                                       dpolicy)
+{
+   EliminateReducedTrueDofs(ess_rtdof_list, dpolicy);
+}
+
+void BlockStaticCondensation::ReduceSolution(const Vector &sol,
+                                             Vector &sc_sol) const
+{
+   MFEM_ASSERT(sol.Size() == dof_offsets.Last(), "'sol' has incorrect size");
+   const int nrdofs = rdof_offsets.Last();
+   Vector sol_r;
+   if (!R)
+   {
+      sc_sol.SetSize(nrdofs);
+      sol_r.SetDataAndSize(sc_sol.GetData(), sc_sol.Size());
+   }
+   else
+   {
+      sol_r.SetSize(nrdofs);
+   }
+   for (int i = 0; i < nrdofs; i++)
+   {
+      sol_r(i) = sol(rdof_edof[i]);
+   }
+   if (R)
+   {
+      // wrap vector into a block vector
+      BlockVector blsol_r(sol_r,rdof_offsets);
+      sc_sol.SetSize(R->Height());
+      R->Mult(blsol_r, sc_sol);
+   }
+}
+
+void BlockStaticCondensation::ReduceSystem(Vector &x, Vector &X,
+                                           Vector &B,
+                                           int copy_interior) const
+{
+   ReduceSolution(x, X);
+   if (parallel)
+   {
+      B.SetSize(pP->Width());
+      pP->MultTranspose(*y,B);
+
+      Vector tmp(B.Size());
+      pS_e->Mult(X,tmp);
+      B-=tmp;
+      for (int j = 0; j<rblocks; j++)
+      {
+         if (!ess_tdofs[j]->Size()) { continue; }
+         HypreParMatrix *Ah = (HypreParMatrix *)(&pS->GetBlock(j,j));
+         Vector diag;
+         Ah->GetDiag(diag);
+         for (int i = 0; i < ess_tdofs[j]->Size(); i++)
+         {
+            int tdof = (*ess_tdofs[j])[i];
+            int gdof = tdof + rtdof_offsets[j];
+            B(gdof) = diag(tdof)*X(gdof);
+         }
+      }
+   }
+   else
+   {
+      if (!P)
+      {
+         S_e->AddMult(X,*y,-1.);
+         S->PartMult(ess_rtdof_list,X,*y);
+         B.MakeRef(*y, 0, y->Size());
+      }
+      else
+      {
+         B.SetSize(P->Width());
+         P->MultTranspose(*y, B);
+         S_e->AddMult(X,B,-1.);
+         S->PartMult(ess_rtdof_list,X,B);
+      }
+   }
+   if (!copy_interior) { X.SetSubVectorComplement(ess_rtdof_list, 0.0); }
+}
+
+
+void BlockStaticCondensation::ComputeSolution(const Vector &sc_sol,
+                                              Vector &sol) const
+{
+
+   const int nrdofs = rdof_offsets.Last();
+   const int nrtdofs = rtdof_offsets.Last();
+   MFEM_VERIFY(sc_sol.Size() == nrtdofs, "'sc_sol' has incorrect size");
+
+   Vector sol_r;
+   if (parallel)
+   {
+      sol_r.SetSize(nrdofs);
+      pP->Mult(sc_sol, sol_r);
+   }
+   else
+   {
+      if (!P)
+      {
+         sol_r.SetDataAndSize(sc_sol.GetData(), sc_sol.Size());
+      }
+      else
+      {
+         sol_r.SetSize(nrdofs);
+         P->Mult(sc_sol, sol_r);
+      }
+   }
+
+
+   if (rdof_offsets.Last() == dof_offsets.Last())
+   {
+      sol = sol_r;
+      return;
+   }
+   else
+   {
+      sol.SetSize(dof_offsets.Last());
+   }
+
+   Vector lsr; // element (local) sc solution vector
+   Vector lsi; // element (local) interior solution vector
+   const int NE = mesh->GetNE();
+
+   Array<int> trace_vdofs;
+   Array<int> vdofs;
+   Array<int> tr_offsets;
+   Vector lsol;
+   for (int iel = 0; iel < NE; iel++)
+   {
+      lsol.SetSize(lmat[iel]->Width() + lmat[iel]->Height());
+      // GetReduceElementIndicesAndOffsets(iel, trace_ldofs, interior_ldofs, tr_offsets);
+      GetReduceElementVDofs(iel, trace_vdofs);
+
+      lsr.SetSize(trace_vdofs.Size());
+      sol_r.GetSubVector(trace_vdofs, lsr);
+      // complete the interior dofs
+
+      lsi.SetSize(lmat[iel]->Height());
+      lmat[iel]->Mult(lsr,lsi);
+      lsi.Neg();
+      lsi+=*lvec[iel];
+
+      Array<int> tr_idx,int_idx,idx_offs;
+      GetReduceElementIndicesAndOffsets(iel,tr_idx, int_idx, idx_offs);
+      lsol.SetSubVector(tr_idx,lsr);
+
+      lsol.SetSubVector(int_idx,lsi);
+
+      GetElementVDofs(iel, vdofs);
+      sol.SetSubVector(vdofs,lsol);
+
+   }
+
+}
+
+BlockStaticCondensation::~BlockStaticCondensation()
+{
+   delete S_e; S_e = nullptr;
+   delete S; S=nullptr;
+   delete y; y=nullptr;
+
+   if (P) { delete P; } P=nullptr;
+   if (R) { delete R; } R=nullptr;
+
+   if (parallel)
+   {
+      delete pS; pS=nullptr;
+      delete pS_e; pS_e=nullptr;
+      for (int i = 0; i<rblocks; i++)
+      {
+         delete ess_tdofs[i];
+      }
+      delete pP; pP=nullptr;
+   }
+
+   for (int i=0; i<lmat.Size(); i++)
+   {
+      delete lmat[i]; lmat[i] = nullptr;
+      delete lvec[i]; lvec[i] = nullptr;
+   }
+}
+
+}
+

fem/blockstaticcond.hpp

@@ -0,0 +1,191 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCK_STATIC_CONDENSATION
+#define MFEM_BLOCK_STATIC_CONDENSATION
+
+#include "../config/config.hpp"
+#include "fespace.hpp"
+
+#ifdef MFEM_USE_MPI
+#include "pfespace.hpp"
+#endif
+
+namespace mfem
+{
+
+
+class BlockStaticCondensation
+{
+   int height, width;
+   int nblocks; // original number of blocks
+   int rblocks; // reduces number of blocks
+   Mesh * mesh = nullptr;
+   bool parallel = false;
+   // original set of Finite Element Spaces
+   Array<FiniteElementSpace *> fes;
+   // indicates if the original space is already a trace space
+   Array<bool> IsTraceSpace;
+
+   // New set of "reduced" Finite Element Spaces
+   // (after static condensation)
+   Array<FiniteElementSpace *> tr_fes;
+
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+
+   Array<int> rdof_offsets;
+   Array<int> rtdof_offsets;
+
+   // Schur complement matrix
+   // S = A_ii - A_ib (A_bb)^{-1} A_bi.
+   BlockMatrix * S = nullptr;
+   BlockMatrix * S_e = nullptr;
+
+   BlockVector * y = nullptr;
+
+   Array<DenseMatrix * > lmat;
+   Array<Vector * > lvec;
+
+   Array<int> rdof_edof;      // Map from reduced dofs to exposed dofs
+   Array<int> ess_rtdof_list;
+
+   BlockMatrix * P = nullptr; // Block Prolongation
+   BlockMatrix * R = nullptr; // Block Restriction
+
+#ifdef MFEM_USE_MPI
+   BlockOperator * pS = nullptr;
+   BlockOperator * pS_e = nullptr;
+   // Block HypreParMatrix for Prolongation
+   BlockOperator * pP = nullptr;
+#endif
+
+   bool Parallel() const { return parallel; }
+
+
+   // tr_idx (trace dofs indices)
+   // int_idx (interior dof indices)
+   void GetReduceElementIndicesAndOffsets(int el, Array<int> & tr_idx,
+                                          Array<int> & int_idx,
+                                          Array<int> & offsets) const;
+
+   void GetReduceElementVDofs(int el, Array<int> & rdofs) const;
+   void GetElementVDofs(int el, Array<int> & vdofs) const;
+
+
+   //  S = A_ii - A_ib (A_bb)^{-1} A_bi.
+   //  y = y_i - A_ib (A_bb)^{-1} y_b
+   void GetLocalShurComplement(int el, const Array<int> & tr_idx,
+                               const Array<int> & int_idx,
+                               const DenseMatrix & elmat, const Vector & elvect,
+                               DenseMatrix & rmat, Vector & rvect);
+
+   void ComputeOffsets();
+
+   void BuildProlongation();
+#ifdef MFEM_USE_MPI
+   void BuildParallelProlongation();
+#endif
+
+   // ess_tdof list for each space
+   Array<Array<int> *> ess_tdofs;
+   void FillEssTdofLists(const Array<int> & ess_tdof_list);
+
+   void ConformingAssemble(int skip_zeros);
+
+   /** Restrict a marker Array on the true FE spaces dofs to a marker Array on
+    the reduced/trace true FE spaces dofs. */
+   void ConvertMarkerToReducedTrueDofs(Array<int> & tdof_marker,
+                                       Array<int> & rtdof_marker);
+public:
+
+   BlockStaticCondensation(Array<FiniteElementSpace *> & fes_);
+
+   ~BlockStaticCondensation();
+
+   void SetSpaces(Array<FiniteElementSpace*> & fes_);
+
+   void Init();
+
+   /** Assemble the contribution to the Schur complement from the given
+       element matrix 'elmat'; save the other blocks internally: A_bb_inv, A_bi,
+       and A_bi. */
+
+   void AssembleReducedSystem(int el, DenseMatrix &elmat,
+                              Vector & elvect);
+
+   /// Finalize the construction of the Schur complement matrix.
+   void Finalize(int skip_zeros = 0);
+
+   /// Determine and save internally essential reduced true dofs.
+   void SetEssentialTrueDofs(const Array<int> &ess_tdof_list);
+
+   /// Eliminate the given reduced true dofs from the Schur complement matrix S.
+   void EliminateReducedTrueDofs(const Array<int> &ess_rtdof_list,
+                                 Matrix::DiagonalPolicy dpolicy);
+
+   void EliminateReducedTrueDofs(Matrix::DiagonalPolicy dpolicy);
+
+   bool HasEliminatedBC() const
+   {
+#ifndef MFEM_USE_MPI
+      return S_e;
+#else
+      return S_e || pS_e;
+#endif
+
+   }
+
+   /// Return the serial Schur complement matrix.
+   BlockMatrix &GetMatrix() { return *S; }
+
+   /// Return the eliminated part of the serial Schur complement matrix.
+   BlockMatrix &GetMatrixElim() { return *S_e; }
+
+#ifdef MFEM_USE_MPI
+   /// Return the parallel Schur complement matrix.
+   BlockOperator &GetParallelMatrix() { return *pS; }
+
+   /// Return the eliminated part of the parallel Schur complement matrix.
+   BlockOperator &GetParallelMatrixElim() { return *pS_e; }
+
+   void ParallelAssemble(BlockMatrix *m);
+#endif
+
+   void FormSystemMatrix(Operator::DiagonalPolicy diag_policy);
+
+   /** Restrict a solution vector on the full FE space dofs to a vector on the
+       reduced/trace true FE space dofs. */
+   void ReduceSolution(const Vector &sol, Vector &sc_sol) const;
+
+   /** @brief Set the reduced solution `X` and r.h.s `B` vectors from the full
+    linear system solution `x` and r.h.s. `b` vectors.
+
+      This method should be called after the internal reduced essential dofs
+      have been set using SetEssentialTrueDofs() and both the Schur complement
+      and its eliminated part have been finalized. */
+   void ReduceSystem(Vector &x, Vector &X, Vector &B,
+                     int copy_interior = 0) const;
+
+   /** Restrict a list of true FE space dofs to a list of reduced/trace true FE
+    space dofs. */
+   void ConvertListToReducedTrueDofs(const Array<int> &ess_tdof_list,
+                                     Array<int> &ess_rtdof_list) const;
+
+   /** Given a solution of the reduced system 'sc_sol' and the RHS 'b' for the
+       full linear system, compute the solution of the full system 'sol'. */
+   void ComputeSolution(const Vector &sc_sol, Vector &sol) const;
+
+};
+
+}
+
+#endif

fem/fem.hpp

@@ -22,6 +22,7 @@
 #include "complex_fem.hpp"
 #include "convergence.hpp"
 #include "lininteg.hpp"
+#include "blockinteg.hpp"
 #include "nonlininteg.hpp"
 #include "bilininteg.hpp"
 #include "fespace.hpp"
@@ -45,6 +46,10 @@
 #include "multigrid.hpp"
 #include "ceed/algebraic.hpp"
 #include "lor.hpp"
+#include "blocklinearform.hpp"
+#include "blockbilinearform.hpp"
+#include "normal_equations.hpp"
+#include "blockstaticcond.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"
@@ -52,6 +57,7 @@
 #include "plinearform.hpp"
 #include "pbilinearform.hpp"
 #include "pnonlinearform.hpp"
+#include "pnormal_equations.hpp"
 #endif
 
 #ifdef MFEM_USE_SIDRE

fem/normal_equations.cpp

@@ -0,0 +1,824 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void NormalEquations::Init()
+{
+   trial_integs.SetSize(trial_fes.Size(), test_fecols.Size());
+   for (int i = 0; i < trial_integs.NumRows(); i++)
+   {
+      for (int j = 0; j < trial_integs.NumCols(); j++)
+      {
+         trial_integs(i,j) = new Array<BilinearFormIntegrator * >();
+      }
+   }
+
+   test_integs.SetSize(test_fecols.Size(), test_fecols.Size());
+   for (int i = 0; i < test_integs.NumRows(); i++)
+   {
+      for (int j = 0; j < test_integs.NumCols(); j++)
+      {
+         test_integs(i,j) = new Array<BilinearFormIntegrator * >();
+      }
+   }
+
+   lfis.SetSize(test_fecols.Size());
+   for (int j = 0; j < lfis.Size(); j++)
+   {
+      lfis[j] = new Array<LinearFormIntegrator * >();
+   }
+
+
+   ComputeOffsets();
+
+   mat = mat_e = NULL;
+   diag_policy = mfem::Operator::DIAG_ONE;
+   height = dof_offsets[nblocks];
+   width = height;
+
+   initialized = true;
+   static_cond = nullptr;
+
+   if (store_matrices)
+   {
+      Bmat.SetSize(mesh->GetNE());
+      fvec.SetSize(mesh->GetNE());
+   }
+}
+
+void NormalEquations::ComputeOffsets()
+{
+   dof_offsets.SetSize(nblocks+1);
+   tdof_offsets.SetSize(nblocks+1);
+   dof_offsets[0] = 0;
+   tdof_offsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      dof_offsets[i+1] = trial_fes[i]->GetVSize();
+      tdof_offsets[i+1] = trial_fes[i]->GetTrueVSize();
+   }
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+}
+
+// Allocate SparseMatrix and RHS
+void NormalEquations::AllocMat()
+{
+   if (static_cond) { return; }
+
+   mat = new BlockMatrix(dof_offsets);
+   mat->owns_blocks = 1;
+
+   for (int i = 0; i<mat->NumRowBlocks(); i++)
+   {
+      int h = dof_offsets[i+1] - dof_offsets[i];
+      for (int j = 0; j<mat->NumColBlocks(); j++)
+      {
+         int w = dof_offsets[j+1] - dof_offsets[j];
+         mat->SetBlock(i,j,new SparseMatrix(h, w));
+      }
+   }
+   y = new BlockVector(dof_offsets);
+   *y = 0.;
+}
+
+void NormalEquations::Finalize(int skip_zeros)
+{
+   if (mat) { mat->Finalize(skip_zeros); }
+   if (mat_e) { mat_e->Finalize(skip_zeros); }
+   if (static_cond) { static_cond->Finalize(); }
+}
+
+/// Adds new Domain BF Integrator. Assumes ownership of @a bfi.
+void NormalEquations::AddTrialIntegrator(
+   BilinearFormIntegrator *bfi, int trial_fes, int test_fes)
+{
+   trial_integs(trial_fes,test_fes)->Append(bfi);
+}
+
+/// Adds new Domain BF Integrator. Assumes ownership of @a bfi.
+void NormalEquations::AddTestIntegrator
+(BilinearFormIntegrator *bfi, int test_fes0, int test_fes1)
+{
+   test_integs(test_fes0,test_fes1)->Append(bfi);
+}
+
+/// Adds new Domain LF Integrator. Assumes ownership of @a bfi.
+void NormalEquations::AddDomainLFIntegrator(
+   LinearFormIntegrator *lfi, int test_fes)
+{
+   lfis[test_fes]->Append(lfi);
+}
+
+void NormalEquations::BuildProlongation()
+{
+   P = new BlockMatrix(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      const SparseMatrix *P_ = trial_fes[i]->GetConformingProlongation();
+      if (P_)
+      {
+         const SparseMatrix *R_ = trial_fes[i]->GetRestrictionMatrix();
+         P->SetBlock(i,i,const_cast<SparseMatrix*>(P_));
+         R->SetBlock(i,i,const_cast<SparseMatrix*>(R_));
+      }
+      else
+      {
+         // do nothing
+      }
+   }
+}
+
+void NormalEquations::ConformingAssemble()
+{
+   Finalize(0);
+   if (!P) { BuildProlongation(); }
+
+   BlockMatrix * Pt = Transpose(*P);
+   BlockMatrix * PtA = mfem::Mult(*Pt, *mat);
+   mat->owns_blocks = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      for (int j = 0; j<nblocks; j++)
+      {
+         SparseMatrix * tmp = &mat->GetBlock(i,j);
+         if (Pt->IsZeroBlock(i,i))
+         {
+            PtA->SetBlock(i,j,tmp);
+         }
+         else
+         {
+            delete tmp;
+         }
+      }
+   }
+   delete mat;
+   if (mat_e)
+   {
+      BlockMatrix *PtAe = mfem::Mult(*Pt, *mat_e);
+      mat_e->owns_blocks = 0;
+      for (int i = 0; i<nblocks; i++)
+      {
+         for (int j = 0; j<nblocks; j++)
+         {
+            SparseMatrix * tmp = &mat_e->GetBlock(i,j);
+            if (Pt->IsZeroBlock(i,i))
+            {
+               PtAe->SetBlock(i,j,tmp);
+            }
+            else
+            {
+               delete tmp;
+            }
+         }
+      }
+      delete mat_e;
+      mat_e = PtAe;
+   }
+   delete Pt;
+
+   mat = mfem::Mult(*PtA, *P);
+
+   PtA->owns_blocks = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      for (int j = 0; j<nblocks; j++)
+      {
+         SparseMatrix * tmp = &PtA->GetBlock(j,i);
+         if (P->IsZeroBlock(i,i))
+         {
+            mat->SetBlock(j,i,tmp);
+         }
+         else
+         {
+            delete tmp;
+         }
+      }
+   }
+   delete PtA;
+
+   if (mat_e)
+   {
+      BlockMatrix *PtAeP = mfem::Mult(*mat_e, *P);
+      mat_e->owns_blocks = 0;
+      for (int i = 0; i<nblocks; i++)
+      {
+         for (int j = 0; j<nblocks; j++)
+         {
+            SparseMatrix * tmp = &mat_e->GetBlock(j,i);
+            if (P->IsZeroBlock(i,i))
+            {
+               PtAeP->SetBlock(j,i,tmp);
+            }
+            else
+            {
+               delete tmp;
+            }
+         }
+      }
+
+      delete mat_e;
+      mat_e = PtAeP;
+   }
+   height = mat->Height();
+   width = mat->Width();
+}
+
+/// Assembles the form i.e. sums over all domain integrators.
+void NormalEquations::Assemble(int skip_zeros)
+{
+   ElementTransformation *eltrans;
+   Array<int> faces, ori;
+
+   DofTransformation * doftrans_i, *doftrans_j;
+   if (mat == NULL)
+   {
+      AllocMat();
+   }
+
+   // loop through the elements
+   int dim = mesh->Dimension();
+   DenseMatrix B, Be, G, Ge, A;
+   Vector vec_e, vec, Gvec, b;
+   Array<int> vdofs;
+
+   // loop through elements
+   for (int iel = 0; iel < mesh -> GetNE(); iel++)
+   {
+      if (dim == 1)
+      {
+         mesh->GetElementVertices(iel, faces);
+      }
+      if (dim == 2)
+      {
+         mesh->GetElementEdges(iel, faces, ori);
+      }
+      else //dim = 3
+      {
+         mesh->GetElementFaces(iel,faces,ori);
+      }
+      int numfaces = faces.Size();
+
+      Array<int> test_offs(test_fecols.Size()+1); test_offs[0] = 0;
+      Array<int> trial_offs(trial_fes.Size()+1); trial_offs = 0;
+
+      eltrans = mesh->GetElementTransformation(iel);
+      for (int j = 0; j < test_fecols.Size(); j++)
+      {
+         int order = test_fecols[j]->GetOrder(); // assuming uniform order
+         test_offs[j+1] = test_fecols_vdims[j]*test_fecols[j]->GetFE(
+                             eltrans->GetGeometryType(),
+                             order)->GetDof();
+      }
+      for (int j = 0; j < trial_fes.Size(); j++)
+      {
+         if (IsTraceFes[j])
+         {
+            for (int ie = 0; ie<faces.Size(); ie++)
+            {
+               trial_offs[j+1] += trial_fes[j]->GetVDim()*trial_fes[j]->GetFaceElement(
+                                     faces[ie])->GetDof();
+            }
+         }
+         else
+         {
+            trial_offs[j+1] = trial_fes[j]->GetVDim() * trial_fes[j]->GetFE(
+                                 iel)->GetDof();
+         }
+      }
+      test_offs.PartialSum();
+      trial_offs.PartialSum();
+
+      G.SetSize(test_offs.Last()); G = 0.0;
+      vec.SetSize(test_offs.Last()); vec = 0.0;
+      B.SetSize(test_offs.Last(),trial_offs.Last()); B = 0.0;
+
+
+      for (int j = 0; j < test_fecols.Size(); j++)
+      {
+         int order = test_fecols[j]->GetOrder();
+
+         eltrans = mesh->GetElementTransformation(iel);
+         const FiniteElement & test_fe =
+            *test_fecols[j]->GetFE(eltrans->GetGeometryType(), order);
+
+         for (int k = 0; k < lfis[j]->Size(); k++)
+         {
+            (*lfis[j])[k]->AssembleRHSElementVect(test_fe,*eltrans,vec_e);
+            vec.AddSubVector(vec_e,test_offs[j]);
+         }
+
+         for (int i = 0; i < test_fecols.Size(); i++)
+         {
+            int order = test_fecols[i]->GetOrder();
+            eltrans = mesh->GetElementTransformation(iel);
+            const FiniteElement & test_fe_i =
+               *test_fecols[i]->GetFE(eltrans->GetGeometryType(), order);
+
+            for (int k = 0; k < test_integs(i,j)->Size(); k++)
+            {
+               if (i==j)
+               {
+                  (*test_integs(i,j))[k]->AssembleElementMatrix(test_fe,*eltrans,Ge);
+               }
+               else
+               {
+                  (*test_integs(i,j))[k]->AssembleElementMatrix2(test_fe_i,test_fe,*eltrans,
+                                                                 Ge);
+               }
+               G.AddSubMatrix(test_offs[j], test_offs[i], Ge);
+            }
+         }
+
+         for (int i = 0; i < trial_fes.Size(); i++)
+         {
+            if (IsTraceFes[i])
+            {
+               for (int k = 0; k < trial_integs(i,j)->Size(); k++)
+               {
+                  int face_dof_offs = 0;
+                  for (int ie = 0; ie < numfaces; ie++)
+                  {
+                     int iface = faces[ie];
+                     FaceElementTransformations * ftr = mesh->GetFaceElementTransformations(iface);
+                     const FiniteElement & tfe = *trial_fes[i]->GetFaceElement(iface);
+                     (*trial_integs(i,j))[k]->AssembleTraceFaceMatrix(iel,tfe,test_fe,*ftr,Be);
+                     B.AddSubMatrix(test_offs[j], trial_offs[i]+face_dof_offs, Be);
+                     face_dof_offs+=Be.Width();
+                  }
+               }
+            }
+            else
+            {
+               const FiniteElement & fe = *trial_fes[i]->GetFE(iel);
+               eltrans = mesh->GetElementTransformation(iel);
+               for (int k = 0; k < trial_integs(i,j)->Size(); k++)
+               {
+                  (*trial_integs(i,j))[k]->AssembleElementMatrix2(fe,test_fe,*eltrans,Be);
+                  B.AddSubMatrix(test_offs[j], trial_offs[i], Be);
+               }
+            }
+         }
+      }
+
+      // Form Normal Equations B^T G^-1 B = B^T G^-1 l
+      Gvec.SetSize(G.Height());
+      b.SetSize(B.Width());
+      A.SetSize(B.Width());
+
+      CholeskyFactors chol(G.GetData());
+      chol.Factor(G.Height());
+
+      chol.LSolve(B.Height(), B.Width(), B.GetData());
+      chol.LSolve(vec.Size(), 1, vec.GetData());
+      if (store_matrices)
+      {
+         Bmat[iel] = new DenseMatrix(B);
+         fvec[iel] = new Vector(vec);
+      }
+      mfem::MultAtB(B,B,A);
+      B.MultTranspose(vec,b);
+
+      if (static_cond)
+      {
+         static_cond->AssembleReducedSystem(iel,A,b);
+      }
+      else
+      {
+         // Assembly
+         for (int i = 0; i<trial_fes.Size(); i++)
+         {
+            Array<int> vdofs_i;
+            doftrans_i = nullptr;
+            if (IsTraceFes[i])
+            {
+               Array<int> face_vdofs;
+               for (int k = 0; k < numfaces; k++)
+               {
+                  int iface = faces[k];
+                  trial_fes[i]->GetFaceVDofs(iface, face_vdofs);
+                  vdofs_i.Append(face_vdofs);
+               }
+            }
+            else
+            {
+               doftrans_i = trial_fes[i]->GetElementVDofs(iel, vdofs_i);
+            }
+            for (int j = 0; j<trial_fes.Size(); j++)
+            {
+               Array<int> vdofs_j;
+               doftrans_j = nullptr;
+
+               if (IsTraceFes[j])
+               {
+                  Array<int> face_vdofs;
+                  for (int k = 0; k < numfaces; k++)
+                  {
+                     int iface = faces[k];
+                     trial_fes[j]->GetFaceVDofs(iface, face_vdofs);
+                     vdofs_j.Append(face_vdofs);
+                  }
+               }
+               else
+               {
+                  doftrans_j = trial_fes[j]->GetElementVDofs(iel, vdofs_j);
+               }
+
+               DenseMatrix Ae;
+               A.GetSubMatrix(trial_offs[i],trial_offs[i+1],
+                              trial_offs[j],trial_offs[j+1], Ae);
+               if (doftrans_i || doftrans_j)
+               {
+                  TransformDual(doftrans_i, doftrans_j, Ae);
+               }
+               mat->GetBlock(i,j).AddSubMatrix(vdofs_i,vdofs_j, Ae);
+            }
+
+            // assemble rhs
+            double * data = b.GetData();
+            Vector vec1;
+            // ref subvector
+            vec1.SetDataAndSize(&data[trial_offs[i]],
+                                trial_offs[i+1]-trial_offs[i]);
+            if (doftrans_i)
+            {
+               doftrans_i->TransformDual(vec1);
+            }
+            y->GetBlock(i).AddElementVector(vdofs_i,vec1);
+         }
+      }
+   }
+}
+
+void NormalEquations::FormLinearSystem(const Array<int>
+                                       &ess_tdof_list,
+                                       Vector &x,
+                                       OperatorHandle &A, Vector &X,
+                                       Vector &B, int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   if (static_cond)
+   {
+      // Schur complement reduction to the exposed dofs
+      static_cond->ReduceSystem(x, X, B, copy_interior);
+   }
+   else if (!P)
+   {
+      EliminateVDofsInRHS(ess_tdof_list, x, *y);
+      X.MakeRef(x, 0, x.Size());
+      B.MakeRef(*y, 0, y->Size());
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+   else // non conforming space
+   {
+      B.SetSize(P->Width());
+
+      P->MultTranspose(*y, B);
+      double *data = y->GetData();
+      Vector tmp;
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (P->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp.SetDataAndSize(&data[offset],tdof_offsets[i+1]-tdof_offsets[i]);
+            B.SetVector(tmp,offset);
+         }
+      }
+
+      X.SetSize(R->Height());
+
+      R->Mult(x, X);
+      data = x.GetData();
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (R->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp.SetDataAndSize(&data[offset],tdof_offsets[i+1]-tdof_offsets[i]);
+            X.SetVector(tmp,offset);
+         }
+      }
+
+      EliminateVDofsInRHS(ess_tdof_list, X, B);
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+}
+
+void NormalEquations::FormSystemMatrix(const Array<int>
+                                       &ess_tdof_list,
+                                       OperatorHandle &A)
+{
+   if (static_cond)
+   {
+      if (!static_cond->HasEliminatedBC())
+      {
+         static_cond->SetEssentialTrueDofs(ess_tdof_list);
+         static_cond->FormSystemMatrix(diag_policy);
+      }
+      A.Reset(&static_cond->GetMatrix(), false);
+   }
+   else
+   {
+      if (!mat_e)
+      {
+         bool conforming = true;
+         for (int i = 0; i<nblocks; i++)
+         {
+            const SparseMatrix *P_ = trial_fes[i]->GetConformingProlongation();
+            if (P_)
+            {
+               conforming = false;
+               break;
+            }
+         }
+         if (!conforming) { ConformingAssemble(); }
+         const int remove_zeros = 0;
+         EliminateVDofs(ess_tdof_list, diag_policy);
+         Finalize(remove_zeros);
+      }
+      A.Reset(mat, false);
+   }
+}
+
+void NormalEquations::EliminateVDofsInRHS(
+   const Array<int> &vdofs, const Vector &x, Vector &b)
+{
+   mat_e->AddMult(x,b,-1.);
+   mat->PartMult(vdofs,x,b);
+}
+
+void NormalEquations::EliminateVDofs(const Array<int> &vdofs,
+                                     Operator::DiagonalPolicy dpolicy)
+{
+   if (mat_e == NULL)
+   {
+      Array<int> offsets;
+
+      offsets.MakeRef( (P) ? tdof_offsets : dof_offsets);
+
+      mat_e = new BlockMatrix(offsets);
+      mat_e->owns_blocks = 1;
+      for (int i = 0; i<mat_e->NumRowBlocks(); i++)
+      {
+         int h = offsets[i+1] - offsets[i];
+         for (int j = 0; j<mat_e->NumColBlocks(); j++)
+         {
+            int w = offsets[j+1] - offsets[j];
+            mat_e->SetBlock(i,j,new SparseMatrix(h, w));
+         }
+      }
+   }
+   mat->EliminateRowCols(vdofs,mat_e,diag_policy);
+}
+
+void NormalEquations::RecoverFEMSolution(const Vector &X,
+                                         Vector &x)
+{
+
+   if (static_cond)
+   {
+      // Private dofs back solve
+      static_cond->ComputeSolution(X, x);
+   }
+   else if (!P)
+   {
+      x.SyncMemory(X);
+   }
+   else
+   {
+      x.SetSize(P->Height());
+      P->Mult(X, x);
+      double *data = X.GetData();
+      Vector tmp;
+      for (int i = 0; i<nblocks; i++)
+      {
+         if (P->IsZeroBlock(i,i))
+         {
+            int offset = tdof_offsets[i];
+            tmp.SetDataAndSize(&data[offset],tdof_offsets[i+1]-tdof_offsets[i]);
+            x.SetVector(tmp,offset);
+         }
+      }
+   }
+}
+
+
+void NormalEquations::ReleaseInitMemory()
+{
+   if (initialized)
+   {
+      for (int k = 0; k< trial_integs.NumRows(); k++)
+      {
+         for (int l = 0; l<trial_integs.NumCols(); l++)
+         {
+            for (int i = 0; i<trial_integs(k,l)->Size(); i++)
+            {
+               delete (*trial_integs(k,l))[i];
+            }
+            delete trial_integs(k,l);
+         }
+      }
+      trial_integs.DeleteAll();
+
+      for (int k = 0; k < test_integs.NumRows(); k++)
+      {
+         for (int l = 0; l < test_integs.NumCols(); l++)
+         {
+            for (int i = 0; i < test_integs(k,l)->Size(); i++)
+            {
+               delete (*test_integs(k,l))[i];
+            }
+            delete test_integs(k,l);
+         }
+      }
+      test_integs.DeleteAll();
+
+      for (int k = 0; k < lfis.Size(); k++)
+      {
+         for (int i = 0; i < lfis[k]->Size(); i++)
+         {
+            delete (*lfis[k])[i];
+         }
+         delete lfis[k];
+      }
+      lfis.DeleteAll();
+   }
+}
+
+
+
+void NormalEquations::Update()
+{
+   delete mat_e; mat_e = nullptr;
+   delete mat; mat = nullptr;
+   delete y; y = nullptr;
+
+   if (P)
+   {
+      delete P; P = nullptr;
+      delete R; R = nullptr;
+   }
+
+   delete static_cond;
+   static_cond = NULL;
+
+   ComputeOffsets();
+
+   diag_policy = mfem::Operator::DIAG_ONE;
+   height = dof_offsets[nblocks];
+   width = height;
+
+   initialized = true;
+
+   if (store_matrices)
+   {
+      for (int i = 0; i<Bmat.Size(); i++)
+      {
+         delete Bmat[i]; Bmat[i] = nullptr;
+         delete fvec[i]; fvec[i] = nullptr;
+      }
+      Bmat.SetSize(mesh->GetNE());
+      fvec.SetSize(mesh->GetNE());
+      for (int i = 0; i<Bmat.Size(); i++)
+      {
+         Bmat[i] = nullptr;
+         fvec[i] = nullptr;
+      }
+   }
+}
+
+
+void NormalEquations::EnableStaticCondensation()
+{
+   static_cond = new BlockStaticCondensation(trial_fes);
+}
+
+
+
+Vector & NormalEquations::ComputeResidual(const BlockVector & x)
+{
+   // Element vector of trial space size
+   Vector u;
+   Array<int> vdofs;
+   Array<int> faces, ori;
+   int dim = mesh->Dimension();
+   residuals.SetSize(mesh->GetNE());
+   // loop through elements
+   for (int iel = 0; iel < mesh -> GetNE(); iel++)
+   {
+      if (dim == 1)
+      {
+         mesh->GetElementVertices(iel, faces);
+      }
+      if (dim == 2)
+      {
+         mesh->GetElementEdges(iel, faces, ori);
+      }
+      else //dim = 3
+      {
+         mesh->GetElementFaces(iel,faces,ori);
+      }
+      int numfaces = faces.Size();
+
+      Array<int> trial_offs(trial_fes.Size()+1); trial_offs = 0;
+
+      for (int j = 0; j < trial_fes.Size(); j++)
+      {
+         if (IsTraceFes[j])
+         {
+            for (int ie = 0; ie<faces.Size(); ie++)
+            {
+               trial_offs[j+1] += trial_fes[j]->GetFaceElement(faces[ie])->GetDof();
+            }
+         }
+         else
+         {
+            trial_offs[j+1] = trial_fes[j]->GetVDim() * trial_fes[j]->GetFE(
+                                 iel)->GetDof();
+         }
+      }
+      trial_offs.PartialSum();
+
+      u.SetSize(trial_offs.Last());
+      double * data = u.GetData();
+      DofTransformation * doftrans = nullptr;
+      for (int i = 0; i<trial_fes.Size(); i++)
+      {
+         vdofs.SetSize(0);
+         doftrans = nullptr;
+         if (IsTraceFes[i])
+         {
+            Array<int> face_vdofs;
+            for (int k = 0; k < numfaces; k++)
+            {
+               int iface = faces[k];
+               trial_fes[i]->GetFaceVDofs(iface, face_vdofs);
+               vdofs.Append(face_vdofs);
+            }
+         }
+         else
+         {
+            doftrans = trial_fes[i]->GetElementVDofs(iel, vdofs);
+         }
+         Vector vec1;
+         vec1.SetDataAndSize(&data[trial_offs[i]],
+                             trial_offs[i+1]-trial_offs[i]);
+         x.GetBlock(i).GetSubVector(vdofs,vec1);
+         if (doftrans)
+         {
+            doftrans->InvTransformPrimal(vec1);
+         }
+      } // end of loop through trial spaces
+
+      Vector v(Bmat[iel]->Height());
+      Bmat[iel]->Mult(u,v);
+      v -= *fvec[iel];
+      residuals[iel] = v.Norml2();
+   } // end of loop through elements
+   return residuals;
+}
+
+
+NormalEquations::~NormalEquations()
+{
+   delete mat_e; mat_e = nullptr;
+   delete mat; mat = nullptr;
+   delete y; y = nullptr;
+
+   ReleaseInitMemory();
+
+   if (P)
+   {
+      delete P;
+      delete R;
+   }
+
+   delete static_cond;
+
+   if (store_matrices)
+   {
+      for (int i = 0; i<mesh->GetNE(); i++)
+      {
+         delete Bmat[i]; Bmat[i] = nullptr;
+         delete fvec[i]; fvec[i] = nullptr;
+      }
+   }
+}
+
+} // namespace mfem

fem/normal_equations.hpp

@@ -0,0 +1,255 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_NORMALEQUATIONS
+#define MFEM_NORMALEQUATIONS
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+#include "blockstaticcond.hpp"
+
+namespace mfem
+{
+
+/** @brief Class representing the whole weak formulation. (Convenient for DPG or
+    Normal Equations) */
+class NormalEquations
+{
+
+protected:
+
+   BlockStaticCondensation *static_cond; ///< Owned.
+
+   bool initialized = false;
+
+   Mesh * mesh = nullptr;
+   int height, width;
+   int nblocks;
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+
+   /// Block matrix \f$ M \f$ to be associated with the Block bilinear form. Owned.
+   BlockMatrix *mat = nullptr;
+
+   /// BlockVector to be associated with the Block linear form
+   BlockVector * y = nullptr;
+
+   /** @brief Block Matrix \f$ M_e \f$ used to store the eliminations
+        from the b.c.  Owned.
+       \f$ M + M_e = M_{original} \f$ */
+   BlockMatrix *mat_e = nullptr;
+
+   // Trial FE spaces
+   Array<FiniteElementSpace * > trial_fes;
+
+   // Flags to determine if a FiniteElementSpace is Trace
+   Array<int> IsTraceFes;
+
+   // Test FE Collections (Broken)
+   Array<FiniteElementCollection *> test_fecols;
+   Array<int> test_fecols_vdims;
+
+   /// Set of Trial Integrators to be applied for matrix B
+   Array2D<Array<BilinearFormIntegrator * > * > trial_integs;
+
+   /// Set of Test Space (broken) Integrators to be applied for matrix G
+   Array2D<Array<BilinearFormIntegrator * > * > test_integs;
+
+   /// Set of Liniear Froem Integrators to be applied.
+   Array<Array<LinearFormIntegrator * > * > lfis;
+
+   BlockMatrix * P = nullptr; // Block Prolongation
+   BlockMatrix * R = nullptr; // Block Restriction
+
+   mfem::Operator::DiagonalPolicy diag_policy;
+
+   void Init();
+   void ReleaseInitMemory();
+
+   // Allocate appropriate SparseMatrix and assign it to mat
+   void AllocMat();
+
+   void ConformingAssemble();
+
+   void ComputeOffsets();
+
+   virtual void BuildProlongation();
+
+   bool store_matrices = false;
+
+   // Store the matrix L^-1 B  and Vector L^-1 l
+   // where G = L L^t
+   Array<DenseMatrix * > Bmat;
+   Array<Vector * > fvec;
+   Vector residuals;
+
+
+private:
+
+public:
+
+   /// Creates bilinear form associated with FE spaces @a *fespaces.
+   NormalEquations()
+   {
+      height = 0.;
+      width = 0;
+   }
+
+   NormalEquations(Array<FiniteElementSpace* > & fes_,
+                   Array<FiniteElementCollection *> & fecol_)
+   {
+      SetSpaces(fes_,fecol_);
+   }
+
+   void SetTestFECollVdim(int test_fec, int vdim)
+   {
+      test_fecols_vdims[test_fec] = vdim;
+   }
+
+
+
+   void SetSpaces(Array<FiniteElementSpace* > & fes_,
+                  Array<FiniteElementCollection *> & fecol_)
+   {
+      trial_fes = fes_;
+      test_fecols = fecol_;
+      test_fecols_vdims.SetSize(test_fecols.Size());
+      test_fecols_vdims = 1;
+      nblocks = trial_fes.Size();
+      mesh = trial_fes[0]->GetMesh();
+
+      IsTraceFes.SetSize(nblocks);
+      // Initialize with False
+      IsTraceFes = false;
+      for (int i = 0; i < nblocks; i++)
+      {
+         IsTraceFes[i] =
+            (dynamic_cast<const H1_Trace_FECollection*>(trial_fes[i]->FEColl()) ||
+             dynamic_cast<const ND_Trace_FECollection*>(trial_fes[i]->FEColl()) ||
+             dynamic_cast<const RT_Trace_FECollection*>(trial_fes[i]->FEColl()));
+      }
+      Init();
+   }
+
+   // Get the size of the bilinear form of the NormalEquations
+   int Size() const { return height; }
+
+   // Pre-allocate the internal SparseMatrix before assembly.
+   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }
+
+   ///  Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Returns a reference to the sparse matrix:  \f$ M \f$
+   BlockMatrix &BlockMat()
+   {
+      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
+      return *mat;
+   }
+
+   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   BlockMatrix &BlockMatElim()
+   {
+      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
+      return *mat_e;
+   }
+
+   /// Adds new Trial Integrator. Assumes ownership of @a bfi.
+   void AddTrialIntegrator(BilinearFormIntegrator *bfi, int trial_fes,
+                           int test_fes);
+
+   /// Adds new Test Integrator. Assumes ownership of @a bfi.
+   void AddTestIntegrator(BilinearFormIntegrator *bfi, int test_fes0,
+                          int test_fes1);
+
+   /// Adds new Domain LF Integrator. Assumes ownership of @a bfi.
+   void AddDomainLFIntegrator(LinearFormIntegrator *bfi, int test_fes);
+
+   /// Assembles the form i.e. sums over all integrators.
+   void Assemble(int skip_zeros = 1);
+
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
+                                 OperatorHandle &A, Vector &X,
+                                 Vector &B, int copy_interior = 0);
+
+   template <typename OpType>
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
+                         OpType &A, Vector &X, Vector &B,
+                         int copy_interior = 0)
+   {
+      OperatorHandle Ah;
+      FormLinearSystem(ess_tdof_list, x, Ah, X, B, copy_interior);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
+   template <typename OpType>
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
+   {
+      OperatorHandle Ah;
+      FormSystemMatrix(ess_tdof_list, Ah);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   void EliminateVDofs(const Array<int> &vdofs,
+                       Operator::DiagonalPolicy dpolicy = Operator::DIAG_ONE);
+
+   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x, Vector &b);
+
+   virtual void RecoverFEMSolution(const Vector &X,Vector &x);
+
+   /// Sets diagonal policy used upon construction of the linear system.
+   /** Policies include:
+
+       - DIAG_ZERO (Set the diagonal values to zero)
+       - DIAG_ONE  (Set the diagonal values to one)
+       - DIAG_KEEP (Keep the diagonal values)
+   */
+   void SetDiagonalPolicy(Operator::DiagonalPolicy policy)
+   {
+      diag_policy = policy;
+   }
+
+   virtual void Update();
+
+   void StoreMatrices(bool store_matrices_ = true)
+   {
+      store_matrices = store_matrices_;
+      if (Bmat.Size() == 0)
+      {
+         Bmat.SetSize(mesh->GetNE());
+         fvec.SetSize(mesh->GetNE());
+         for (int i =0; i<mesh->GetNE(); i++)
+         {
+            Bmat[i] = nullptr;
+            fvec[i] = nullptr;
+         }
+      }
+   }
+
+   void EnableStaticCondensation();
+
+   Vector & ComputeResidual(const BlockVector & x);
+
+   /// Destroys bilinear form.
+   virtual ~NormalEquations();
+
+};
+
+} // namespace mfem
+
+#endif

fem/pnormal_equations.cpp

@@ -0,0 +1,223 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void ParNormalEquations::FillEssTdofLists(const Array<int> & ess_tdof_list)
+{
+   int j;
+   for (int i = 0; i<ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+      for (j = 0; j < nblocks; j++)
+      {
+         if (tdof_offsets[j+1] > tdof) { break; }
+      }
+      ess_tdofs[j]->Append(tdof-tdof_offsets[j]);
+   }
+}
+
+void ParNormalEquations::Assemble(int skip_zeros)
+{
+   NormalEquations::Assemble(skip_zeros);
+}
+
+void ParNormalEquations::ParallelAssemble(BlockMatrix *m)
+{
+   if (!P) { BuildProlongation(); }
+
+   p_mat = new BlockOperator(tdof_offsets);
+   p_mat_e = new BlockOperator(tdof_offsets);
+   p_mat->owns_blocks = 1;
+   p_mat_e->owns_blocks = 1;
+   HypreParMatrix * A = nullptr;
+   HypreParMatrix * PtAP = nullptr;
+   for (int i = 0; i<nblocks; i++)
+   {
+      HypreParMatrix * Pi = (HypreParMatrix*)(&P->GetBlock(i,i));
+      for (int j = 0; j<nblocks; j++)
+      {
+         if (m->IsZeroBlock(i,j)) { continue; }
+         if (i == j)
+         {
+            // Make block diagonal square hypre matrix
+            A = new HypreParMatrix(trial_pfes[i]->GetComm(), trial_pfes[i]->GlobalVSize(),
+                                   trial_pfes[i]->GetDofOffsets(),&m->GetBlock(i,i));
+            PtAP = RAP(A,Pi);
+            delete A;
+            p_mat_e->SetBlock(i,i,PtAP->EliminateRowsCols(*ess_tdofs[i]));
+         }
+         else
+         {
+            HypreParMatrix * Pj = (HypreParMatrix*)(&P->GetBlock(j,j));
+            A = new HypreParMatrix(trial_pfes[i]->GetComm(), trial_pfes[i]->GlobalVSize(),
+                                   trial_pfes[j]->GlobalVSize(), trial_pfes[i]->GetDofOffsets(),
+                                   trial_pfes[j]->GetDofOffsets(), &m->GetBlock(i,j));
+            PtAP = RAP(Pi,A,Pj);
+            delete A;
+            p_mat_e->SetBlock(i,j,PtAP->EliminateCols(*ess_tdofs[j]));
+            PtAP->EliminateRows(*ess_tdofs[i]);
+         }
+         p_mat->SetBlock(i,j,PtAP);
+      }
+   }
+}
+
+
+void ParNormalEquations::BuildProlongation()
+{
+   P = new BlockOperator(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+
+   for (int i = 0; i<nblocks; i++)
+   {
+      HypreParMatrix * P_ = trial_pfes[i]->Dof_TrueDof_Matrix();
+      P->SetBlock(i,i,P_);
+      const SparseMatrix * R_ = trial_pfes[i]->GetRestrictionMatrix();
+      R->SetBlock(i,i,const_cast<SparseMatrix*>(R_));
+   }
+}
+
+void ParNormalEquations::FormLinearSystem(const Array<int>
+                                          &ess_tdof_list,
+                                          Vector &x,
+                                          OperatorHandle &A, Vector &X,
+                                          Vector &B, int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+
+
+   if (static_cond)
+   {
+      static_cond->ReduceSystem(x, X, B, copy_interior);
+   }
+   else
+   {
+      B.SetSize(P->Width());
+      P->MultTranspose(*y,B);
+      X.SetSize(R->Height());
+      R->Mult(x,X);
+
+      // eliminate tdof is RHS
+      // B -= Ae*X
+      Vector tmp(B.Size());
+      p_mat_e->Mult(X,tmp);
+      B-=tmp;
+
+      for (int j = 0; j<nblocks; j++)
+      {
+         if (!ess_tdofs[j]->Size()) { continue; }
+         HypreParMatrix *Ah = (HypreParMatrix *)(&p_mat->GetBlock(j,j));
+         Vector diag;
+         Ah->GetDiag(diag);
+         for (int i = 0; i < ess_tdofs[j]->Size(); i++)
+         {
+            int tdof = (*ess_tdofs[j])[i];
+            int gdof = tdof + tdof_offsets[j];
+            B(gdof) = diag(tdof)*X(gdof);
+         }
+      }
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+}
+
+void ParNormalEquations::FormSystemMatrix(const Array<int>
+                                          &ess_tdof_list,
+                                          OperatorHandle &A)
+{
+   if (static_cond)
+   {
+      if (!static_cond->HasEliminatedBC())
+      {
+         static_cond->SetEssentialTrueDofs(ess_tdof_list);
+         static_cond->FormSystemMatrix(Operator::DiagonalPolicy::DIAG_ONE);
+      }
+      A.Reset(&static_cond->GetParallelMatrix(), false);
+   }
+   else
+   {
+      FillEssTdofLists(ess_tdof_list);
+      if (mat)
+      {
+         const int remove_zeros = 0;
+         Finalize(remove_zeros);
+         ParallelAssemble(mat);
+         delete mat;
+         mat = nullptr;
+         delete mat_e;
+         mat_e = nullptr;
+      }
+      A.Reset(p_mat,false);
+   }
+
+}
+
+
+
+void ParNormalEquations::RecoverFEMSolution(const Vector &X,
+                                            Vector &x)
+{
+
+   if (static_cond)
+   {
+      static_cond->ComputeSolution(X, x);
+   }
+   else
+   {
+      x.SetSize(P->Height());
+      P->Mult(X, x);
+   }
+}
+
+void ParNormalEquations::Update()
+{
+   NormalEquations::Update();
+   delete p_mat_e;
+   p_mat_e = nullptr;
+   delete p_mat;
+   p_mat = nullptr;
+   for (int i = 0; i<nblocks; i++)
+   {
+      delete ess_tdofs[i];
+      ess_tdofs[i] = new Array<int>();
+   }
+   delete P;
+   P = nullptr;
+   delete R;
+   R = nullptr;
+}
+
+ParNormalEquations::~ParNormalEquations()
+{
+   delete p_mat_e;
+   p_mat_e = nullptr;
+   delete p_mat;
+   p_mat = nullptr;
+   for (int i = 0; i<nblocks; i++)
+   {
+      delete ess_tdofs[i];
+   }
+   delete P;
+   delete R;
+}
+
+} // namespace mfem
+
+#endif

fem/pnormal_equations.hpp

@@ -0,0 +1,120 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PNORMALEQUATIONS
+#define MFEM_PNORMALEQUATIONS
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include <mpi.h>
+#include "pfespace.hpp"
+#include "normal_equations.hpp"
+
+namespace mfem
+{
+
+/** @brief Class representing the whole weak formulation. (Convenient for DPG or
+    Normal Equations) */
+class ParNormalEquations : public NormalEquations
+{
+
+protected:
+   // Trial FE spaces
+   Array<ParFiniteElementSpace * > trial_pfes;
+
+   // ess_tdof list for each space
+   Array<Array<int> *> ess_tdofs;
+
+   // split ess_tdof_list give in global tdof (for all spaces)
+   // to individual lists for each space
+   // (this can be changed i.e., the lists to be given by the user)
+   void FillEssTdofLists(const Array<int> & ess_tdof_list);
+
+   // Block operator of HypreParMatrix
+   BlockOperator * P = nullptr; // Block Prolongation
+   BlockMatrix * R = nullptr; // Block Restriction
+
+   // Block operator of HypreParMatrix
+   BlockOperator * p_mat = nullptr;
+   BlockOperator * p_mat_e = nullptr;
+
+   void BuildProlongation();
+
+private:
+
+public:
+
+   ParNormalEquations() {}
+
+   /// Creates bilinear form associated with FE spaces @a *fespaces.
+   ParNormalEquations(Array<ParFiniteElementSpace* > & trial_pfes_,
+                      Array<FiniteElementCollection* > & fecol_)
+      : NormalEquations()
+   {
+      SetParSpaces(trial_pfes_,fecol_);
+   }
+
+   void SetParSpaces(Array<ParFiniteElementSpace* > & trial_pfes_,
+                     Array<FiniteElementCollection* > & fecol_)
+   {
+      trial_pfes = trial_pfes_;
+      ess_tdofs.SetSize(trial_pfes.Size());
+
+      Array<FiniteElementSpace * > trial_sfes(trial_pfes.Size());
+      for (int i = 0; i<trial_sfes.Size(); i++)
+      {
+         trial_sfes[i] = (FiniteElementSpace *)trial_pfes[i];
+         ess_tdofs[i] = new Array<int>();
+      }
+
+      SetSpaces(trial_sfes,fecol_);
+   }
+
+
+   /// Assembles the form i.e. sums over all domain integrators.
+   void Assemble(int skip_zeros = 1);
+
+   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   /** The returned matrix has to be deleted by the caller. */
+
+   void ParallelAssemble(BlockMatrix *mat);
+
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
+                         OperatorHandle &A, Vector &X,
+                         Vector &B, int copy_interior = 0);
+
+   void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                         OperatorHandle &A);
+
+   /** Call this method after solving a linear system constructed using the
+       FormLinearSystem method to recover the solution as a ParGridFunction-size
+       vector in x. Use the same arguments as in the FormLinearSystem call. */
+   virtual void RecoverFEMSolution(const Vector &X, Vector &x);
+
+   virtual void Update();
+
+   /// Destroys bilinear form.
+   virtual ~ParNormalEquations();
+
+
+
+};
+
+} // namespace mfem
+
+
+#endif // MFEM_USE_MPI
+
+
+#endif

linalg/blockmatrix.cpp

@@ -15,7 +15,7 @@
 #include "sparsemat.hpp"
 #include "blockvector.hpp"
 #include "blockmatrix.hpp"
-
+#include "../fem/fespace.hpp"
 namespace mfem
 {
 
@@ -320,6 +320,55 @@ void BlockMatrix::EliminateRowCol(Array<int> & ess_bc_dofs, Vector & sol,
    }
 }
 
+
+void BlockMatrix::EliminateRowCols(Array<int> vdofs, BlockMatrix *Ae,
+                                   DiagonalPolicy dpolicy)
+{
+   MFEM_VERIFY(nRowBlocks == nColBlocks,
+               "BlockMatrix::EliminateRowCols supported only for"
+               "nRowBlocks = nColBlocks");
+
+   std::vector<Array<int>> cols(nRowBlocks);
+   std::vector<Array<int>> rows(nRowBlocks);
+   SparseMatrix * tmp = nullptr;
+
+
+   for (int k = 0; k < vdofs.Size(); k++)
+   {
+      int vdof = (vdofs[k]) >=0 ? vdofs[k] : -1 - vdofs[k];
+      // find block
+      int iblock, dof;
+      findGlobalCol(vdof,iblock,dof);
+      cols[iblock].Append(dof);
+      tmp = &GetBlock(iblock,iblock);
+      if (tmp)
+      {
+         tmp->EliminateRowCol(dof,Ae->GetBlock(iblock,iblock), dpolicy);
+      }
+   }
+
+   // Eliminate col from off-diagonal blocks
+   for (int j = 0; j<nColBlocks; j++)
+   {
+      if (!cols[j].Size()) { continue; }
+      Array<int> colmarker;
+      int blocksize = col_offsets[j+1] - col_offsets[j];
+      mfem::FiniteElementSpace::ListToMarker(cols[j],blocksize,colmarker);
+      for (int i = 0; i<nRowBlocks; i++)
+      {
+         if (i == j) { continue; }
+         tmp = &GetBlock(i,j);
+         if (tmp) { tmp->EliminateCols(colmarker,Ae->GetBlock(i,j)); }
+         for (int k = 0; k < cols[j].Size(); k++)
+         {
+            tmp = &GetBlock(j,i);
+            if (tmp) { tmp->EliminateRow(cols[j][k]); }
+         }
+      }
+   }
+}
+
+
 void BlockMatrix::EliminateZeroRows(const double threshold)
 {
    MFEM_VERIFY(nRowBlocks == nColBlocks, "not a square matrix");
@@ -471,6 +520,46 @@ void BlockMatrix::AddMultTranspose(const Vector & x, Vector & y,
    }
 }
 
+
+void BlockMatrix::PartMult(const Array<int> &rows, const Vector &x,
+                           Vector &y) const
+{
+   Array<int> cols;
+   Vector srow;
+   for (int i = 0; i<rows.Size(); i++)
+   {
+      int dof = (rows[i]>=0) ? rows[i] : -1-rows[i];
+      GetRow(dof,cols,srow);
+
+      double s=0.0;
+      for (int k = 0; k <cols.Size(); k++)
+      {
+         s += srow[k] * x[cols[k]];
+      }
+      y[dof] = s;
+   }
+}
+void BlockMatrix::PartAddMult(const Array<int> &rows, const Vector &x,
+                              Vector &y,
+                              const double a) const
+{
+   Array<int> cols;
+   Vector srow;
+   for (int i = 0; i<rows.Size(); i++)
+   {
+      int dof = (rows[i]>=0) ? rows[i] : -1-rows[i];
+      GetRow(dof,cols,srow);
+
+      double s=0.0;
+      for (int k = 0; k <cols.Size(); k++)
+      {
+         s += srow[k] * x[cols[k]];
+      }
+      y[dof] += a * s;
+   }
+}
+
+
 SparseMatrix * BlockMatrix::CreateMonolithic() const
 {
    int nnz = NumNonZeroElems();

linalg/blockmatrix.hpp

@@ -71,6 +71,9 @@ public:
        treated according to that policy. */
    void EliminateRowCol(int rc, DiagonalPolicy dpolicy = DIAG_ONE);
 
+   void EliminateRowCols(Array<int> vdofs, BlockMatrix *Ae,
+                         DiagonalPolicy dpolicy = DIAG_ONE);
+
    //! Symmetric elimination of the marked degree of freedom.
    /**
      @param ess_bc_dofs  marker of the degree of freedom to be eliminated
@@ -133,11 +136,17 @@ public:
                                  const double val = 1.) const;
    ///@}
 
+   void PartMult(const Array<int> &rows, const Vector &x, Vector &y) const;
+   void PartAddMult(const Array<int> &rows, const Vector &x, Vector &y,
+                    const double a=1.0) const;
+
    //! Destructor
    virtual ~BlockMatrix();
    //! If owns_blocks the SparseMatrix objects Aij will be deallocated.
    int owns_blocks;
 
+   virtual Type GetType() const { return MFEM_Block_Matrix; }
+
 private:
    //! Given a global row iglobal finds to which row iloc in block iblock belongs to.
    inline void findGlobalRow(int iglobal, int & iblock, int & iloc) const;

linalg/blockoperator.hpp

@@ -113,6 +113,8 @@ public:
    //! delete all blocks that are set (non-NULL); the default value is zero.
    int owns_blocks;
 
+   virtual Type GetType() const { return MFEM_Block_Operator; }
+
 private:
    //! Number of block rows
    int nRowBlocks;

linalg/densemat.cpp

@@ -1718,6 +1718,233 @@ void DenseMatrix::AddMatrix(double a, const DenseMatrix &A, int ro, int co)
    }
 }
 
+void DenseMatrix::GetSubMatrix(const Array<int> & idx, DenseMatrix & A) const
+{
+   int k = idx.Size();
+   A.SetSize(k);
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = idx[i];
+      for (int j = 0; j<k; j++)
+      {
+         jj = idx[j];
+         adata[i+j*k] = this->data[ii+jj*height];
+      }
+   }
+}
+
+void DenseMatrix::GetSubMatrix(const Array<int> & idx_i,
+                               const Array<int> & idx_j, DenseMatrix & A) const
+{
+   int k = idx_i.Size();
+   int l = idx_j.Size();
+   A.SetSize(k,l);
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = idx_i[i];
+      for (int j = 0; j<l; j++)
+      {
+         jj = idx_j[j];
+         adata[i+j*k] = this->data[ii+jj*height];
+      }
+   }
+}
+
+void DenseMatrix::GetSubMatrix(int ibeg, int iend, DenseMatrix & A)
+{
+   int k = iend - ibeg + 1;
+   A.SetSize(k);
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = ibeg + i;
+      for (int j = 0; j<k; j++)
+      {
+         jj = ibeg + j;
+         adata[i+j*k] = this->data[ii+jj*height];
+      }
+   }
+}
+
+void DenseMatrix::GetSubMatrix(int ibeg, int iend, int jbeg, int jend,
+                               DenseMatrix & A)
+{
+   int k = iend - ibeg;
+   int l = jend - jbeg;
+   A.SetSize(k,l);
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = ibeg + i;
+      for (int j = 0; j<l; j++)
+      {
+         jj = jbeg + j;
+         adata[i+j*k] = this->data[ii+jj*height];
+      }
+   }
+}
+
+void DenseMatrix::SetSubMatrix(const Array<int> & idx, const DenseMatrix & A)
+{
+   int k = idx.Size();
+   MFEM_VERIFY(A.Height() == k &&
+               A.Width() == k, "DenseMatrix::SetSubMatrix:Inconsistent matrix dimensions");
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = idx[i];
+      for (int j = 0; j<k; j++)
+      {
+         jj = idx[j];
+         this->data[ii+jj*height] = adata[i+j*k];
+      }
+   }
+}
+
+void DenseMatrix::SetSubMatrix(const Array<int> & idx_i,
+                               const Array<int> & idx_j, const DenseMatrix & A)
+{
+   int k = idx_i.Size();
+   int l = idx_j.Size();
+   MFEM_VERIFY(k == A.Height() &&
+               l == A.Width(),"DenseMatrix::SetSubMatrix:Inconsistent matrix dimensions");
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = idx_i[i];
+      for (int j = 0; j<l; j++)
+      {
+         jj = idx_j[j];
+         this->data[ii+jj*height] = adata[i+j*k];
+      }
+   }
+}
+
+void DenseMatrix::SetSubMatrix(int ibeg, const DenseMatrix & A)
+{
+   int k = A.Height();
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = ibeg + i;
+      for (int j = 0; j<k; j++)
+      {
+         jj = ibeg + j;
+         this->data[ii+jj*height] = adata[i+j*k];
+      }
+   }
+}
+
+void DenseMatrix::SetSubMatrix(int ibeg, int jbeg, const DenseMatrix & A)
+{
+   int k = A.Height();
+   int l = A.Width();
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = ibeg + i;
+      for (int j = 0; j<l; j++)
+      {
+         jj = jbeg + j;
+         this->data[ii+jj*height] = adata[i+j*k];
+      }
+   }
+}
+
+void DenseMatrix::AddSubMatrix(const Array<int> & idx, const DenseMatrix & A)
+{
+   int k = idx.Size();
+   MFEM_VERIFY(A.Height() == k &&
+               A.Width() == k, "DenseMatrix::SetSubMatrix:Inconsistent matrix dimensions");
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = idx[i];
+      for (int j = 0; j<k; j++)
+      {
+         jj = idx[j];
+         this->data[ii+jj*height] += adata[i+j*k];
+      }
+   }
+}
+
+void DenseMatrix::AddSubMatrix(const Array<int> & idx_i,
+                               const Array<int> & idx_j, const DenseMatrix & A)
+{
+   int k = idx_i.Size();
+   int l = idx_j.Size();
+   MFEM_VERIFY(k == A.Height() &&
+               l == A.Width(),"DenseMatrix::SetSubMatrix:Inconsistent matrix dimensions");
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = idx_i[i];
+      for (int j = 0; j<l; j++)
+      {
+         jj = idx_j[j];
+         this->data[ii+jj*height] += adata[i+j*k];
+      }
+   }
+}
+
+void DenseMatrix::AddSubMatrix(int ibeg, const DenseMatrix & A)
+{
+   int k = A.Height();
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = ibeg + i;
+      for (int j = 0; j<k; j++)
+      {
+         jj = ibeg + j;
+         this->data[ii+jj*height] += adata[i+j*k];
+      }
+   }
+}
+
+void DenseMatrix::AddSubMatrix(int ibeg, int jbeg, const DenseMatrix & A)
+{
+   int k = A.Height();
+   int l = A.Width();
+   double * adata = A.Data();
+
+   int ii, jj;
+   for (int i = 0; i<k; i++)
+   {
+      ii = ibeg + i;
+      for (int j = 0; j<l; j++)
+      {
+         jj = jbeg + j;
+         this->data[ii+jj*height] += adata[i+j*k];
+      }
+   }
+}
+
+
 void DenseMatrix::AddToVector(int offset, Vector &v) const
 {
    const int n = height * width;
@@ -2857,6 +3084,25 @@ void AddMult_a_VVt(const double a, const Vector &v, DenseMatrix &VVt)
    }
 }
 
+void RAP(const DenseMatrix &A, const DenseMatrix &P, DenseMatrix & RAP)
+{
+   DenseMatrix RA(P.Width(),A.Width());
+   MultAtB(P,A,RA);
+   RAP.SetSize(RA.Height(), P.Width());
+   Mult(RA,P, RAP);
+}
+
+/// General R^tAP with given R, A and P
+void RAP(const DenseMatrix &Rt, const DenseMatrix &A,
+         const DenseMatrix &P, DenseMatrix & RAP)
+{
+   DenseMatrix RA(Rt.Width(),A.Width());
+   MultAtB(Rt,A,RA);
+   RAP.SetSize(RA.Height(), P.Width());
+   Mult(RA,P, RAP);
+}
+
+
 
 bool LUFactors::Factor(int m, double TOL)
 {

linalg/densemat.hpp

@@ -355,6 +355,29 @@ public:
    /// Perform (ro+i,co+j)+=a*A(i,j) for 0<=i<A.Height, 0<=j<A.Width
    void AddMatrix(double a, const DenseMatrix &A, int ro, int co);
 
+   /// GetSubMatrix
+   void GetSubMatrix(const Array<int> & idx, DenseMatrix & A) const;
+   void GetSubMatrix(const Array<int> & idx_i, const Array<int> & idx_j,
+                     DenseMatrix & A) const;
+
+   // Get submatrix i,j ∈ [ibeg, iend)
+   void GetSubMatrix(int ibeg, int iend, DenseMatrix & A);
+   // Get submatrix i ∈ [ibeg, iend), j ∈ [jbeg, jend)
+   void GetSubMatrix(int ibeg, int iend, int jbeg, int jend, DenseMatrix & A);
+
+   void SetSubMatrix(const Array<int> & idx, const DenseMatrix & A);
+   void SetSubMatrix(const Array<int> & idx_i, const Array<int> & idx_j,
+                     const DenseMatrix & A);
+   void SetSubMatrix(int ibeg, const DenseMatrix & A);
+   void SetSubMatrix(int ibeg, int jbeg, const DenseMatrix & A);
+
+   void AddSubMatrix(const Array<int> & idx, const DenseMatrix & A);
+   void AddSubMatrix(const Array<int> & idx_i, const Array<int> & idx_j,
+                     const DenseMatrix & A);
+   void AddSubMatrix(int ibeg, const DenseMatrix & A);
+   void AddSubMatrix(int ibeg, int jbeg, const DenseMatrix & A);
+
+
    /// Add the matrix 'data' to the Vector 'v' at the given 'offset'
    void AddToVector(int offset, Vector &v) const;
    /// Get the matrix 'data' from the Vector 'v' at the given 'offset'
@@ -523,6 +546,12 @@ void AddMult_a_VWt(const double a, const Vector &v, const Vector &w,
 /// VVt += a * v v^t
 void AddMult_a_VVt(const double a, const Vector &v, DenseMatrix &VVt);
 
+/// Computes matrix P^t * A * P
+void RAP(const DenseMatrix &A, const DenseMatrix &P, DenseMatrix & RAP);
+
+/// Computes the matrix Rt^t * A * P
+void RAP(const DenseMatrix &Rt, const DenseMatrix &A,
+         const DenseMatrix &P, DenseMatrix & RAP);
 
 /** Abstract class that can compute factorization of external data and perform various
     operations with the factored data. */

linalg/operator.hpp

@@ -254,18 +254,20 @@ public:
    /** This enumeration is primarily used with class OperatorHandle. */
    enum Type
    {
-      ANY_TYPE,         ///< ID for the base class Operator, i.e. any type.
-      MFEM_SPARSEMAT,   ///< ID for class SparseMatrix.
-      Hypre_ParCSR,     ///< ID for class HypreParMatrix.
-      PETSC_MATAIJ,     ///< ID for class PetscParMatrix, MATAIJ format.
-      PETSC_MATIS,      ///< ID for class PetscParMatrix, MATIS format.
-      PETSC_MATSHELL,   ///< ID for class PetscParMatrix, MATSHELL format.
-      PETSC_MATNEST,    ///< ID for class PetscParMatrix, MATNEST format.
-      PETSC_MATHYPRE,   ///< ID for class PetscParMatrix, MATHYPRE format.
-      PETSC_MATGENERIC, ///< ID for class PetscParMatrix, unspecified format.
-      Complex_Operator, ///< ID for class ComplexOperator.
+      ANY_TYPE,              ///< ID for the base class Operator, i.e. any type.
+      MFEM_SPARSEMAT,        ///< ID for class SparseMatrix.
+      Hypre_ParCSR,          ///< ID for class HypreParMatrix.
+      PETSC_MATAIJ,          ///< ID for class PetscParMatrix, MATAIJ format.
+      PETSC_MATIS,           ///< ID for class PetscParMatrix, MATIS format.
+      PETSC_MATSHELL,        ///< ID for class PetscParMatrix, MATSHELL format.
+      PETSC_MATNEST,         ///< ID for class PetscParMatrix, MATNEST format.
+      PETSC_MATHYPRE,        ///< ID for class PetscParMatrix, MATHYPRE format.
+      PETSC_MATGENERIC,      ///< ID for class PetscParMatrix, unspecified format.
+      Complex_Operator,      ///< ID for class ComplexOperator.
       MFEM_ComplexSparseMat, ///< ID for class ComplexSparseMatrix.
-      Complex_Hypre_ParCSR   ///< ID for class ComplexHypreParMatrix.
+      Complex_Hypre_ParCSR,   ///< ID for class ComplexHypreParMatrix.
+      MFEM_Block_Matrix,     ///< ID for class BlockMatrix.
+      MFEM_Block_Operator,   ///< ID for the base class BlockOperator.
    };
 
    /// Return the type ID of the Operator class.

linalg/vector.cpp

@@ -289,6 +289,19 @@ void Vector::SetVector(const Vector &v, int offset)
    }
 }
 
+void Vector::AddSubVector(const Vector &v, int offset)
+{
+   MFEM_ASSERT(v.Size() + offset <= size, "invalid sub-vector");
+
+   const int vs = v.Size();
+   const double *vp = v.data;
+   double *p = data + offset;
+   for (int i = 0; i < vs; i++)
+   {
+      p[i] += vp[i];
+   }
+}
+
 void Vector::Neg()
 {
    const bool use_dev = UseDevice();

linalg/vector.hpp

@@ -314,6 +314,8 @@ public:
 
    void SetVector(const Vector &v, int offset);
 
+   void AddSubVector(const Vector &v, int offset);
+
    /// (*this) = -(*this)
    void Neg();