Merge pull request #586 from mfem/laplacian_dev

Adding the Hessian and Laplacian to the NURBS shapefunctions [laplacian_dev]

.gitignore

@@ -182,9 +182,31 @@ miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
 
-miniapps/nurbs/ex1
-miniapps/nurbs/ex1p
-miniapps/nurbs/ex11p
+miniapps/toys/automata
+miniapps/toys/life
+miniapps/toys/mandel
+miniapps/toys/rubik
+miniapps/toys/snake
+miniapps/toys/lissajous
+miniapps/toys/mondrian
+
+miniapps/toys/snake-init.mesh
+miniapps/toys/snake-user.mesh
+miniapps/toys/snake-joined.mesh
+miniapps/toys/snake-c*.mesh
+miniapps/toys/automata.gf
+miniapps/toys/automata.mesh
+miniapps/toys/rubik-init.mesh
+miniapps/toys/mandel.mesh
+miniapps/toys/life.gf
+miniapps/toys/life.mesh
+miniapps/toys/lissajous.mesh
+miniapps/toys/lissajous.gf
+miniapps/toys/mondrian.mesh
+
+miniapps/nurbs/nurbs_ex1
+miniapps/nurbs/nurbs_ex1p
+miniapps/nurbs/nurbs_ex11p
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol.*

CHANGELOG

@@ -67,6 +67,10 @@ Discretization improvements
   in physical space. See INSTALL for details on building MFEM with GSLIB, and
   miniapps/gslib for examples of how to use this feature.
 
+- Added support for serendipity elements of arbitrary order on affinely-mapped
+  square elements. Basis functions for these elements can be visualized using
+  an option in the display-basis miniapp.
+
 - Added support for complex-valued finite element operators and fields using a
   2x2 block structured linear system to mimic complex arithmetic. New classes
   include: ComplexGridFunction, SesquilinearForm, ComplexLinearForm, and their
@@ -82,6 +86,8 @@ Discretization improvements
   See the new methods AssembleDiagonal in BilinearForm, AssembleDiagonalPA in
   BilinearFormIntegrator and the implementations in fem/bilininteg_*.cpp.
 
+- Added second order derivatives of NURBS shape functions.
+
 - Added initial support for NonlinearForms to support the partial assembly mode.
 
 - Added a nonlinear vector valued convection integrator (Q u \cdot grad u, v)
@@ -112,6 +118,13 @@ Linear and nonlinear solvers
 
 - Added Adams-Bashforth and Adams-Moulton time integrators.
 
+- Added a block ILU(0) preconditioner for DG-type discretizations. Example 9
+  (DG advection) now takes advantage of this for implicit time integration.
+
+- Added a LinearSolve(A,X) convenience method to solve dense linear systems. In
+  the trivial cases, i.e., square matrices of size 1 or 2, the system is solved
+  directly, otherwise, LU factorization is employed.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added two new miniapps: Find Points (serial + parallel) and Field Diff in
@@ -131,6 +144,9 @@ New and updated examples and miniapps
 
 - New options to reorder and partition the mesh in the mesh-explorer miniapp.
 
+- The mesh-explorer miniapp now supports visualization of boundary attributes of
+  3D meshes (key 'b').
+
 - The (p)mesh-optimizer miniapp has been updated to demonstrate mesh
   optimization for an AMR mesh.
 
@@ -140,6 +156,14 @@ New and updated examples and miniapps
 - Added a modification of ex9 in examples/hiop that demonstrates the nonlinear
   constrained optimization interface and the use of the SLBQP and HiOp solvers.
 
+- Added a collection of 7 playful miniapps in miniapps/toys that illustrate the
+  meshing and visualization features of the library in more relaxed settings.
+  The toys include simulations of cellular automata, Rubik's cube, Mandelbrot
+  set, a tool to convert any image to mfem mesh, and more.
+
+- Example 9 and 9p now support implicit time integration, using the new block
+  ILU(0) solvers as preconditioners for the linear system.
+
 Improved testing
 ----------------
 - Added a new directory, tests/scripts, with several shell scripts that perform

doc/CodeDocumentation.conf.in

@@ -771,12 +771,13 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/examples/hiop \
                          @MFEM_SOURCE_DIR@/examples/sundials \
                          @MFEM_SOURCE_DIR@/miniapps/common \
-                         @MFEM_SOURCE_DIR@/miniapps/meshing \
-                         @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
-                         @MFEM_SOURCE_DIR@/miniapps/gslib \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
-                         @MFEM_SOURCE_DIR@/miniapps/performance
+                         @MFEM_SOURCE_DIR@/miniapps/gslib \
+                         @MFEM_SOURCE_DIR@/miniapps/meshing \
+                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
+                         @MFEM_SOURCE_DIR@/miniapps/performance \
+                         @MFEM_SOURCE_DIR@/miniapps/tools \
+                         @MFEM_SOURCE_DIR@/miniapps/toys
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

examples/ex10.cpp

@@ -179,7 +179,9 @@ int main(int argc, char *argv[])
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                   "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
                   "            11 - Forward Euler, 12 - RK2,\n\t"
-                  "            13 - RK3 SSP, 14 - RK4.");
+                  "            13 - RK3 SSP, 14 - RK4."
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",

examples/ex10p.cpp

@@ -193,7 +193,9 @@ int main(int argc, char *argv[])
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                   "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
                   "            11 - Forward Euler, 12 - RK2,\n\t"
-                  "            13 - RK3 SSP, 14 - RK4.");
+                  "            13 - RK3 SSP, 14 - RK4."
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",

examples/ex9.cpp

@@ -9,6 +9,7 @@
 //    ex9 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
+//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 13 -tf 9
 //    ex9 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
@@ -21,13 +22,13 @@
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of explicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) and ParaView (paraview.org) is also
-//               illustrated.
+//               bilinear forms in MFEM (face integrators), the use of implicit
+//               and explicit ODE time integrators, the definition of periodic
+//               boundary conditions through periodic meshes, as well as the use
+//               of GLVis for persistent visualization of a time-evolving
+//               solution. The saving of time-dependent data files for external
+//               visualization with VisIt (visit.llnl.gov) and ParaView
+//               (paraview.org) is also illustrated.
 
 #include "mfem.hpp"
 #include <fstream>
@@ -53,6 +54,54 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
+class DG_Solver : public Solver
+{
+private:
+   SparseMatrix &M, &K, A;
+   GMRESSolver linear_solver;
+   BlockILU prec;
+   double dt;
+public:
+   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
+      : M(M_),
+        K(K_),
+        prec(fes.GetFE(0)->GetDof(),
+             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
+        dt(-1.0)
+   {
+      linear_solver.iterative_mode = false;
+      linear_solver.SetRelTol(1e-9);
+      linear_solver.SetAbsTol(0.0);
+      linear_solver.SetMaxIter(100);
+      linear_solver.SetPrintLevel(0);
+      linear_solver.SetPreconditioner(prec);
+   }
+
+   void SetTimeStep(double dt_)
+   {
+      if (dt_ != dt)
+      {
+         dt = dt_;
+         // Form operator A = M - dt*K
+         A = K;
+         A *= -dt;
+         A += M;
+
+         // this will also call SetOperator on the preconditioner
+         linear_solver.SetOperator(A);
+      }
+   }
+
+   void SetOperator(const Operator &op)
+   {
+      linear_solver.SetOperator(op);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      linear_solver.Mult(x, y);
+   }
+};
 
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -66,13 +115,16 @@ private:
    const Vector &b;
    DSmoother M_prec;
    CGSolver M_solver;
+   DG_Solver dg_solver;
 
    mutable Vector z;
 
 public:
-   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b);
+   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b,
+                const FiniteElementSpace &fes);
 
    virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
 
    virtual ~FE_Evolution() { }
 };
@@ -108,7 +160,11 @@ int main(int argc, char *argv[])
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                   "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - Backward Euler,\n\t"
+                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -145,11 +201,21 @@ int main(int argc, char *argv[])
    ODESolver *ode_solver = NULL;
    switch (ode_solver_type)
    {
+      // Explicit methods
       case 1: ode_solver = new ForwardEulerSolver; break;
       case 2: ode_solver = new RK2Solver(1.0); break;
       case 3: ode_solver = new RK3SSPSolver; break;
       case 4: ode_solver = new RK4Solver; break;
       case 6: ode_solver = new RK6Solver; break;
+      // Implicit (L-stable) methods
+      case 11: ode_solver = new BackwardEulerSolver; break;
+      case 12: ode_solver = new SDIRK23Solver(2); break;
+      case 13: ode_solver = new SDIRK33Solver; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+
       default:
          cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
          return 3;
@@ -279,7 +345,7 @@ int main(int argc, char *argv[])
    // 8. Define the time-dependent evolution operator describing the ODE
    //    right-hand side, and perform time-integration (looping over the time
    //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m.SpMat(), k.SpMat(), b);
+   FE_Evolution adv(m.SpMat(), k.SpMat(), b, fes);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -337,8 +403,10 @@ int main(int argc, char *argv[])
 
 
 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b)
-   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), z(_M.Size())
+FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b,
+                           const FiniteElementSpace &fes)
+   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), dg_solver(M, K, fes),
+     z(_M.Size())
 {
    M_solver.SetPreconditioner(M_prec);
    M_solver.SetOperator(M);
@@ -358,6 +426,13 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
    M_solver.Mult(z, y);
 }
 
+void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
+{
+   K.Mult(x, z);
+   z += b;
+   dg_solver.SetTimeStep(dt);
+   dg_solver.Mult(z, k);
+}
 
 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)

examples/ex9p.cpp

@@ -9,6 +9,7 @@
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-hexagon.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9
+//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 13 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-mixed.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
@@ -21,13 +22,13 @@
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of explicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) and ParaView (paraview.org) is also
-//               illustrated.
+//               bilinear forms in MFEM (face integrators), the use of implicit
+//               and explicit ODE time integrators, the definition of periodic
+//               boundary conditions through periodic meshes, as well as the use
+//               of GLVis for persistent visualization of a time-evolving
+//               solution. The saving of time-dependent data files for external
+//               visualization with VisIt (visit.llnl.gov) and ParaView
+//               (paraview.org) is also illustrated.
 
 #include "mfem.hpp"
 #include <fstream>
@@ -52,6 +53,66 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
+class DG_Solver : public Solver
+{
+private:
+   HypreParMatrix &M, &K;
+   SparseMatrix M_diag;
+   HypreParMatrix *A;
+   GMRESSolver linear_solver;
+   BlockILU prec;
+   double dt;
+public:
+   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
+      : M(M_),
+        K(K_),
+        A(NULL),
+        linear_solver(M.GetComm()),
+        prec(fes.GetFE(0)->GetDof(),
+             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
+        dt(-1.0)
+   {
+      linear_solver.iterative_mode = false;
+      linear_solver.SetRelTol(1e-9);
+      linear_solver.SetAbsTol(0.0);
+      linear_solver.SetMaxIter(100);
+      linear_solver.SetPrintLevel(0);
+      linear_solver.SetPreconditioner(prec);
+
+      M.GetDiag(M_diag);
+   }
+
+   void SetTimeStep(double dt_)
+   {
+      if (dt_ != dt)
+      {
+         dt = dt_;
+         // Form operator A = M - dt*K
+         delete A;
+         A = Add(-dt, K, 0.0, K);
+         SparseMatrix A_diag;
+         A->GetDiag(A_diag);
+         A_diag.Add(1.0, M_diag);
+         // this will also call SetOperator on the preconditioner
+         linear_solver.SetOperator(*A);
+      }
+   }
+
+   void SetOperator(const Operator &op)
+   {
+      linear_solver.SetOperator(op);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      linear_solver.Mult(x, y);
+   }
+
+   ~DG_Solver()
+   {
+      delete A;
+   }
+};
 
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -65,13 +126,16 @@ private:
    const Vector &b;
    HypreSmoother M_prec;
    CGSolver M_solver;
+   DG_Solver dg_solver;
 
    mutable Vector z;
 
 public:
-   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b);
+   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b,
+                const FiniteElementSpace &fes);
 
    virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
 
    virtual ~FE_Evolution() { }
 };
@@ -116,7 +180,11 @@ int main(int argc, char *argv[])
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                   "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - Backward Euler,\n\t"
+                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
+                  "            22 - Implicit Midpoint Method,\n\t"
+                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -160,11 +228,20 @@ int main(int argc, char *argv[])
    ODESolver *ode_solver = NULL;
    switch (ode_solver_type)
    {
+      // Explicit methods
       case 1: ode_solver = new ForwardEulerSolver; break;
       case 2: ode_solver = new RK2Solver(1.0); break;
       case 3: ode_solver = new RK3SSPSolver; break;
       case 4: ode_solver = new RK4Solver; break;
       case 6: ode_solver = new RK6Solver; break;
+      // Implicit (L-stable) methods
+      case 11: ode_solver = new BackwardEulerSolver; break;
+      case 12: ode_solver = new SDIRK23Solver(2); break;
+      case 13: ode_solver = new SDIRK33Solver; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
       default:
          if (myid == 0)
          {
@@ -330,7 +407,7 @@ int main(int argc, char *argv[])
    // 10. Define the time-dependent evolution operator describing the ODE
    //     right-hand side, and perform time-integration (looping over the time
    //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*M, *K, *B);
+   FE_Evolution adv(*M, *K, *B, *fes);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -411,9 +488,10 @@ int main(int argc, char *argv[])
 
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
-                           const Vector &_b)
+                           const Vector &_b, const FiniteElementSpace &fes)
    : TimeDependentOperator(_M.Height()),
-     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height())
+     M(_M), K(_K), b(_b), M_solver(M.GetComm()),
+     dg_solver(M, K, fes), z(_M.Height())
 {
    M_prec.SetType(HypreSmoother::Jacobi);
    M_solver.SetPreconditioner(M_prec);
@@ -426,6 +504,14 @@ FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
    M_solver.SetPrintLevel(0);
 }
 
+void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
+{
+   K.Mult(x, z);
+   z += b;
+   dg_solver.SetTimeStep(dt);
+   dg_solver.Mult(z, k);
+}
+
 void FE_Evolution::Mult(const Vector &x, Vector &y) const
 {
    // y = M^{-1} (K x + b)

fem/CMakeLists.txt

@@ -17,6 +17,9 @@ set(SRCS
   bilininteg_divergence.cpp
   bilininteg_gradient.cpp
   bilininteg_mass.cpp
+  bilininteg_divergence.cpp
+  bilininteg_gradient.cpp
+  bilininteg_convection.cpp
   bilininteg_vecdiffusion.cpp
   bilininteg_vecmass.cpp
   coefficient.cpp

fem/bilinearform.hpp

@@ -153,6 +153,9 @@ public:
    /** This method must be called before assembly. */
    void SetAssemblyLevel(AssemblyLevel assembly_level);
 
+   /// Get the assembly level
+   AssemblyLevel GetAssemblyLevel() {return assembly;}
+
    /** Enable the use of static condensation. For details see the description
        for class StaticCondensation in fem/staticcond.hpp This method should be
        called before assembly. If the number of unknowns after static

fem/bilininteg.cpp

@@ -901,6 +901,16 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
 }
 
 
+const IntegrationRule &ConvectionIntegrator::GetRule(const FiniteElement
+                                                     &trial_fe,
+                                                     const FiniteElement &test_fe,
+                                                     ElementTransformation &Trans)
+{
+   int order = Trans.OrderGrad(&trial_fe) + Trans.Order() + test_fe.GetOrder();
+
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
 void ConvectionIntegrator::AssembleElementMatrix(
    const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -1831,81 +1841,26 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
    const FiniteElement &trial_fe, const FiniteElement &test_fe,
    ElementTransformation &Trans, DenseMatrix &elmat)
 {
-   if ( test_fe.GetRangeType() == FiniteElement::SCALAR && VQ )
+   if (test_fe.GetRangeType() == FiniteElement::SCALAR
+       && trial_fe.GetRangeType() == FiniteElement::VECTOR)
    {
       // assume test_fe is scalar FE and trial_fe is vector FE
       int dim  = test_fe.GetDim();
       int trial_dof = trial_fe.GetDof();
       int test_dof = test_fe.GetDof();
       double w;
-
-      if (MQ)
-         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
-                    "   is not implemented for tensor materials");
-
-#ifdef MFEM_THREAD_SAFE
-      DenseMatrix trial_vshape(trial_dof, dim);
-      Vector shape(test_dof);
-      Vector D(dim);
-#else
-      trial_vshape.SetSize(trial_dof, dim);
-      shape.SetSize(test_dof);
-      D.SetSize(dim);
-#endif
-
-      elmat.SetSize (test_dof, trial_dof);
-
-      const IntegrationRule *ir = IntRule;
-      if (ir == NULL)
-      {
-         int order = (Trans.OrderW() + test_fe.GetOrder() + trial_fe.GetOrder());
-         ir = &IntRules.Get(test_fe.GetGeomType(), order);
-      }
-
-      elmat = 0.0;
-      for (int i = 0; i < ir->GetNPoints(); i++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(i);
-
-         Trans.SetIntPoint (&ip);
-
-         trial_fe.CalcVShape(Trans, trial_vshape);
-         test_fe.CalcShape(ip, shape);
-
-         w = ip.weight * Trans.Weight();
-         VQ->Eval(D, Trans, ip);
-         D *= w;
-
-         for (int d = 0; d < dim; d++)
-         {
-            for (int j = 0; j < test_dof; j++)
-            {
-               for (int k = 0; k < trial_dof; k++)
-               {
-                  elmat(j, k) += D[d] * shape(j) * trial_vshape(k, d);
-               }
-            }
-         }
-      }
-   }
-   else if ( test_fe.GetRangeType() == FiniteElement::SCALAR )
-   {
-      // assume test_fe is scalar FE and trial_fe is vector FE
-      int dim  = test_fe.GetDim();
-      int trial_dof = trial_fe.GetDof();
-      int test_dof = test_fe.GetDof();
-      double w;
-
-      if (VQ || MQ)
-         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
-                    "   is not implemented for vector/tensor permeability");
+      double Kv;
 
 #ifdef MFEM_THREAD_SAFE
       DenseMatrix trial_vshape(trial_dof, dim);
       Vector shape(test_dof);
+      Vector D(VQ ? VQ->GetVDim() : 0);
+      DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #else
       trial_vshape.SetSize(trial_dof, dim);
       shape.SetSize(test_dof);
+      D.SetSize(VQ ? VQ->GetVDim() : 0);
+      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif
 
       elmat.SetSize (dim*test_dof, trial_dof);
@@ -1928,24 +1883,64 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
          test_fe.CalcShape(ip, shape);
 
          w = ip.weight * Trans.Weight();
-         if (Q)
+         if (VQ)
          {
-            w *= Q -> Eval (Trans, ip);
-         }
-
-         for (int d = 0; d < dim; d++)
-         {
-            for (int j = 0; j < test_dof; j++)
+            VQ->Eval(D, Trans, ip);
+            D *= w;
+            for (int d = 0; d < dim; d++)
             {
-               for (int k = 0; k < trial_dof; k++)
+               for (int j = 0; j < test_dof; j++)
                {
-                  elmat(d * test_dof + j, k) += w * shape(j) * trial_vshape(k, d);
+                  for (int k = 0; k < trial_dof; k++)
+                  {
+                     elmat(d * test_dof + j, k) +=
+                        shape(j) * D(d) * trial_vshape(k, d);
+                  }
+               }
+            }
+         }
+         else if (MQ)
+         {
+            MQ->Eval(K, Trans, ip);
+            K *= w;
+            for (int d = 0; d < dim; d++)
+            {
+               for (int j = 0; j < test_dof; j++)
+               {
+                  for (int k = 0; k < trial_dof; k++)
+                  {
+                     Kv = 0.0;
+                     for (int vd = 0; vd < dim; vd++)
+                     {
+                        Kv += K(d, vd) * trial_vshape(k, vd);
+                     }
+                     elmat(d * test_dof + j, k) += shape(j) * Kv;
+                  }
+               }
+            }
+         }
+         else
+         {
+            if (Q)
+            {
+               w *= Q->Eval(Trans, ip);
+            }
+            for (int d = 0; d < dim; d++)
+            {
+               for (int j = 0; j < test_dof; j++)
+               {
+                  for (int k = 0; k < trial_dof; k++)
+                  {
+                     elmat(d * test_dof + j, k) +=
+                        w * shape(j) * trial_vshape(k, d);
+                  }
                }
             }
          }
       }
    }
-   else
+   else if (test_fe.GetRangeType() == FiniteElement::VECTOR
+            && trial_fe.GetRangeType() == FiniteElement::VECTOR)
    {
       // assume both test_fe and trial_fe are vector FE
       int dim  = test_fe.GetDim();
@@ -1953,17 +1948,18 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
       int test_dof = test_fe.GetDof();
       double w;
 
-      if (VQ || MQ)
-         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
-                    "   is not implemented for vector/tensor permeability");
-
 #ifdef MFEM_THREAD_SAFE
       DenseMatrix trial_vshape(trial_dof, dim);
       DenseMatrix test_vshape(test_dof,dim);
+      Vector D(VQ ? VQ->GetVDim() : 0);
+      DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #else
       trial_vshape.SetSize(trial_dof, dim);
       test_vshape.SetSize(test_dof,dim);
+      D.SetSize(VQ ? VQ->GetVDim() : 0);
+      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif
+      DenseMatrix tmp(trial_vshape.Height(), K.Width());
 
       elmat.SetSize (test_dof, trial_dof);
 
@@ -1985,23 +1981,34 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
          test_fe.CalcVShape(Trans, test_vshape);
 
          w = ip.weight * Trans.Weight();
-         if (Q)
+         if (MQ)
          {
-            w *= Q -> Eval (Trans, ip);
+            MQ->Eval(K, Trans, ip);
+            K *= w;
+            Mult(test_vshape,K,tmp);
+            AddMultABt(tmp,trial_vshape,elmat);
          }
-
-         for (int d = 0; d < dim; d++)
+         else if (VQ)
          {
-            for (int j = 0; j < test_dof; j++)
+            VQ->Eval(D, Trans, ip);
+            D *= w;
+            AddMultADBt(test_vshape,D,trial_vshape,elmat);
+         }
+         else
+         {
+            if (Q)
             {
-               for (int k = 0; k < trial_dof; k++)
-               {
-                  elmat(j, k) += w * test_vshape(j, d) * trial_vshape(k, d);
-               }
+               w *= Q -> Eval (Trans, ip);
             }
+            AddMult_a_ABt(w,test_vshape,trial_vshape,elmat);
          }
       }
    }
+   else
+   {
+      mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+                 "   is not implemented for given trial and test bases.");
+   }
 }
 
 void VectorDivergenceIntegrator::AssembleElementMatrix2(

fem/bilininteg.hpp

@@ -1927,12 +1927,29 @@ private:
    Vector shape, vec2, BdFidxT;
 #endif
 
+   // PA extension
+   Vector pa_data;
+   Vector coeff;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq, dofs1D, quad1D;
+
 public:
    ConvectionIntegrator(VectorCoefficient &q, double a = 1.0)
       : Q(&q) { alpha = a; }
    virtual void AssembleElementMatrix(const FiniteElement &,
                                       ElementTransformation &,
                                       DenseMatrix &);
+
+   using BilinearFormIntegrator::AssemblePA;
+
+   virtual void AssemblePA(const FiniteElementSpace&);
+
+   virtual void AddMultPA(const Vector&, Vector&) const;
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans);
 };
 
 /// alpha (q . grad u, v) using the "group" FE discretization
@@ -2181,7 +2198,10 @@ public:
                                    const Vector &elfun);
 };
 
-/// Integrator for (Q u, v) for VectorFiniteElements
+/** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
+    vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
+    H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
+    vi are in H1. */
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
@@ -2192,6 +2212,7 @@ private:
    Vector shape;
    Vector D;
    DenseMatrix K;
+   DenseMatrix partelmat;
    DenseMatrix test_vshape;
    DenseMatrix trial_vshape;
 #endif

fem/bilininteg_convection.cpp

@@ -0,0 +1,500 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// PA Convection Integrator
+
+// PA Convection Assemble kernel
+void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   // Assuming the same element type
+   Mesh *mesh = fes.GetMesh();
+   if (mesh->GetNE() == 0) { return; }
+   const FiniteElement &el = *fes.GetFE(0);
+   ElementTransformation *T = mesh->GetElementTransformation(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
+   dim = mesh->Dimension();
+   ne = fes.GetMesh()->GetNE();
+   nq = ir->GetNPoints();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
+                                    GeometricFactors::JACOBIANS);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   dofs1D = maps->ndof;
+   quad1D = maps->nqpt;
+   pa_data.SetSize(dim*ne*nq, Device::GetMemoryType());
+
+   const int NE = ne;
+   const int NQ = nq;
+   Vector e_coeff(dim);
+   coeff.SetSize(dim*ne*nq);
+   auto h_C = Reshape(coeff.HostWrite(),dim,nq, ne);
+
+   if ( Q == nullptr)
+   {
+      for (int e=0; e<NE; ++e)
+      {
+         for (int q=0; q<nq; ++q)
+         {
+            for (int idim=0; idim < dim; ++idim)
+            {
+               h_C(idim,q,e) = alpha;
+            }
+         }
+      }
+   }
+   else
+   {
+      for (int e=0; e<NE; ++e)
+      {
+         ElementTransformation& Te = *fes.GetElementTransformation(e);
+         for (int q=0; q<nq; ++q)
+         {
+            for (int idim=0; idim < dim; ++idim)
+            {
+               Q->Eval(e_coeff, Te, ir->IntPoint(q));
+               h_C(idim,q,e) = alpha*e_coeff(idim);
+            }
+         }
+      }
+   }
+
+   auto C = Reshape(coeff.Read(),dim,nq, ne);
+
+   if (dim==2)
+   {
+      auto w = ir->GetWeights().Read();
+      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
+      auto v = Reshape(pa_data.Write(), 2, NQ, NE);
+
+      MFEM_FORALL(e, NE,
+      {
+         for (int q=0; q<NQ; ++q)
+         {
+            const double J11 = J(q,0,0,e);
+            const double J21 = J(q,1,0,e);
+            const double J12 = J(q,0,1,e);
+            const double J22 = J(q,1,1,e);
+
+            const double cx = C(0,q,e);
+            const double cy = C(1,q,e);
+            const double w_coeff = w[q];
+            v(0,q,e) =    w_coeff*(cx * J22 - cy * J12);
+            v(1,q,e) =  - w_coeff*(cx * J21 - cy * J11);
+         }
+      });
+   }//dim = 2
+
+   if (dim==3)
+   {
+      auto w = ir->GetWeights().Read();
+      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
+      auto v = Reshape(pa_data.Write(), 3, NQ, NE);
+
+      MFEM_FORALL(e, NE,
+      {
+         for (int q=0; q<NQ; ++q)
+         {
+
+            const double J00 = J(q,0,0,e);
+            const double J01 = J(q,0,1,e);
+            const double J02 = J(q,0,2,e);
+
+            const double J10 = J(q,1,0,e);
+            const double J11 = J(q,1,1,e);
+            const double J12 = J(q,1,2,e);
+
+            const double J20 = J(q,2,0,e);
+            const double J21 = J(q,2,1,e);
+            const double J22 = J(q,2,2,e);
+
+            const double A00 = (J11 * J22) - (J12 * J21);
+            const double A01 = (J02 * J21) - (J01 * J22);
+            const double A02 = (J01 * J12) - (J02 * J11);
+
+            const double A10 = (J12 * J20) - (J10 * J22);
+            const double A11 = (J00 * J22) - (J02 * J20);
+            const double A12 = (J02 * J10) - (J00 * J12);
+
+            const double A20 = (J10 * J21) - (J11 * J20);
+            const double A21 = (J01 * J20) - (J00 * J21);
+            const double A22 = (J00 * J11) - (J01 * J10);
+
+            const double w_coeff = w[q];
+
+            double cx = C(0,q,e);
+            double cy = C(1,q,e);
+            double cz = C(2,q,e);
+
+            v(0,q,e) = w_coeff*(cx*A00 + cy*A01 + cz*A02);
+            v(1,q,e) = w_coeff*(cx*A10 + cy*A11 + cz*A12);
+            v(2,q,e) = w_coeff*(cx*A20 + cy*A21 + cz*A22);
+         }
+
+      });
+   }//dim = 3
+}
+
+// PA Convection Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0> static
+void PAConvectionApply2D(const int NE,
+                         const Array<double> &b,
+                         const Array<double> &g,
+                         const Array<double> &bt,
+                         const Array<double> &gt,
+                         const Vector &_op,
+                         const Vector &_x,
+                         Vector &_y,
+                         const int d1d = 0,
+                         const int q1d = 0)
+{
+   const int DIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto D = Reshape(_op.Read(), DIM, Q1D, Q1D, NE);
+   auto xloc = Reshape(_x.Read(), D1D, D1D, NE);
+   auto yloc = Reshape(_y.ReadWrite(), D1D, D1D, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      // the following variables are evaluated at compile time
+      constexpr int iDIM     = 2;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      double U[iDIM][max_D1D][max_Q1D];
+      for (int j1=0; j1<Q1D; ++j1)
+      {
+         for (int i2=0; i2<D1D; ++i2)
+         {
+
+            double dot0=0.0; double dot1=0.0;
+            for (int i1=0; i1<D1D; ++i1)
+            {
+               dot0 += G(j1,i1)*xloc(i1, i2, e);
+               dot1 += B(j1,i1)*xloc(i1, i2, e);
+            }
+            U[0][i2][j1] = dot0;
+            U[1][i2][j1] = dot1;
+         }
+      }
+
+      double W[iDIM][max_Q1D][max_Q1D];
+      for (int j1=0; j1<Q1D; ++j1)
+      {
+         for (int i2=0; i2<Q1D; ++i2)
+         {
+
+            double dot0=0.0; double dot1=0.0;
+            for (int i1=0; i1<D1D; ++i1)
+            {
+               dot0 += B(j1,i1)*U[0][i1][i2];
+               dot1 += G(j1,i1)*U[1][i1][i2];
+            }
+            W[0][i2][j1] = dot0;
+            W[1][i2][j1] = dot1;
+         }
+      }
+
+      double Z[max_Q1D][max_Q1D];
+      for (int k2=0; k2<Q1D; ++k2)
+      {
+         for (int k1=0; k1<Q1D; ++k1)
+         {
+
+            double dot(0.0);
+            for (int c=0; c<2; ++c)
+            {
+               dot += D(c, k1, k2, e) * W[c][k1][k2];
+            }
+            Z[k1][k2] = dot;
+         }
+      }
+
+      double Q[max_Q1D][max_D1D];
+      for (int j1=0; j1<D1D; ++j1)
+      {
+         for (int i2=0; i2<Q1D; ++i2)
+         {
+
+            double dot(0.0);
+            for (int i1=0; i1<Q1D; ++i1)
+            {
+               dot += Bt(j1, i1)*Z[i1][i2];
+            }
+            Q[i2][j1] = dot;
+         }
+      }
+
+      for (int j1=0; j1<D1D; ++j1)
+      {
+         for (int i2=0; i2<D1D; ++i2)
+         {
+
+            double dot(0.0);
+            for (int i1=0; i1<Q1D; ++i1)
+            {
+               dot += Bt(j1, i1)*Q[i1][i2];
+            }
+            yloc(i2,j1,e) += dot;
+         }
+      }
+   });
+
+}
+
+
+// PA Convection Apply 3D kernel
+template<const int T_D1D = 0,
+         const int T_Q1D = 0> static
+void PAConvectionApply3D(const int NE,
+                         const Array<double> &b,
+                         const Array<double> &g,
+                         const Array<double> &bt,
+                         const Array<double> &gt,
+                         const Vector &_op,
+                         const Vector &_x,
+                         Vector &_y,
+                         int d1d = 0, int q1d = 0)
+{
+   const int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto D = Reshape(_op.Read(), DIM, Q1D,Q1D, Q1D, NE);
+   auto xloc = Reshape(_x.Read(), D1D, D1D, D1D, NE);
+   auto yloc = Reshape(_y.ReadWrite(), D1D, D1D, D1D, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      //qpt x dof x dof
+      double BX[max_Q1D][max_Q1D][max_Q1D];
+      double GX[max_Q1D][max_Q1D][max_Q1D];
+      for (int j1=0; j1<Q1D; ++j1)
+      {
+         for (int i3=0; i3<D1D; ++i3)
+         {
+            for (int i2=0; i2<D1D; ++i2)
+            {
+
+               BX[i2][i3][j1] = 0.0;
+               GX[i2][i3][j1] = 0.0;
+               for (int i1=0; i1<D1D; ++i1)
+               {
+                  BX[i2][i3][j1] += B(j1, i1) * xloc(i1,i2,i3,e);
+                  GX[i2][i3][j1] += G(j1, i1) * xloc(i1,i2,i3,e);
+               }
+            }
+         }
+      }
+
+      double BBX[max_D1D][max_Q1D][max_Q1D];
+      double GBX[max_D1D][max_Q1D][max_Q1D];
+      double BGX[max_D1D][max_Q1D][max_Q1D];
+
+      for (int j1=0; j1<Q1D; ++j1)
+      {
+         for (int i3=0; i3<Q1D; ++i3)
+         {
+            for (int i2=0; i2<D1D; ++i2)
+            {
+
+               BBX[i2][i3][j1] = 0.0;
+               GBX[i2][i3][j1] = 0.0;
+               BGX[i2][i3][j1] = 0.0;
+               for (int i1=0; i1<D1D; ++i1)
+               {
+                  BBX[i2][i3][j1] += B(j1, i1) * BX[i1][i2][i3];
+                  GBX[i2][i3][j1] += G(j1, i1) * BX[i1][i2][i3];
+                  BGX[i2][i3][j1] += B(j1, i1) * GX[i1][i2][i3];
+               }
+
+            }
+         }
+      }
+
+      double GBBX[max_Q1D][max_Q1D][max_Q1D];
+      double BGBX[max_Q1D][max_Q1D][max_Q1D];
+      double BBGX[max_Q1D][max_Q1D][max_Q1D];
+
+      for (int j1=0; j1<Q1D; ++j1)
+      {
+         for (int i3=0; i3<Q1D; ++i3)
+         {
+            for (int i2=0; i2<Q1D; ++i2)
+            {
+
+               GBBX[i2][i3][j1] = 0.0;
+               BGBX[i2][i3][j1] = 0.0;
+               BBGX[i2][i3][j1] = 0.0;
+               for (int i1=0; i1<D1D; ++i1)
+               {
+                  GBBX[i2][i3][j1] += G(j1, i1) * BBX[i1][i2][i3];
+                  BGBX[i2][i3][j1] += B(j1, i1) * GBX[i1][i2][i3];
+                  BBGX[i2][i3][j1] += B(j1, i1) * BGX[i1][i2][i3];
+               }
+
+            }
+         }
+      }
+
+      double Z[max_Q1D][max_Q1D][max_Q1D];
+      for (int k3=0; k3<Q1D; ++k3)
+      {
+         for (int k2=0; k2<Q1D; ++k2)
+         {
+            for (int k1=0; k1<Q1D; ++k1)
+            {
+
+               double dot(0.0);
+               {
+                  dot += D(0, k1, k2, k3, e) * BBGX[k1][k2][k3];
+                  dot += D(1, k1, k2, k3, e) * BGBX[k1][k2][k3];
+                  dot += D(2, k1, k2, k3, e) * GBBX[k1][k2][k3];
+               }
+               Z[k1][k2][k3] = dot;
+            }
+         }
+      }
+
+      //Apply (B1d)^T 3 more times
+      double BZ[max_Q1D][max_Q1D][max_Q1D];
+      for (int j1=0; j1<D1D; ++j1)
+      {
+         for (int i3=0; i3<Q1D; ++i3)
+         {
+            for (int i2=0; i2<Q1D; ++i2)
+            {
+
+               BZ[i2][i3][j1]=0.0;
+               for (int i1=0; i1<Q1D; ++i1)
+               {
+                  BZ[i2][i3][j1] += Bt(j1,i1)*Z[i1][i2][i3];
+               }
+
+            }
+         }
+      }
+
+      double BBZ[max_Q1D][max_Q1D][max_Q1D];
+      for (int j1=0; j1<D1D; ++j1)
+      {
+         for (int i3=0; i3<D1D; ++i3)
+         {
+            for (int i2=0; i2<Q1D; ++i2)
+            {
+
+               BBZ[i2][i3][j1]=0.0;
+               for (int i1=0; i1<Q1D; ++i1)
+               {
+                  BBZ[i2][i3][j1] += Bt(j1,i1)*BZ[i1][i2][i3];
+               }
+
+            }
+         }
+      }
+
+      for (int j1=0; j1<D1D; ++j1)
+      {
+         for (int i3=0; i3<D1D; ++i3)
+         {
+            for (int i2=0; i2<D1D; ++i2)
+            {
+
+               double dot(0.0);
+               for (int i1=0; i1<Q1D; ++i1)
+               {
+                  dot += Bt(j1,i1)*BBZ[i1][i2][i3];
+               }
+               yloc(i2,i3,j1,e) += dot;
+            }
+         }
+      }
+
+   });
+
+}
+
+
+static void PAConvectionApply(const int dim,
+                              const int D1D,
+                              const int Q1D,
+                              const int NE,
+                              const Array<double> &B,
+                              const Array<double> &G,
+                              const Array<double> &Bt,
+                              const Array<double> &Gt,
+                              const Vector &op,
+                              const Vector &x,
+                              Vector &y)
+{
+
+   if (dim==2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         //case 0x22: PAConvectionApply2D<2,2>(NE, B, G, Bt, Gt, op, x, y); break;
+         //case 0x33: PAConvectionApply2D<3,3>(NE, B, G, Bt, Gt, op, x, y); break;
+         //case 0x44: PAConvectionApply2D<4,4>(NE, B, G, Bt, Gt, op, x, y); break;
+         //case 0x55: PAConvectionApply2D<5,5>(NE, B, G, Bt, Gt, op, x, y); break;
+         default: PAConvectionApply2D(NE, B, G, Bt, Gt, op, x, y,D1D,Q1D); break;
+      }
+      return;
+   }
+   if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         //case 0x23: PAConvectionApply3D<2,3>(NE, B, G, Bt, G, op, x, y); break;
+         //case 0x34: PAConvectionApply3D<3,4>(NE, B, G, Bt, G, op, x, y); break;
+         //case 0x45: PAConvectionApply3D<4,5>(NE, B, G, Bt, G, op, x, y); break;
+         //case 0x56: PAConvectionApply3D<5,6>(NE, B, G, Bt, G, op, x, y); break;
+         default: PAConvectionApply3D(NE, B, G, Bt, G, op, x, y,D1D,Q1D); break;
+      }
+      return;
+   }
+   MFEM_ABORT("Unknown kernel.");
+
+}
+
+void ConvectionIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   PAConvectionApply(dim, dofs1D, quad1D, ne,maps->B, maps->G, maps->Bt, maps->Gt,
+                     pa_data, x, y);
+}
+
+} // namespace mfem

fem/complex_fem.cpp

@@ -19,7 +19,7 @@ namespace mfem
 ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
    : Vector(2*(fes->GetVSize()))
 {
-   gfr = new GridFunction(fes, &data[0]);
+   gfr = new GridFunction(fes, data);
    gfi = new GridFunction(fes, &data[fes->GetVSize()]);
 }
 
@@ -43,8 +43,8 @@ ComplexGridFunction::Update()
       this->SetSize(2 * vsize);
 
       // Create temporary vectors which point to the new data array
-      Vector gf_r(&data[0], vsize);
-      Vector gf_i(&data[vsize], vsize);
+      Vector gf_r(data, vsize);
+      Vector gf_i((data) ? &data[vsize] : data, vsize);
 
       // Copy the updated GridFunctions into the new data array
       gf_r = *gfr;
@@ -52,8 +52,8 @@ ComplexGridFunction::Update()
 
       // Replace the individual data arrays with pointers into the new data
       // array
-      gfr->NewDataAndSize(&data[0], vsize);
-      gfi->NewDataAndSize(&data[vsize], vsize);
+      gfr->NewDataAndSize(data, vsize);
+      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
    }
    else
    {
@@ -62,8 +62,8 @@ ComplexGridFunction::Update()
       this->SetSize(2 * vsize);
 
       // Point the individual GridFunctions to the new data array
-      gfr->NewDataAndSize(&data[0], vsize);
-      gfi->NewDataAndSize(&data[vsize], vsize);
+      gfr->NewDataAndSize(data, vsize);
+      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
 
       // These updates will only set the proper 'sequence' value within
       // the individual GridFunction objects because their sizes are
@@ -124,10 +124,20 @@ ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *f,
    : Vector(2*(f->GetVSize())),
      conv(convention)
 {
-   lfr = new LinearForm(f, &data[0]);
+   lfr = new LinearForm(f, data);
    lfi = new LinearForm(f, &data[f->GetVSize()]);
 }
 
+ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
+                                     LinearForm *lf_r, LinearForm *lf_i,
+                                     ComplexOperator::Convention convention)
+   : Vector(2*(fes->GetVSize())),
+     conv(convention)
+{
+   lfr = new LinearForm(fes, lf_r);  lfr->SetData(data);
+   lfi = new LinearForm(fes, lf_i);  lfi->SetData(&data[fes->GetVSize()]);
+}
+
 ComplexLinearForm::~ComplexLinearForm()
 {
    delete lfr;
@@ -190,8 +200,8 @@ ComplexLinearForm::Update(FiniteElementSpace *fes)
    int vsize = fes->GetVSize();
    SetSize(2 * vsize);
 
-   Vector vlfr(&data[0], vsize);
-   Vector vlfi(&data[vsize], vsize);
+   Vector vlfr(data, vsize);
+   Vector vlfi((data) ? &data[vsize] : data, vsize);
 
    lfr->Update(fes, vlfr, 0);
    lfi->Update(fes, vlfi, 0);
@@ -216,6 +226,19 @@ ComplexLinearForm::operator()(const ComplexGridFunction &gf) const
                           (*lfr)(gf.imag()) + s * (*lfi)(gf.real()));
 }
 
+bool SesquilinearForm::RealInteg()
+{
+   int nint = blfr->GetFBFI()->Size() + blfr->GetDBFI()->Size() +
+              blfr->GetBBFI()->Size() + blfr->GetBFBFI()->Size();
+   return (nint != 0);
+}
+
+bool SesquilinearForm::ImagInteg()
+{
+   int nint = blfi->GetFBFI()->Size() + blfi->GetDBFI()->Size() +
+              blfi->GetBBFI()->Size() + blfi->GetBFBFI()->Size();
+   return (nint != 0);
+}
 
 SesquilinearForm::SesquilinearForm(FiniteElementSpace *f,
                                    ComplexOperator::Convention convention)
@@ -224,6 +247,19 @@ SesquilinearForm::SesquilinearForm(FiniteElementSpace *f,
      blfi(new BilinearForm(f))
 {}
 
+SesquilinearForm::SesquilinearForm(FiniteElementSpace *f,
+                                   BilinearForm *bfr, BilinearForm *bfi,
+                                   ComplexOperator::Convention convention)
+   : conv(convention),
+     blfr(new BilinearForm(f,bfr)),
+     blfi(new BilinearForm(f,bfi))
+{}
+
+void SesquilinearForm::SetDiagonalPolicy(mfem::Matrix::DiagonalPolicy dpolicy)
+{
+   diag_policy = dpolicy;
+}
+
 SesquilinearForm::~SesquilinearForm()
 {
    delete blfr;
@@ -297,7 +333,6 @@ SesquilinearForm::AssembleComplexSparseMatrix()
    return new ComplexSparseMatrix(&blfr->SpMat(),
                                   &blfi->SpMat(),
                                   false, false, conv);
-
 }
 
 void
@@ -311,8 +346,6 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    int vsize  = fes->GetVSize();
 
-   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
-
    // Allocate temporary vectors
    Vector b_0(vsize);  b_0 = 0.0;
 
@@ -324,38 +357,103 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
    MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
    Vector b_r(b.GetData(), vsize);
    Vector b_i(&(b.GetData())[vsize], vsize);
-   b_i *= s;
 
-   SparseMatrix * A_r = new SparseMatrix;
-   SparseMatrix * A_i = new SparseMatrix;
-   Vector X_0, B_0;
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
 
-   b_0 = b_r;
-   blfr->FormLinearSystem(ess_tdof_list, x_r, b_r, *A_r, X_0, B_0, ci);
+   int tvsize = fes->GetTrueVSize();
+   SparseMatrix * A_r = nullptr;
+   SparseMatrix * A_i = nullptr;
 
-   int tvsize = B_0.Size();
    X.SetSize(2 * tvsize);
    B.SetSize(2 * tvsize);
-   Vector X_r(X.GetData(), tvsize);
+
+   Vector X_0(tvsize), B_0(tvsize);
+   Vector X_r(X.GetData(),tvsize);
    Vector X_i(&(X.GetData())[tvsize], tvsize);
    Vector B_r(B.GetData(), tvsize);
    Vector B_i(&(B.GetData())[tvsize], tvsize);
-   X_r = X_0; B_r = B_0;
 
-   b_0 = 0.0;
-   blfi->FormLinearSystem(ess_tdof_list, x_i, b_0, *A_i, X_0, B_0, false);
-   B_r -= B_0;
+   if (RealInteg())
+   {
+      A_r = new SparseMatrix;
+      blfr->SetDiagonalPolicy(diag_policy);
 
-   b_0 = b_i;
-   blfr->FormLinearSystem(ess_tdof_list, x_i, b_0, *A_r, X_0, B_0, ci);
-   X_i = X_0; B_i = B_0;
+      b_0 = b_r;
+      blfr->FormLinearSystem(ess_tdof_list, x_r, b_0, *A_r, X_0, B_0, ci);
+      X_r = X_0; B_r = B_0;
 
-   b_0 = 0.0;
-   blfi->FormLinearSystem(ess_tdof_list, x_r, b_0, *A_i, X_0, B_0, false);
-   B_i += B_0;
+      b_0 = b_i;
+      blfr->FormLinearSystem(ess_tdof_list, x_i, b_0, *A_r, X_0, B_0, ci);
+      X_i = X_0; B_i = B_0;
 
-   B_i *= s;
-   b_i *= s;
+      if (ImagInteg())
+      {
+         A_i = new SparseMatrix;
+         blfi->SetDiagonalPolicy(mfem::Matrix::DiagonalPolicy::DIAG_ZERO);
+
+         b_0 = 0.0;
+         blfi->FormLinearSystem(ess_tdof_list, x_i, b_0, *A_i, X_0, B_0, false);
+         B_r -= B_0;
+
+         b_0 = 0.0;
+         blfi->FormLinearSystem(ess_tdof_list, x_r, b_0, *A_i, X_0, B_0, false);
+         B_i += B_0;
+      }
+   }
+   else if (ImagInteg())
+   {
+      A_i = new SparseMatrix;
+      blfi->SetDiagonalPolicy(diag_policy);
+
+      b_0 = b_i;
+      blfi->FormLinearSystem(ess_tdof_list, x_r, b_0, *A_i, X_0, B_0, ci);
+      X_r = X_0; B_i = B_0;
+
+      b_0 = b_r; b_0 *= -1.0;
+      blfi->FormLinearSystem(ess_tdof_list, x_i, b_0, *A_i, X_0, B_0, ci);
+      X_i = X_0; B_r = B_0; B_r *= -1.0;
+   }
+   else
+   {
+      MFEM_ABORT("Real and Imaginary part of the Sesquilinear form are empty");
+   }
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      B_i *= -1.0;
+      b_i *= -1.0;
+   }
+   // A = A_r + i A_i
+   A.Clear();
+   ComplexSparseMatrix * A_sp;
+   A_sp = new ComplexSparseMatrix(A_r, A_i, true, true, conv);
+   A.Reset<ComplexSparseMatrix>(A_sp, true);
+}
+
+void
+SesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                   OperatorHandle &A)
+
+{
+   SparseMatrix * A_r = nullptr;
+   SparseMatrix * A_i = nullptr;
+
+   if (RealInteg())
+   {
+      A_r = new SparseMatrix;
+      blfr->SetDiagonalPolicy(diag_policy);
+      blfr->FormSystemMatrix(ess_tdof_list, *A_r);
+   }
+   if (ImagInteg())
+   {
+      A_i = new SparseMatrix;
+      blfr->SetDiagonalPolicy(diag_policy);
+      blfi->FormSystemMatrix(ess_tdof_list, *A_i);
+   }
+   if (!RealInteg() && !ImagInteg())
+   {
+      MFEM_ABORT("Both Real and Imaginary part of the Sesquilinear form are empty");
+   }
 
    // A = A_r + i A_i
    A.Clear();
@@ -406,8 +504,8 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
 ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
    : Vector(2*(pfes->GetVSize()))
 {
-   pgfr = new ParGridFunction(pfes, &data[0]);
-   pgfi = new ParGridFunction(pfes, &data[pfes->GetVSize()]);
+   pgfr = new ParGridFunction(pfes, data);
+   pgfi = new ParGridFunction(pfes, (data) ? &data[pfes->GetVSize()]:data);
 }
 
 void
@@ -430,8 +528,8 @@ ParComplexGridFunction::Update()
       this->SetSize(2 * vsize);
 
       // Create temporary vectors which point to the new data array
-      Vector gf_r(&data[0], vsize);
-      Vector gf_i(&data[vsize], vsize);
+      Vector gf_r(data, vsize);
+      Vector gf_i((data) ? &data[vsize] : data, vsize);
 
       // Copy the updated GridFunctions into the new data array
       gf_r = *pgfr;
@@ -439,8 +537,8 @@ ParComplexGridFunction::Update()
 
       // Replace the individual data arrays with pointers into the new data
       // array
-      pgfr->NewDataAndSize(&data[0], vsize);
-      pgfi->NewDataAndSize(&data[vsize], vsize);
+      pgfr->NewDataAndSize(data, vsize);
+      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
    }
    else
    {
@@ -449,8 +547,8 @@ ParComplexGridFunction::Update()
       this->SetSize(2 * vsize);
 
       // Point the individual GridFunctions to the new data array
-      pgfr->NewDataAndSize(&data[0], vsize);
-      pgfi->NewDataAndSize(&data[vsize], vsize);
+      pgfr->NewDataAndSize(data, vsize);
+      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
 
       // These updates will only set the proper 'sequence' value within the
       // individual GridFunction objects because their sizes are already correct
@@ -514,7 +612,7 @@ ParComplexGridFunction::Distribute(const Vector *tv)
 
    double * tvd = tv->GetData();
    Vector tvr(tvd, size);
-   Vector tvi(&tvd[size], size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);
 
    pgfr->Distribute(tvr);
    pgfi->Distribute(tvi);
@@ -528,7 +626,7 @@ ParComplexGridFunction::ParallelProject(Vector &tv) const
 
    double * tvd = tv.GetData();
    Vector tvr(tvd, size);
-   Vector tvi(&tvd[size], size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);
 
    pgfr->ParallelProject(tvr);
    pgfi->ParallelProject(tvi);
@@ -541,8 +639,32 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
    : Vector(2*(pfes->GetVSize())),
      conv(convention)
 {
-   plfr = new ParLinearForm(pfes, &data[0]);
-   plfi = new ParLinearForm(pfes, &data[pfes->GetVSize()]);
+   plfr = new ParLinearForm(pfes, data);
+   plfi = new ParLinearForm(pfes, (data) ? &data[pfes->GetVSize()]:data);
+
+   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();
+
+   int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
+   tdof_offsets = new HYPRE_Int[n+1];
+
+   for (int i=0; i<=n; i++)
+   {
+      tdof_offsets[i] = 2 * tdof_offsets_fes[i];
+   }
+}
+
+
+ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
+                                           ParLinearForm *plf_r, ParLinearForm *plf_i,
+                                           ComplexOperator::Convention
+                                           convention)
+   : Vector(2*(pfes->GetVSize())),
+     conv(convention)
+{
+   plfr = new ParLinearForm(pfes, plf_r);
+   plfr->SetData(data);
+   plfi = new ParLinearForm(pfes, plf_i);
+   plfi->SetData((data) ? &data[pfes->GetVSize()]:data);
 
    HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();
 
@@ -611,8 +733,8 @@ ParComplexLinearForm::Update(ParFiniteElementSpace *pf)
    int vsize = pfes->GetVSize();
    SetSize(2 * vsize);
 
-   Vector vplfr(&data[0], vsize);
-   Vector vplfi(&data[vsize], vsize);
+   Vector vplfr(data, vsize);
+   Vector vplfi((data) ? &data[vsize] : data, vsize);
 
    plfr->Update(pfes, vplfr, 0);
    plfi->Update(pfes, vplfi, 0);
@@ -636,7 +758,7 @@ ParComplexLinearForm::ParallelAssemble(Vector &tv)
 
    double * tvd = tv.GetData();
    Vector tvr(tvd, size);
-   Vector tvi(&tvd[size], size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);
 
    plfr->ParallelAssemble(tvr);
    plfi->ParallelAssemble(tvi);
@@ -655,7 +777,7 @@ ParComplexLinearForm::ParallelAssemble()
 
    double * tvd = tv->GetData();
    Vector tvr(tvd, size);
-   Vector tvi(&tvd[size], size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);
 
    plfr->ParallelAssemble(tvr);
    plfi->ParallelAssemble(tvi);
@@ -672,6 +794,21 @@ ParComplexLinearForm::operator()(const ParComplexGridFunction &gf) const
 }
 
 
+
+bool ParSesquilinearForm::RealInteg()
+{
+   int nint = pblfr->GetFBFI()->Size() + pblfr->GetDBFI()->Size() +
+              pblfr->GetBBFI()->Size() + pblfr->GetBFBFI()->Size();
+   return (nint != 0);
+}
+
+bool ParSesquilinearForm::ImagInteg()
+{
+   int nint = pblfi->GetFBFI()->Size() + pblfi->GetDBFI()->Size() +
+              pblfi->GetBBFI()->Size() + pblfi->GetBFBFI()->Size();
+   return (nint != 0);
+}
+
 ParSesquilinearForm::ParSesquilinearForm(ParFiniteElementSpace *pf,
                                          ComplexOperator::Convention
                                          convention)
@@ -680,6 +817,14 @@ ParSesquilinearForm::ParSesquilinearForm(ParFiniteElementSpace *pf,
      pblfi(new ParBilinearForm(pf))
 {}
 
+ParSesquilinearForm::ParSesquilinearForm(ParFiniteElementSpace *pf,
+                                         ParBilinearForm *pbfr, ParBilinearForm *pbfi,
+                                         ComplexOperator::Convention convention)
+   : conv(convention),
+     pblfr(new ParBilinearForm(pf,pbfr)),
+     pblfi(new ParBilinearForm(pf,pbfi))
+{}
+
 ParSesquilinearForm::~ParSesquilinearForm()
 {
    delete pblfr;
@@ -765,9 +910,8 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                       Vector &X, Vector &B,
                                       int ci)
 {
-   int vsize = x.Size() / 2;
-
-   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
+   ParFiniteElementSpace * pfes = pblfr->ParFESpace();
+   int vsize = pfes->GetVSize();
 
    // Allocate temporary vectors
    Vector b_0(vsize);  b_0 = 0.0;
@@ -779,61 +923,151 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
    MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
    Vector b_r(b.GetData(), vsize);
    Vector b_i(&(b.GetData())[vsize], vsize);
-   b_i *= s;
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
+
+   int tvsize = pfes->GetTrueVSize();
 
    OperatorHandle A_r, A_i;
-   Vector X_0, B_0;
 
-   b_0 = b_r;
-   pblfr->FormLinearSystem(ess_tdof_list, x_r, b_0, A_r, X_0, B_0, ci);
-
-   int tvsize = B_0.Size();
    X.SetSize(2 * tvsize);
    B.SetSize(2 * tvsize);
-   Vector X_r(X.GetData(), tvsize);
+
+   Vector X_0(tvsize), B_0(tvsize);
+   Vector X_r(X.GetData(),tvsize);
    Vector X_i(&(X.GetData())[tvsize], tvsize);
    Vector B_r(B.GetData(), tvsize);
    Vector B_i(&(B.GetData())[tvsize], tvsize);
-   X_r = X_0; B_r = B_0;
 
-   b_0 = 0.0;
-   pblfi->FormLinearSystem(ess_tdof_list, x_i, b_0, A_i, X_0, B_0, false);
-   B_r -= B_0;
+   if (RealInteg())
+   {
+      b_0 = b_r;
+      pblfr->FormLinearSystem(ess_tdof_list, x_r, b_0, A_r, X_0, B_0, ci);
+      X_r = X_0; B_r = B_0;
 
-   b_0 = b_i;
-   pblfr->FormLinearSystem(ess_tdof_list, x_i, b_0, A_r, X_0, B_0, ci);
-   X_i = X_0; B_i = B_0;
+      b_0 = b_i;
+      pblfr->FormLinearSystem(ess_tdof_list, x_i, b_0, A_r, X_0, B_0, ci);
+      X_i = X_0; B_i = B_0;
 
-   b_0 = 0.0;
-   pblfi->FormLinearSystem(ess_tdof_list, x_r, b_0, A_i, X_0, B_0, false);
-   B_i += B_0;
+      if (ImagInteg())
+      {
+         b_0 = 0.0;
+         pblfi->FormLinearSystem(ess_tdof_list, x_i, b_0, A_i, X_0, B_0, false);
+         B_r -= B_0;
 
-   B_i *= s;
-   b_i *= s;
+         b_0 = 0.0;
+         pblfi->FormLinearSystem(ess_tdof_list, x_r, b_0, A_i, X_0, B_0, false);
+         B_i += B_0;
+      }
+   }
+   else if (ImagInteg())
+   {
+      b_0 = b_i;
+      pblfi->FormLinearSystem(ess_tdof_list, x_r, b_0, A_i, X_0, B_0, ci);
+      X_r = X_0; B_i = B_0;
+
+      b_0 = b_r; b_0 *= -1.0;
+      pblfi->FormLinearSystem(ess_tdof_list, x_i, b_0, A_i, X_0, B_0, ci);
+      X_i = X_0; B_r = B_0; B_r *= -1.0;
+   }
+   else
+   {
+      MFEM_ABORT("Real and Imaginary part of the Sesquilinear form are empty");
+   }
 
    // Modify RHS and offdiagonal blocks (Imaginary parts of the matrix) to
    // conform with standard essential BC treatment i.e. zero out rows and
    // columns and place ones on the diagonal.
-   if ( A_i.Type() == Operator::Hypre_ParCSR )
+   if (RealInteg() && ImagInteg())
    {
-      int n = ess_tdof_list.Size();
-      int j;
-
-      HypreParMatrix * Ah;  A_i.Get(Ah);
-      hypre_ParCSRMatrix * Aih =
-         (hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(*Ah);
-      for (int k=0; k<n; k++)
+      if ( A_i.Type() == Operator::Hypre_ParCSR )
       {
-         j=ess_tdof_list[k];
-         Aih->diag->data[Aih->diag->i[j]] = 0.0;
-         B_r(j) = X_r(j);
-         B_i(j) = X_i(j);
+         HypreParMatrix * Ah;  A_i.Get(Ah);
+         int n = ess_tdof_list.Size();
+         hypre_ParCSRMatrix * Aih =
+            (hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(*Ah);
+         for (int k=0; k<n; k++)
+         {
+            int j=ess_tdof_list[k];
+            Aih->diag->data[Aih->diag->i[j]] = 0.0;
+            B_r(j) = X_r(j);
+            B_i(j) = X_i(j);
+         }
+      }
+   }
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      B_i *= -1.0;
+      b_i *= -1.0;
+   }
+   // A = A_r + i A_i
+   A.Clear();
+   if ( A_r.Type() == Operator::Hypre_ParCSR ||
+        A_i.Type() == Operator::Hypre_ParCSR )
+   {
+      ComplexHypreParMatrix * A_hyp =
+         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
+                                   A_i.As<HypreParMatrix>(),
+                                   A_r.OwnsOperator(),
+                                   A_i.OwnsOperator(),
+                                   conv);
+      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op =
+         new ComplexOperator(A_r.As<Operator>(),
+                             A_i.As<Operator>(),
+                             A_r.OwnsOperator(),
+                             A_i.OwnsOperator(),
+                             conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+}
+
+void
+ParSesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                      OperatorHandle &A)
+{
+   OperatorHandle A_r, A_i;
+   if (RealInteg())
+   {
+      pblfr->FormSystemMatrix(ess_tdof_list, A_r);
+   }
+   if (ImagInteg())
+   {
+      pblfi->FormSystemMatrix(ess_tdof_list, A_i);
+   }
+   if (!RealInteg() && !ImagInteg())
+   {
+      MFEM_ABORT("Both Real and Imaginary part of the Sesquilinear form are empty");
+   }
+
+   // Modify offdiagonal blocks (Imaginary parts of the matrix) to
+   // conform with standard essential BC treatment i.e. zero out rows and
+   // columns and place ones on the diagonal.
+   if (RealInteg() && ImagInteg())
+   {
+      if ( A_i.Type() == Operator::Hypre_ParCSR )
+      {
+         int n = ess_tdof_list.Size();
+         int j;
+
+         HypreParMatrix * Ah;  A_i.Get(Ah);
+         hypre_ParCSRMatrix * Aih =
+            (hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(*Ah);
+         for (int k=0; k<n; k++)
+         {
+            j=ess_tdof_list[k];
+            Aih->diag->data[Aih->diag->i[j]] = 0.0;
+         }
       }
    }
 
    // A = A_r + i A_i
    A.Clear();
-   if ( A_r.Type() == Operator::Hypre_ParCSR &&
+   if ( A_r.Type() == Operator::Hypre_ParCSR ||
         A_i.Type() == Operator::Hypre_ParCSR )
    {
       ComplexHypreParMatrix * A_hyp =

fem/complex_fem.hpp

@@ -99,6 +99,17 @@ public:
                      ComplexOperator::Convention
                      convention = ComplexOperator::HERMITIAN);
 
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
+       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .
+
+       The pointer @a fes is not owned by the newly constructed object.
+
+       The integrators are copied as pointers and they are not owned by the
+       newly constructed ComplexLinearForm. */
+   ComplexLinearForm(FiniteElementSpace *fes, LinearForm *lf_r, LinearForm *lf_i,
+                     ComplexOperator::Convention
+                     convention = ComplexOperator::HERMITIAN);
+
    virtual ~ComplexLinearForm();
 
    ComplexOperator::Convention GetConvention() const { return conv; }
@@ -155,6 +166,7 @@ public:
    std::complex<double> operator()(const ComplexGridFunction &gf) const;
 };
 
+
 /** Class for sesquilinear form
 
     A sesquilinear form is a generalization of a bilinear form to complex-valued
@@ -175,13 +187,33 @@ class SesquilinearForm
 private:
    ComplexOperator::Convention conv;
 
+   /** This data member allows one to specify what should be done to the
+       diagonal matrix entries and corresponding RHS values upon elimination of
+       the constrained DoFs. */
+   mfem::Matrix::DiagonalPolicy diag_policy = mfem::Matrix::DIAG_ONE;
+
    BilinearForm *blfr;
    BilinearForm *blfi;
 
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
+   bool RealInteg();
+   bool ImagInteg();
+
 public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
+       the same integrators as the BilinearForms @a bfr and @a bfi .
+
+       The pointer @a fes is not owned by the newly constructed object.
+
+       The integrators are copied as pointers and they are not owned by the
+       newly constructed SesquilinearForm. */
+   SesquilinearForm(FiniteElementSpace *fes, BilinearForm *bfr, BilinearForm *bfi,
+                    ComplexOperator::Convention
+                    convention = ComplexOperator::HERMITIAN);
 
    ComplexOperator::Convention GetConvention() const { return conv; }
    void SetConvention(const ComplexOperator::Convention &
@@ -241,6 +273,9 @@ public:
                          OperatorHandle &A, Vector &X, Vector &B,
                          int copy_interior = 0);
 
+   void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                         OperatorHandle &A);
+
    /** Call this method after solving a linear system constructed using the
        FormLinearSystem method to recover the solution as a ParGridFunction-size
        vector in x. Use the same arguments as in the FormLinearSystem call. */
@@ -248,6 +283,12 @@ public:
 
    virtual void Update(FiniteElementSpace *nfes = NULL);
 
+   /// Sets diagonal policy used upon construction of the linear system
+   void SetDiagonalPolicy(mfem::Matrix::DiagonalPolicy dpolicy);
+
+   /// Returns the diagonal policy of the sesquilinear form
+   Matrix::DiagonalPolicy GetDiagonalPolicy() const {return diag_policy;}
+
    virtual ~SesquilinearForm();
 };
 
@@ -359,6 +400,19 @@ public:
                         ComplexOperator::Convention
                         convention = ComplexOperator::HERMITIAN);
 
+   /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
+       using the same integrators as the LinearForms @a plfr (real) and @a plfi
+       (imag) .
+
+      The pointer @a fes is not owned by the newly constructed object.
+
+      The integrators are copied as pointers and they are not owned by the newly
+      constructed ParComplexLinearForm. */
+   ParComplexLinearForm(ParFiniteElementSpace *pf, ParLinearForm *plf_r,
+                        ParLinearForm *plf_i,
+                        ComplexOperator::Convention
+                        convention = ComplexOperator::HERMITIAN);
+
    virtual ~ParComplexLinearForm();
 
    ComplexOperator::Convention GetConvention() const { return conv; }
@@ -444,11 +498,28 @@ private:
    ParBilinearForm *pblfr;
    ParBilinearForm *pblfi;
 
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
+   bool RealInteg();
+   bool ImagInteg();
+
 public:
    ParSesquilinearForm(ParFiniteElementSpace *pf,
                        ComplexOperator::Convention
                        convention = ComplexOperator::HERMITIAN);
 
+   /** @brief Create a ParSesquilinearForm on the ParFiniteElementSpace @a pf,
+       using the same integrators as the ParBilinearForms @a pbfr and @a pbfi .
+
+       The pointer @a pf is not owned by the newly constructed object.
+
+       The integrators are copied as pointers and they are not owned by the
+       newly constructed ParSesquilinearForm. */
+   ParSesquilinearForm(ParFiniteElementSpace *pf, ParBilinearForm *pbfr,
+                       ParBilinearForm *pbfi,
+                       ComplexOperator::Convention
+                       convention = ComplexOperator::HERMITIAN);
+
    ComplexOperator::Convention GetConvention() const { return conv; }
    void SetConvention(const ComplexOperator::Convention &
                       convention) { conv = convention; }
@@ -513,6 +584,9 @@ public:
                          OperatorHandle &A, Vector &X, Vector &B,
                          int copy_interior = 0);
 
+   void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                         OperatorHandle &A);
+
    /** Call this method after solving a linear system constructed using the
        FormLinearSystem method to recover the solution as a ParGridFunction-size
        vector in x. Use the same arguments as in the FormLinearSystem call. */

fem/eltrans.cpp

@@ -413,6 +413,26 @@ const DenseMatrix &IsoparametricTransformation::EvalJacobian()
    return dFdx;
 }
 
+const DenseMatrix &IsoparametricTransformation::EvalHessian()
+{
+   MFEM_ASSERT(space_dim == PointMat.Height(),
+               "the IsoparametricTransformation has not been finalized;"
+               " call FinilizeTransformation() after setup");
+   MFEM_ASSERT((EvalState & HESSIAN_MASK) == 0, "");
+
+   int Dim = FElem->GetDim();
+   d2shape.SetSize(FElem->GetDof(), (Dim*(Dim+1))/2);
+   d2Fdx2.SetSize(PointMat.Height(), d2shape.Width());
+   if (d2shape.Width() > 0)
+   {
+      FElem->CalcHessian(*IntPoint, d2shape);
+      Mult(PointMat, d2shape, d2Fdx2);
+   }
+   EvalState |= HESSIAN_MASK;
+
+   return d2Fdx2;
+}
+
 int IsoparametricTransformation::OrderJ()
 {
    switch (FElem->Space())

fem/eltrans.hpp

@@ -25,6 +25,7 @@ class ElementTransformation
 protected:
    const IntegrationPoint *IntPoint;
    DenseMatrix dFdx, adjJ, invJ;
+   DenseMatrix d2Fdx2;
    double Wght;
    int EvalState;
    enum StateMasks
@@ -32,7 +33,8 @@ protected:
       JACOBIAN_MASK = 1,
       WEIGHT_MASK   = 2,
       ADJUGATE_MASK = 4,
-      INVERSE_MASK  = 8
+      INVERSE_MASK  = 8,
+      HESSIAN_MASK  = 16
    };
    Geometry::Type geom;
    int space_dim;
@@ -40,6 +42,7 @@ protected:
    // Evaluate the Jacobian of the transformation at the IntPoint and store it
    // in dFdx.
    virtual const DenseMatrix &EvalJacobian() = 0;
+   virtual const DenseMatrix &EvalHessian() = 0;
 
    double EvalWeight();
    const DenseMatrix &EvalAdjugateJ();
@@ -68,6 +71,9 @@ public:
    const DenseMatrix &Jacobian()
    { return (EvalState & JACOBIAN_MASK) ? dFdx : EvalJacobian(); }
 
+   const DenseMatrix &Hessian()
+   { return (EvalState & HESSIAN_MASK) ? d2Fdx2 : EvalHessian(); }
+
    double Weight() { return (EvalState & WEIGHT_MASK) ? Wght : EvalWeight(); }
 
    const DenseMatrix &AdjugateJacobian()
@@ -285,7 +291,7 @@ public:
 class IsoparametricTransformation : public ElementTransformation
 {
 private:
-   DenseMatrix dshape;
+   DenseMatrix dshape,d2shape;
    Vector shape;
 
    const FiniteElement *FElem;
@@ -294,7 +300,9 @@ private:
    // Evaluate the Jacobian of the transformation at the IntPoint and store it
    // in dFdx.
    virtual const DenseMatrix &EvalJacobian();
-
+   // Evaluate the Hessian of the transformation at the IntPoint and store it
+   // in d2Fdx2.
+   virtual const DenseMatrix &EvalHessian();
 public:
    void SetFE(const FiniteElement *FE) { FElem = FE; geom = FE->GetGeomType(); }
    const FiniteElement* GetFE() const { return FElem; }

fem/fe.cpp

@@ -203,6 +203,170 @@ void FiniteElement::CalcPhysDShape(ElementTransformation &Trans,
    Mult(vshape, Trans.InverseJacobian(), dshape);
 }
 
+void FiniteElement::CalcPhysLaplacian(ElementTransformation &Trans,
+                                      Vector &Laplacian) const
+{
+   MFEM_ASSERT(MapType == VALUE, "");
+
+   // Simpler routine if mapping is affine
+   if (Trans.Hessian().FNorm2() < 1e-20)
+   {
+      CalcPhysLinLaplacian(Trans, Laplacian);
+      return;
+   }
+
+   // Compute full Hessian first if non-affine
+   int size = (Dim*(Dim+1))/2;
+   DenseMatrix hess(Dof, size);
+   CalcPhysHessian(Trans,hess);
+
+   if (Dim == 3)
+   {
+      for (int nd = 0; nd < Dof; nd++)
+      {
+         Laplacian[nd] = hess(nd,0) + hess(nd,4) + hess(nd,5);
+      }
+   }
+   else if (Dim == 2)
+   {
+      for (int nd = 0; nd < Dof; nd++)
+      {
+         Laplacian[nd] = hess(nd,0) + hess(nd,2);
+      }
+   }
+   else
+   {
+      for (int nd = 0; nd < Dof; nd++)
+      {
+         Laplacian[nd] = hess(nd,0);
+      }
+   }
+}
+
+
+// Assume a linear mapping
+void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
+                                         Vector &Laplacian) const
+{
+   MFEM_ASSERT(MapType == VALUE, "");
+   int size = (Dim*(Dim+1))/2;
+   DenseMatrix hess(Dof, size);
+   DenseMatrix Gij(Dim,Dim);
+   Vector scale(size);
+
+   CalcHessian (Trans.GetIntPoint(), hess);
+   MultAAt(Trans.InverseJacobian(), Gij);
+
+   if (Dim == 3)
+   {
+      scale[0] =   Gij(0,0);
+      scale[1] = 2*Gij(0,1);
+      scale[2] = 2*Gij(0,2);
+
+      scale[3] = 2*Gij(1,2);
+      scale[4] =   Gij(2,2);
+
+      scale[5] =   Gij(1,1);
+   }
+   else if (Dim == 2)
+   {
+      scale[0] =   Gij(0,0);
+      scale[1] = 2*Gij(0,1);
+      scale[2] =   Gij(1,1);
+   }
+   else
+   {
+      scale[0] =   Gij(0,0);
+   }
+
+   for (int nd = 0; nd < Dof; nd++)
+   {
+      Laplacian[nd] = 0.0;
+      for (int ii = 0; ii < size; ii++)
+      {
+         Laplacian[nd] += hess(nd,ii)*scale[ii];
+      }
+   }
+
+}
+
+void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,
+                                     DenseMatrix& Hessian) const
+{
+   MFEM_ASSERT(MapType == VALUE, "");
+
+   // Roll 2-Tensors in vectors and 4-Tensor in Matrix, exploiting symmetry
+   Array<int> map(Dim*Dim);
+   if (Dim == 3)
+   {
+      map[0] = 0;
+      map[1] = 1;
+      map[2] = 2;
+
+      map[3] = 1;
+      map[4] = 5;
+      map[5] = 3;
+
+      map[6] = 2;
+      map[7] = 3;
+      map[8] = 4;
+   }
+   else if (Dim == 2)
+   {
+      map[0] = 0;
+      map[1] = 1;
+
+      map[2] = 1;
+      map[3] = 2;
+   }
+   else
+   {
+      map[0] = 0;
+   }
+
+   // Hessian in ref coords
+   int size = (Dim*(Dim+1))/2;
+   DenseMatrix hess(Dof, size);
+   CalcHessian(Trans.GetIntPoint(), hess);
+
+   // Gradient in physical coords
+   if (Trans.Hessian().FNorm2() > 1e-10)
+   {
+      DenseMatrix grad(Dof, Dim);
+      CalcPhysDShape(Trans, grad);
+      DenseMatrix gmap(Dof, size);
+      Mult(grad,Trans.Hessian(),gmap);
+      hess -= gmap;
+   }
+
+   // LHM
+   DenseMatrix lhm(size,size);
+   DenseMatrix invJ = Trans.Jacobian();
+   lhm = 0.0;
+   for (int i = 0; i < Dim; i++)
+   {
+      for (int j = 0; j < Dim; j++)
+      {
+         for (int k = 0; k < Dim; k++)
+         {
+            for (int l = 0; l < Dim; l++)
+            {
+               lhm(map[i*Dim+j],map[k*Dim+l]) += invJ(i,k)*invJ(j,l);
+            }
+         }
+      }
+   }
+   // Correct multiplicity
+   Vector mult(size);
+   mult = 0.0;
+   for (int i = 0; i < Dim*Dim; i++) { mult[map[i]]++; }
+   lhm.InvRightScaling(mult);
+
+   // Hessian in physical coords
+   lhm.Invert();
+   Mult( hess, lhm, Hessian);
+}
+
 const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &,
                                              DofToQuad::Mode) const
 {
@@ -1750,6 +1914,233 @@ void BiQuad2DFiniteElement::ProjectDelta(int vertex, Vector &dofs) const
 #endif
 }
 
+
+H1Ser_QuadrilateralElement::H1Ser_QuadrilateralElement(const int p)
+   : ScalarFiniteElement(2, Geometry::SQUARE, (p*p + 3*p +6) / 2, p,
+                         FunctionSpace::Qk)
+{
+   // Store the dof_map of the associated TensorBasisElement, which will be used
+   // to create the serendipity dof map.  Its size is larger than the size of
+   // the serendipity element.
+   TensorBasisElement tbeTemp =
+      TensorBasisElement(2, p, BasisType::GaussLobatto,
+                         TensorBasisElement::DofMapType::Sr_DOF_MAP);
+   const Array<int> tp_dof_map = tbeTemp.GetDofMap();
+
+   const double *cp = poly1d.ClosedPoints(p, BasisType::GaussLobatto);
+
+   // Fixing the Nodes is exactly the same as the H1_QuadrilateralElement
+   // constructor except we only use those values of the associated tensor
+   // product dof_map that are <= the number of serendipity Dofs e.g. only DoFs
+   // 0-7 out of the 9 tensor product dofs (at quadratic order)
+   int o = 0;
+
+   for (int j = 0; j <= p; j++)
+   {
+      for (int i = 0; i <= p; i++)
+      {
+         if (tp_dof_map[o] < Nodes.Size())
+         {
+            Nodes.IntPoint(tp_dof_map[o]).x = cp[i];
+            Nodes.IntPoint(tp_dof_map[o]).y = cp[j];
+         }
+         o++;
+      }
+   }
+}
+
+void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
+                                           Vector &shape) const
+{
+   int p = (this)->GetOrder();
+   double x = ip.x, y = ip.y;
+
+   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Vector nodalX(p+1);
+   Vector nodalY(p+1);
+
+   edgeNodalBasis.Eval(x, nodalX);
+   edgeNodalBasis.Eval(y, nodalY);
+
+   // First, fix edge-based shape functions. Use a nodal interpolant for edge
+   // points, weighted by the linear function that vanishes on opposite edge.
+   for (int i = 0; i < p-1; i++)
+   {
+      shape(4 + 0*(p-1) + i) = (nodalX(i+1))*(1.-y);         // south edge 0->1
+      shape(4 + 1*(p-1) + i) = (nodalY(i+1))*x;              // east edge  1->2
+      shape(4 + 3*(p-1) - i - 1) = (nodalX(i+1)) * y;        // north edge 3->2
+      shape(4 + 4*(p-1) - i - 1) = (nodalY(i+1)) * (1. - x); // west edge  0->3
+   }
+
+   BiLinear2DFiniteElement bilinear = BiLinear2DFiniteElement();
+   Vector bilinearsAtIP(4);
+   bilinear.CalcShape(ip, bilinearsAtIP);
+
+   const double *edgePts(poly1d.ClosedPoints(p, BasisType::GaussLobatto));
+
+   // Next, set the shape function associated with vertex V, evaluated at (x,y)
+   // to be: bilinear function associated to V, evaluated at (x,y) - sum (shape
+   // function at edge point P, weighted by bilinear function for V evaluated at
+   // P) where the sum is taken only for points P on edges incident to V.
+
+   double vtx0fix =0;
+   double vtx1fix =0;
+   double vtx2fix =0;
+   double vtx3fix =0;
+   for (int i = 0; i<p-1; i++)
+   {
+      vtx0fix += (1-edgePts[i+1])*(shape(4 + i) +
+                                   shape(4 + 4*(p-1) - i - 1)); // bot+left edge
+      vtx1fix += (1-edgePts[i+1])*(shape(4 + 1*(p-1) + i) +
+                                   shape(4 + (p-2)-i));        // right+bot edge
+      vtx2fix += (1-edgePts[i+1])*(shape(4 + 2*(p-1) + i) +
+                                   shape(1 + 2*p-i));          // top+right edge
+      vtx3fix += (1-edgePts[i+1])*(shape(4 + 3*(p-1) + i) +
+                                   shape(3*p - i));            // left+top edge
+   }
+   shape(0) = bilinearsAtIP(0) - vtx0fix;
+   shape(1) = bilinearsAtIP(1) - vtx1fix;
+   shape(2) = bilinearsAtIP(2) - vtx2fix;
+   shape(3) = bilinearsAtIP(3) - vtx3fix;
+
+   // Interior basis functions appear starting at order p=4. These are non-nodal
+   // bubble functions.
+   if (p > 3)
+   {
+      double *legX = new double[p-1];
+      double *legY = new double[p-1];
+      Poly_1D *storeLegendre = new Poly_1D();
+
+      storeLegendre->CalcLegendre(p-2, x, legX);
+      storeLegendre->CalcLegendre(p-2, y, legY);
+
+      int interior_total = 0;
+      for (int j = 4; j < p + 1; j++)
+      {
+         for (int k = 0; k < j-3; k++)
+         {
+            shape(4 + 4*(p-1) + interior_total)
+               = legX[k] * legY[j-4-k] * x * (1. - x) * y * (1. - y);
+            interior_total++;
+         }
+      }
+
+      delete[] legX;
+      delete[] legY;
+      delete storeLegendre;
+   }
+}
+
+void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
+                                            DenseMatrix &dshape) const
+{
+   int p = (this)->GetOrder();
+   double x = ip.x, y = ip.y;
+
+   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Vector nodalX(p+1);
+   Vector DnodalX(p+1);
+   Vector nodalY(p+1);
+   Vector DnodalY(p+1);
+
+   edgeNodalBasis.Eval(x, nodalX, DnodalX);
+   edgeNodalBasis.Eval(y, nodalY, DnodalY);
+
+   for (int i = 0; i < p-1; i++)
+   {
+      dshape(4 + 0*(p-1) + i,0) =  DnodalX(i+1) * (1.-y);
+      dshape(4 + 0*(p-1) + i,1) = -nodalX(i+1);
+      dshape(4 + 1*(p-1) + i,0) =  nodalY(i+1);
+      dshape(4 + 1*(p-1) + i,1) =  DnodalY(i+1)*x;
+      dshape(4 + 3*(p-1) - i - 1,0) =  DnodalX(i+1)*y;
+      dshape(4 + 3*(p-1) - i - 1,1) =  nodalX(i+1);
+      dshape(4 + 4*(p-1) - i - 1,0) = -nodalY(i+1);
+      dshape(4 + 4*(p-1) - i - 1,1) =  DnodalY(i+1) * (1.-x);
+   }
+
+   BiLinear2DFiniteElement bilinear = BiLinear2DFiniteElement();
+   DenseMatrix DbilinearsAtIP(4);
+   bilinear.CalcDShape(ip, DbilinearsAtIP);
+
+   const double *edgePts(poly1d.ClosedPoints(p, BasisType::GaussLobatto));
+
+   dshape(0,0) = DbilinearsAtIP(0,0);
+   dshape(0,1) = DbilinearsAtIP(0,1);
+   dshape(1,0) = DbilinearsAtIP(1,0);
+   dshape(1,1) = DbilinearsAtIP(1,1);
+   dshape(2,0) = DbilinearsAtIP(2,0);
+   dshape(2,1) = DbilinearsAtIP(2,1);
+   dshape(3,0) = DbilinearsAtIP(3,0);
+   dshape(3,1) = DbilinearsAtIP(3,1);
+
+   for (int i = 0; i<p-1; i++)
+   {
+      dshape(0,0) -= (1-edgePts[i+1])*(dshape(4 + 0*(p-1) + i, 0) +
+                                       dshape(4 + 4*(p-1) - i - 1,0));
+      dshape(0,1) -= (1-edgePts[i+1])*(dshape(4 + 0*(p-1) + i, 1) +
+                                       dshape(4 + 4*(p-1) - i - 1,1));
+      dshape(1,0) -= (1-edgePts[i+1])*(dshape(4 + 1*(p-1) + i, 0) +
+                                       dshape(4 + (p-2)-i, 0));
+      dshape(1,1) -= (1-edgePts[i+1])*(dshape(4 + 1*(p-1) + i, 1) +
+                                       dshape(4 + (p-2)-i, 1));
+      dshape(2,0) -= (1-edgePts[i+1])*(dshape(4 + 2*(p-1) + i, 0) +
+                                       dshape(1 + 2*p-i, 0));
+      dshape(2,1) -= (1-edgePts[i+1])*(dshape(4 + 2*(p-1) + i, 1) +
+                                       dshape(1 + 2*p-i, 1));
+      dshape(3,0) -= (1-edgePts[i+1])*(dshape(4 + 3*(p-1) + i, 0) +
+                                       dshape(3*p - i, 0));
+      dshape(3,1) -= (1-edgePts[i+1])*(dshape(4 + 3*(p-1) + i, 1) +
+                                       dshape(3*p - i, 1));
+   }
+
+   if (p > 3)
+   {
+      double *legX = new double[p-1];
+      double *legY = new double[p-1];
+      double *DlegX = new double[p-1];
+      double *DlegY = new double[p-1];
+      Poly_1D *storeLegendre = new Poly_1D();
+
+      storeLegendre->CalcLegendre(p-2, x, legX, DlegX);
+      storeLegendre->CalcLegendre(p-2, y, legY, DlegY);
+
+      int interior_total = 0;
+      for (int j = 4; j < p + 1; j++)
+      {
+         for (int k = 0; k < j-3; k++)
+         {
+            dshape(4 + 4*(p-1) + interior_total, 0) =
+               legY[j-4-k]*y*(1-y) * (DlegX[k]*x*(1-x) + legX[k]*(1-2*x));
+            dshape(4 + 4*(p-1) + interior_total, 1) =
+               legX[k]*x*(1-x) * (DlegY[j-4-k]*y*(1-y) + legY[j-4-k]*(1-2*y));
+            interior_total++;
+         }
+      }
+      delete[] legX;
+      delete[] legY;
+      delete[] DlegX;
+      delete[] DlegY;
+      delete storeLegendre;
+   }
+}
+
+void H1Ser_QuadrilateralElement::GetLocalInterpolation(ElementTransformation
+                                                       &Trans,
+                                                       DenseMatrix &I) const
+{
+   // For p<=4, the basis is nodal; for p>4, the quad-interior functions are
+   // non-nodal.
+   if (Order <= 4)
+   {
+      NodalLocalInterpolation(Trans, I, *this);
+   }
+   else
+   {
+      ScalarLocalInterpolation(Trans, I, *this);
+   }
+}
+
+
 BiQuadPos2DFiniteElement::BiQuadPos2DFiniteElement()
    : PositiveFiniteElement(2, Geometry::SQUARE, 9, 2, FunctionSpace::Qk)
 {
@@ -6965,7 +7356,7 @@ TensorBasisElement::TensorBasisElement(const int dims, const int p,
    : b_type(btype),
      basis1d(poly1d.GetBasis(p, b_type))
 {
-   if (dmtype == H1_DOF_MAP)
+   if (dmtype == H1_DOF_MAP || dmtype == Sr_DOF_MAP)
    {
       switch (dims)
       {
@@ -11792,6 +12183,30 @@ void NURBS1DFiniteElement::CalcDShape(const IntegrationPoint &ip,
    add(sum, grad, -dsum*sum*sum, shape_x, grad);
 }
 
+void NURBS1DFiniteElement::CalcHessian (const IntegrationPoint &ip,
+                                        DenseMatrix &hessian) const
+{
+   Vector grad(Dof);
+   Vector hess(hessian.Data(), Dof);
+
+   kv[0]->CalcShape (shape_x,  ijk[0], ip.x);
+   kv[0]->CalcDShape(grad,     ijk[0], ip.x);
+   kv[0]->CalcD2Shape(hess,    ijk[0], ip.x);
+
+   double sum = 0.0, dsum = 0.0, d2sum = 0.0;
+   for (int i = 0; i <= Order; i++)
+   {
+      sum   += (shape_x(i) *= weights(i));
+      dsum  += (   grad(i) *= weights(i));
+      d2sum += (   hess(i) *= weights(i));
+   }
+
+   sum = 1.0/sum;
+   add(sum, hess, -2*dsum*sum*sum, grad, hess);
+   add(1.0, hess, (-d2sum + 2*dsum*dsum*sum)*sum*sum, shape_x, hess);
+}
+
+
 void NURBS2DFiniteElement::SetOrder() const
 {
    Orders[0] = kv[0]->GetOrder();
@@ -11800,10 +12215,13 @@ void NURBS2DFiniteElement::SetOrder() const
    shape_y.SetSize(Orders[1]+1);
    dshape_x.SetSize(Orders[0]+1);
    dshape_y.SetSize(Orders[1]+1);
+   d2shape_x.SetSize(Orders[0]+1);
+   d2shape_y.SetSize(Orders[1]+1);
 
    Order = max(Orders[0], Orders[1]);
    Dof = (Orders[0] + 1)*(Orders[1] + 1);
    u.SetSize(Dof);
+   du.SetSize(Dof);
    weights.SetSize(Dof);
 }
 
@@ -11861,7 +12279,65 @@ void NURBS2DFiniteElement::CalcDShape(const IntegrationPoint &ip,
    }
 }
 
-//---------------------------------------------------------------------
+void NURBS2DFiniteElement::CalcHessian (const IntegrationPoint &ip,
+                                        DenseMatrix &hessian) const
+{
+   double sum, dsum[2], d2sum[3];
+
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+
+   kv[0]->CalcDShape(dshape_x, ijk[0], ip.x);
+   kv[1]->CalcDShape(dshape_y, ijk[1], ip.y);
+
+   kv[0]->CalcD2Shape(d2shape_x, ijk[0], ip.x);
+   kv[1]->CalcD2Shape(d2shape_y, ijk[1], ip.y);
+
+   sum = dsum[0] = dsum[1] = 0.0;
+   d2sum[0] = d2sum[1] = d2sum[2] = 0.0;
+   for (int o = 0, j = 0; j <= Orders[1]; j++)
+   {
+      const double sy = shape_y(j), dsy = dshape_y(j), d2sy = d2shape_y(j);
+      for (int i = 0; i <= Orders[0]; i++, o++)
+      {
+         const double sx = shape_x(i), dsx = dshape_x(i), d2sx = d2shape_x(i);
+         sum += ( u(o) = sx*sy*weights(o) );
+
+         dsum[0] += ( du(o,0) = dsx*sy*weights(o) );
+         dsum[1] += ( du(o,1) = sx*dsy*weights(o) );
+
+         d2sum[0] += ( hessian(o,0) = d2sx*sy*weights(o) );
+         d2sum[1] += ( hessian(o,1) = dsx*dsy*weights(o) );
+         d2sum[2] += ( hessian(o,2) = sx*d2sy*weights(o) );
+      }
+   }
+
+   sum = 1.0/sum;
+   dsum[0] *= sum;
+   dsum[1] *= sum;
+
+   d2sum[0] *= sum;
+   d2sum[1] *= sum;
+   d2sum[2] *= sum;
+
+   for (int o = 0; o < Dof; o++)
+   {
+      hessian(o,0) = hessian(o,0)*sum
+                     - 2*du(o,0)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[0] - d2sum[0]);
+
+      hessian(o,1) = hessian(o,1)*sum
+                     - du(o,0)*sum*dsum[1]
+                     - du(o,1)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[1] - d2sum[1]);
+
+      hessian(o,2) = hessian(o,2)*sum
+                     - 2*du(o,1)*sum*dsum[1]
+                     + u[o]*sum*(2*dsum[1]*dsum[1] - d2sum[2]);
+   }
+}
+
+
 void NURBS3DFiniteElement::SetOrder() const
 {
    Orders[0] = kv[0]->GetOrder();
@@ -11875,9 +12351,14 @@ void NURBS3DFiniteElement::SetOrder() const
    dshape_y.SetSize(Orders[1]+1);
    dshape_z.SetSize(Orders[2]+1);
 
+   d2shape_x.SetSize(Orders[0]+1);
+   d2shape_y.SetSize(Orders[1]+1);
+   d2shape_z.SetSize(Orders[2]+1);
+
    Order = max(max(Orders[0], Orders[1]), Orders[2]);
    Dof = (Orders[0] + 1)*(Orders[1] + 1)*(Orders[2] + 1);
    u.SetSize(Dof);
+   du.SetSize(Dof);
    weights.SetSize(Dof);
 }
 
@@ -11951,10 +12432,100 @@ void NURBS3DFiniteElement::CalcDShape(const IntegrationPoint &ip,
    }
 }
 
+void NURBS3DFiniteElement::CalcHessian (const IntegrationPoint &ip,
+                                        DenseMatrix &hessian) const
+{
+   double sum, dsum[3], d2sum[6];
+
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+   kv[2]->CalcShape ( shape_z, ijk[2], ip.z);
+
+   kv[0]->CalcDShape(dshape_x, ijk[0], ip.x);
+   kv[1]->CalcDShape(dshape_y, ijk[1], ip.y);
+   kv[2]->CalcDShape(dshape_z, ijk[2], ip.z);
+
+   kv[0]->CalcD2Shape(d2shape_x, ijk[0], ip.x);
+   kv[1]->CalcD2Shape(d2shape_y, ijk[1], ip.y);
+   kv[2]->CalcD2Shape(d2shape_z, ijk[2], ip.z);
+
+   sum = dsum[0] = dsum[1] = dsum[2] = 0.0;
+   d2sum[0] = d2sum[1] = d2sum[2] = d2sum[3] = d2sum[4] = d2sum[5] = 0.0;
+
+   for (int o = 0, k = 0; k <= Orders[2]; k++)
+   {
+      const double sz = shape_z(k), dsz = dshape_z(k), d2sz = d2shape_z(k);
+      for (int j = 0; j <= Orders[1]; j++)
+      {
+         const double sy = shape_y(j), dsy = dshape_y(j), d2sy = d2shape_y(j);
+         for (int i = 0; i <= Orders[0]; i++, o++)
+         {
+            const double sx = shape_x(i), dsx = dshape_x(i), d2sx = d2shape_x(i);
+            sum += ( u(o) = sx*sy*sz*weights(o) );
+
+            dsum[0] += ( du(o,0) = dsx*sy*sz*weights(o) );
+            dsum[1] += ( du(o,1) = sx*dsy*sz*weights(o) );
+            dsum[2] += ( du(o,2) = sx*sy*dsz*weights(o) );
+
+            d2sum[0] += ( hessian(o,0) = d2sx*sy*sz*weights(o) );
+            d2sum[1] += ( hessian(o,1) = dsx*dsy*sz*weights(o) );
+            d2sum[2] += ( hessian(o,2) = dsx*sy*dsz*weights(o) );
+
+            d2sum[3] += ( hessian(o,3) = sx*dsy*dsz*weights(o) );
+
+            d2sum[4] += ( hessian(o,4) = sx*sy*d2sz*weights(o) );
+            d2sum[5] += ( hessian(o,5) = sx*d2sy*sz*weights(o) );
+         }
+      }
+   }
+
+   sum = 1.0/sum;
+   dsum[0] *= sum;
+   dsum[1] *= sum;
+   dsum[2] *= sum;
+
+   d2sum[0] *= sum;
+   d2sum[1] *= sum;
+   d2sum[2] *= sum;
+
+   d2sum[3] *= sum;
+   d2sum[4] *= sum;
+   d2sum[5] *= sum;
+
+   for (int o = 0; o < Dof; o++)
+   {
+      hessian(o,0) = hessian(o,0)*sum
+                     - 2*du(o,0)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[0] - d2sum[0]);
+
+      hessian(o,1) = hessian(o,1)*sum
+                     - du(o,0)*sum*dsum[1]
+                     - du(o,1)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[1] - d2sum[1]);
+
+      hessian(o,2) = hessian(o,2)*sum
+                     - du(o,0)*sum*dsum[2]
+                     - du(o,2)*sum*dsum[0]
+                     + u[o]*sum*(2*dsum[0]*dsum[2] - d2sum[2]);
+
+      hessian(o,3) = hessian(o,3)*sum
+                     - du(o,1)*sum*dsum[2]
+                     - du(o,2)*sum*dsum[1]
+                     + u[o]*sum*(2*dsum[1]*dsum[2] - d2sum[3]);
+
+      hessian(o,4) = hessian(o,4)*sum
+                     - 2*du(o,2)*sum*dsum[2]
+                     + u[o]*sum*(2*dsum[2]*dsum[2] - d2sum[4]);
+
+      hessian(o,5) = hessian(o,5)*sum
+                     - 2*du(o,1)*sum*dsum[1]
+                     + u[o]*sum*(2*dsum[1]*dsum[1] - d2sum[5]);
+
+   }
+}
 
 // Global object definitions
 
-
 // Object declared in mesh/triangle.hpp.
 // Defined here to ensure it is constructed before 'Geometries'.
 Linear2DFiniteElement TriangleFE;

fem/fe.hpp

@@ -36,7 +36,8 @@ public:
       OpenUniform     = 3,  ///< Nodes: x_i = (i+1)/(n+1), i=0,...,n-1
       ClosedUniform   = 4,  ///< Nodes: x_i = i/(n-1),     i=0,...,n-1
       OpenHalfUniform = 5,  ///< Nodes: x_i = (i+1/2)/n,   i=0,...,n-1
-      NumBasisTypes   = 6   /**< Keep track of maximum types to prevent
+      Serendipity     = 6,  ///< Serendipity basis (squares / cubes)
+      NumBasisTypes   = 7   /**< Keep track of maximum types to prevent
                                  hard-coding */
    };
    /** @brief If the input does not represents a valid BasisType, abort with an
@@ -67,6 +68,7 @@ public:
          case OpenUniform:     return Quadrature1D::OpenUniform;
          case ClosedUniform:   return Quadrature1D::ClosedUniform;
          case OpenHalfUniform: return Quadrature1D::OpenHalfUniform;
+         case Serendipity:     return Quadrature1D::GaussLobatto;
       }
       return Quadrature1D::Invalid;
    }
@@ -110,6 +112,7 @@ public:
          case 'u': return OpenUniform;
          case 'U': return ClosedUniform;
          case 'o': return OpenHalfUniform;
+         case 's': return GaussLobatto;
       }
       MFEM_ABORT("unknown BasisType identifier");
       return -1;
@@ -419,10 +422,29 @@ public:
 
    virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
 
-   /** each row of h contains the upper triangular part of the hessian
-       of one shape function; the order in 2D is {u_xx, u_xy, u_yy} */
+   /** @brief Evaluate the Hessians of all shape functions of a scalar finite
+       element in reference space at the given point @a ip. */
+   /** Each row of the result DenseMatrix @a Hessian contains upper triangular
+       part of the Hessian of one shape function.
+       The order in 2D is {u_xx, u_xy, u_yy}.
+       The size (#Dof x (#Dim (#Dim-1)/2) of @a Hessian must be set in advance.*/
    virtual void CalcHessian (const IntegrationPoint &ip,
-                             DenseMatrix &h) const;
+                             DenseMatrix &Hessian) const;
+
+   /** @brief Evaluate the Hessian of all shape functions of a scalar finite
+       element in reference space at the given point @a ip. */
+   /** The size (#Dof, #Dim*(#Dim+1)/2) of @a Hessian must be set in advance. */
+   virtual void CalcPhysHessian(ElementTransformation &Trans,
+                                DenseMatrix& Hessian) const;
+
+   /** @brief Evaluate the Laplacian of all shape functions of a scalar finite
+       element in reference space at the given point @a ip. */
+   /** The size (#Dof) of @a Laplacian must be set in advance. */
+   virtual void CalcPhysLaplacian(ElementTransformation &Trans,
+                                  Vector& Laplacian) const;
+
+   virtual void CalcPhysLinLaplacian(ElementTransformation &Trans,
+                                     Vector& Laplacian) const;
 
    /** @brief Return the local interpolation matrix @a I (Dof x Dof) where the
        fine element is the image of the base geometry under the given
@@ -1701,9 +1723,6 @@ private:
    static void CalcMono(const int p, const double x, double *u);
    static void CalcMono(const int p, const double x, double *u, double *d);
 
-   static void CalcLegendre(const int p, const double x, double *u);
-   static void CalcLegendre(const int p, const double x, double *u, double *d);
-
    static void CalcChebyshev(const int p, const double x, double *u);
    static void CalcChebyshev(const int p, const double x, double *u, double *d);
    static void CalcChebyshev(const int p, const double x, double *u, double *d,
@@ -1792,6 +1811,9 @@ public:
    static void CalcBernstein(const int p, const double x, double *u, double *d)
    { CalcBinomTerms(p, x, 1. - x, u, d); }
 
+   static void CalcLegendre(const int p, const double x, double *u);
+   static void CalcLegendre(const int p, const double x, double *u, double *d);
+
    ~Poly_1D();
 };
 
@@ -1803,12 +1825,14 @@ protected:
    int b_type;
    Array<int> dof_map;
    Poly_1D::Basis &basis1d;
+   Array<int> inv_dof_map;
 
 public:
    enum DofMapType
    {
       L2_DOF_MAP = 0,
-      H1_DOF_MAP = 1
+      H1_DOF_MAP = 1,
+      Sr_DOF_MAP = 2,  // Sr = Serendipity
    };
 
    TensorBasisElement(const int dims, const int p, const int btype,
@@ -1968,6 +1992,18 @@ public:
 };
 
 
+class H1Ser_QuadrilateralElement : public ScalarFiniteElement
+{
+public:
+   H1Ser_QuadrilateralElement(const int p);
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const;
+   using FiniteElement::Project;
+};
+
 class H1Pos_HexahedronElement : public PositiveTensorFiniteElement
 {
 private:
@@ -2888,57 +2924,70 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian (const IntegrationPoint &ip,
+                             DenseMatrix &hessian) const;
 };
 
 class NURBS2DFiniteElement : public NURBSFiniteElement
 {
 protected:
-   mutable Vector u, shape_x, shape_y, dshape_x, dshape_y;
+   mutable Vector u, shape_x, shape_y, dshape_x, dshape_y, d2shape_x, d2shape_y;
+   mutable DenseMatrix du;
 
 public:
    NURBS2DFiniteElement(int p)
       : NURBSFiniteElement(2, Geometry::SQUARE, (p + 1)*(p + 1), p,
                            FunctionSpace::Qk),
-        u(Dof), shape_x(p + 1), shape_y(p + 1), dshape_x(p + 1), dshape_y(p + 1)
+        u(Dof), shape_x(p + 1), shape_y(p + 1), dshape_x(p + 1),
+        dshape_y(p + 1), d2shape_x(p + 1), d2shape_y(p + 1), du(Dof,2)
    { Orders[0] = Orders[1] = p; }
 
    NURBS2DFiniteElement(int px, int py)
       : NURBSFiniteElement(2, Geometry::SQUARE, (px + 1)*(py + 1),
                            std::max(px, py), FunctionSpace::Qk),
         u(Dof), shape_x(px + 1), shape_y(py + 1), dshape_x(px + 1),
-        dshape_y(py + 1)
+        dshape_y(py + 1), d2shape_x(px + 1), d2shape_y(py + 1), du(Dof,2)
    { Orders[0] = px; Orders[1] = py; }
 
    virtual void SetOrder() const;
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian (const IntegrationPoint &ip,
+                             DenseMatrix &hessian) const;
 };
 
 class NURBS3DFiniteElement : public NURBSFiniteElement
 {
 protected:
-   mutable Vector u, shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z;
+   mutable Vector u, shape_x, shape_y, shape_z;
+   mutable Vector dshape_x, dshape_y, dshape_z;
+   mutable Vector d2shape_x, d2shape_y, d2shape_z;
+   mutable DenseMatrix du;
 
 public:
    NURBS3DFiniteElement(int p)
       : NURBSFiniteElement(3, Geometry::CUBE, (p + 1)*(p + 1)*(p + 1), p,
                            FunctionSpace::Qk),
         u(Dof), shape_x(p + 1), shape_y(p + 1), shape_z(p + 1),
-        dshape_x(p + 1), dshape_y(p + 1), dshape_z(p + 1)
+        dshape_x(p + 1), dshape_y(p + 1), dshape_z(p + 1),
+        d2shape_x(p + 1), d2shape_y(p + 1), d2shape_z(p + 1), du(Dof,3)
    { Orders[0] = Orders[1] = Orders[2] = p; }
 
    NURBS3DFiniteElement(int px, int py, int pz)
       : NURBSFiniteElement(3, Geometry::CUBE, (px + 1)*(py + 1)*(pz + 1),
                            std::max(std::max(px,py),pz), FunctionSpace::Qk),
         u(Dof), shape_x(px + 1), shape_y(py + 1), shape_z(pz + 1),
-        dshape_x(px + 1), dshape_y(py + 1), dshape_z(pz + 1)
+        dshape_x(px + 1), dshape_y(py + 1), dshape_z(pz + 1),
+        d2shape_x(px + 1), d2shape_y(py + 1), d2shape_z(pz + 1), du(Dof,3)
    { Orders[0] = px; Orders[1] = py; Orders[2] = pz; }
 
    virtual void SetOrder() const;
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
+   virtual void CalcHessian (const IntegrationPoint &ip,
+                             DenseMatrix &hessian) const;
 };
 
 } // namespace mfem

fem/fe_coll.cpp

@@ -162,6 +162,10 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    {
       fec = new H1Pos_FECollection(atoi(name + 10), atoi(name + 6));
    }
+   else if (!strncmp(name, "H1Ser_", 6))
+   {
+      fec = new H1Ser_FECollection(atoi(name + 10), atoi(name + 6));
+   }
    else if (!strncmp(name, "H1@", 3))
    {
       fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
@@ -1520,6 +1524,11 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          snprintf(h1_name, 32, "H1Pos_%dD_P%d", dim, p);
          break;
       }
+      case BasisType::Serendipity:
+      {
+         snprintf(h1_name, 32, "H1Ser_%dD_P%d", dim, p);
+         break;
+      }
       default:
       {
          MFEM_VERIFY(Quadrature1D::CheckClosed(pt_type) !=
@@ -1582,6 +1591,18 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          H1_Elements[Geometry::TRIANGLE] = new H1Pos_TriangleElement(p);
          H1_Elements[Geometry::SQUARE] = new H1Pos_QuadrilateralElement(p);
       }
+      else if (b_type == BasisType::Serendipity)
+      {
+         // Note: in fe_coll.hpp the DofForGeometry(Geometry::Type) method
+         // returns H1_dof[GeomType], so we need to fix the value of H1_dof here
+         // for the serendipity case.
+
+         // formula for number of interior serendipity DoFs (when p>1)
+         H1_dof[Geometry::SQUARE] = (pm3*pm2)/2;
+         H1_Elements[Geometry::SQUARE] = new H1Ser_QuadrilateralElement(p);
+         // allows for mixed tri/quad meshes
+         H1_Elements[Geometry::TRIANGLE] = new H1Pos_TriangleElement(p);
+      }
       else
       {
          H1_Elements[Geometry::TRIANGLE] = new H1_TriangleElement(p, btype);
@@ -1616,20 +1637,60 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
       {
          QuadDofOrd[i] = QuadDofOrd[i-1] + QuadDof;
       }
-      // see Mesh::GetQuadOrientation in mesh/mesh.cpp
-      for (int j = 0; j < pm1; j++)
+
+      // For serendipity order >=4, the QuadDofOrd array must be re-defined. We
+      // do this by computing the corresponding tensor product QuadDofOrd array
+      // or two orders less, which contains enough DoFs for their serendipity
+      // basis. This could be optimized.
+      if (b_type == BasisType::Serendipity)
       {
-         for (int i = 0; i < pm1; i++)
+         if (p < 4)
          {
-            int o = i + j*pm1;
-            QuadDofOrd[0][o] = i + j*pm1;  // (0,1,2,3)
-            QuadDofOrd[1][o] = j + i*pm1;  // (0,3,2,1)
-            QuadDofOrd[2][o] = j + (pm2 - i)*pm1;  // (1,2,3,0)
-            QuadDofOrd[3][o] = (pm2 - i) + j*pm1;  // (1,0,3,2)
-            QuadDofOrd[4][o] = (pm2 - i) + (pm2 - j)*pm1;  // (2,3,0,1)
-            QuadDofOrd[5][o] = (pm2 - j) + (pm2 - i)*pm1;  // (2,1,0,3)
-            QuadDofOrd[6][o] = (pm2 - j) + i*pm1;  // (3,0,1,2)
-            QuadDofOrd[7][o] = i + (pm2 - j)*pm1;  // (3,2,1,0)
+            // no face dofs --> don't need to adjust QuadDofOrd
+         }
+         else  // p >= 4 --> have face dofs
+         {
+            // Exactly the same as tensor product case, but with all orders
+            // reduced by 2 e.g. in case p=5 it builds a 2x2 array, even though
+            // there are only 3 serendipity dofs.
+            // In the tensor product case, the i and j index tensor directions,
+            // and o index from 0 to (pm1)^2,
+            const int pm4 = pm3 -1;
+
+            for (int j = 0; j < pm3; j++)   // pm3 instead of pm1, etc
+            {
+               for (int i = 0; i < pm3; i++)
+               {
+                  int o = i + j*pm3;
+                  QuadDofOrd[0][o] = i + j*pm3;  // (0,1,2,3)
+                  QuadDofOrd[1][o] = j + i*pm3;  // (0,3,2,1)
+                  QuadDofOrd[2][o] = j + (pm4 - i)*pm3;  // (1,2,3,0)
+                  QuadDofOrd[3][o] = (pm4 - i) + j*pm3;  // (1,0,3,2)
+                  QuadDofOrd[4][o] = (pm4 - i) + (pm4 - j)*pm3;  // (2,3,0,1)
+                  QuadDofOrd[5][o] = (pm4 - j) + (pm4 - i)*pm3;  // (2,1,0,3)
+                  QuadDofOrd[6][o] = (pm4 - j) + i*pm3;  // (3,0,1,2)
+                  QuadDofOrd[7][o] = i + (pm4 - j)*pm3;  // (3,2,1,0)
+               }
+            }
+
+         }
+      }
+      else // not serendipity
+      {
+         for (int j = 0; j < pm1; j++)
+         {
+            for (int i = 0; i < pm1; i++)
+            {
+               int o = i + j*pm1;
+               QuadDofOrd[0][o] = i + j*pm1;  // (0,1,2,3)
+               QuadDofOrd[1][o] = j + i*pm1;  // (0,3,2,1)
+               QuadDofOrd[2][o] = j + (pm2 - i)*pm1;  // (1,2,3,0)
+               QuadDofOrd[3][o] = (pm2 - i) + j*pm1;  // (1,0,3,2)
+               QuadDofOrd[4][o] = (pm2 - i) + (pm2 - j)*pm1;  // (2,3,0,1)
+               QuadDofOrd[5][o] = (pm2 - j) + (pm2 - i)*pm1;  // (2,1,0,3)
+               QuadDofOrd[6][o] = (pm2 - j) + i*pm1;  // (3,0,1,2)
+               QuadDofOrd[7][o] = i + (pm2 - j)*pm1;  // (3,2,1,0)
+            }
          }
       }
 

fem/fe_coll.hpp

@@ -120,6 +120,15 @@ public:
       : H1_FECollection(p, dim, BasisType::Positive) { }
 };
 
+/** Arbitrary order H1-conforming (continuous) serendipity finite elements;
+    Current implementation works in 2D only; 3D version is in development. */
+class H1Ser_FECollection : public H1_FECollection
+{
+public:
+   explicit H1Ser_FECollection(const int p, const int dim = 2)
+      : H1_FECollection(p, dim, BasisType::Serendipity) { };
+};
+
 /** Arbitrary order "H^{1/2}-conforming" trace finite elements defined on the
     interface between mesh elements (faces,edges,vertices); these are the trace
     FEs of the H1-conforming FEs. */

fem/gridfunc.cpp

@@ -475,6 +475,101 @@ const
    GetValues(i, ir, vals, vdim);
 }
 
+void GridFunction::GetLaplacians(int i, const IntegrationRule &ir, Vector &laps,
+                                 int vdim)
+const
+{
+   Array<int> dofs;
+   int n = ir.GetNPoints();
+   laps.SetSize(n);
+   fes->GetElementDofs(i, dofs);
+   fes->DofsToVDofs(vdim-1, dofs);
+   const FiniteElement *FElem = fes->GetFE(i);
+   ElementTransformation *ET;
+   ET = fes->GetElementTransformation(i);
+   MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
+               "invalid FE map type");
+
+   int dof = FElem->GetDof();
+   Vector DofLap(dof), loc_data(dof);
+   GetSubVector(dofs, loc_data);
+   for (int k = 0; k < n; k++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(k);
+      ET->SetIntPoint(&ip);
+      FElem->CalcPhysLaplacian(*ET, DofLap);
+      laps(k) = DofLap * loc_data;
+   }
+}
+
+void GridFunction::GetLaplacians(int i, const IntegrationRule &ir, Vector &laps,
+                                 DenseMatrix &tr, int vdim)
+const
+{
+   ElementTransformation *ET;
+   ET = fes->GetElementTransformation(i);
+   ET->Transform(ir, tr);
+
+   GetLaplacians(i, ir, laps, vdim);
+}
+
+
+void GridFunction::GetHessians(int i, const IntegrationRule &ir,
+                               DenseMatrix &hess,
+                               int vdim)
+const
+{
+
+   Array<int> dofs;
+   int n = ir.GetNPoints();
+   fes->GetElementDofs(i, dofs);
+   fes->DofsToVDofs(vdim-1, dofs);
+   const FiniteElement *FElem = fes->GetFE(i);
+   ElementTransformation *ET;
+   ET = fes->GetElementTransformation(i);
+   int dim = FElem->GetDim();
+   int size = (dim*(dim+1))/2;
+
+   MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
+               "invalid FE map type");
+
+   int dof = FElem->GetDof();
+   DenseMatrix DofHes(dof, size);
+   hess.SetSize(n, size);
+
+   Vector loc_data(dof);
+   GetSubVector(dofs, loc_data);
+
+   hess = 0.0;
+   for (int k = 0; k < n; k++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(k);
+      ET->SetIntPoint(&ip);
+      FElem->CalcPhysHessian(*ET, DofHes);
+
+      for (int i = 0; i < size; i++)
+      {
+         for (int d = 0; d < dof; d++)
+         {
+            hess(k,i) += DofHes(d,i) * loc_data[d];
+         }
+      }
+   }
+}
+
+void GridFunction::GetHessians(int i, const IntegrationRule &ir,
+                               DenseMatrix &hess,
+                               DenseMatrix &tr, int vdim)
+const
+{
+   ElementTransformation *ET;
+   ET = fes->GetElementTransformation(i);
+   ET->Transform(ir, tr);
+
+   GetHessians(i, ir, hess, vdim);
+}
+
+
 int GridFunction::GetFaceValues(int i, int side, const IntegrationRule &ir,
                                 Vector &vals, DenseMatrix &tr,
                                 int vdim) const

fem/gridfunc.hpp

@@ -152,6 +152,18 @@ public:
    void GetValues(int i, const IntegrationRule &ir, Vector &vals,
                   DenseMatrix &tr, int vdim = 1) const;
 
+   void GetLaplacians(int i, const IntegrationRule &ir, Vector &laps,
+                      int vdim = 1) const;
+
+   void GetLaplacians(int i, const IntegrationRule &ir, Vector &laps,
+                      DenseMatrix &tr, int vdim = 1) const;
+
+   void GetHessians(int i, const IntegrationRule &ir, DenseMatrix &hess,
+                    int vdim = 1) const;
+
+   void GetHessians(int i, const IntegrationRule &ir, DenseMatrix &hess,
+                    DenseMatrix &tr, int vdim = 1) const;
+
    int GetFaceValues(int i, int side, const IntegrationRule &ir, Vector &vals,
                      DenseMatrix &tr, int vdim = 1) const;
 

fem/libceed/ceed.cpp

@@ -134,7 +134,12 @@ void InitCeedTensorBasisAndRestriction(const mfem::FiniteElementSpace &fes,
          tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
       }
    }
-   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(),
+   CeedInterlaceMode imode = CEED_NONINTERLACED;
+   if (fes.GetOrdering()==Ordering::byVDIM)
+   {
+      imode = CEED_INTERLACED;
+   }
+   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
                              fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }

fem/libceed/diffusion.cpp

@@ -40,9 +40,19 @@ void CeedPADiffusionAssemble(const FiniteElementSpace &fes,
                                      &ceedData.mesh_restr);
    CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);
 
-   CeedElemRestrictionCreateIdentity(ceed, nelem, nqpts,
-                                     nqpts * nelem, dim * (dim + 1) / 2, &ceedData.restr_i);
-   CeedElemRestrictionCreateIdentity(ceed, nelem, nqpts,
+   CeedInterlaceMode imode = CEED_NONINTERLACED;
+   if (fes.GetOrdering()==Ordering::byVDIM)
+   {
+      imode = CEED_INTERLACED;
+   }
+   CeedElemRestrictionCreateIdentity(ceed, imode, nelem, nqpts, nqpts * nelem,
+                                     dim * (dim + 1) / 2, &ceedData.restr_i);
+   CeedInterlaceMode mesh_imode = CEED_NONINTERLACED;
+   if (mesh_fes->GetOrdering()==Ordering::byVDIM)
+   {
+      mesh_imode = CEED_INTERLACED;
+   }
+   CeedElemRestrictionCreateIdentity(ceed, mesh_imode, nelem, nqpts,
                                      nqpts * nelem, 1, &ceedData.mesh_restr_i);
 
    CeedVectorCreate(ceed, mesh->GetNodes()->Size(), &ceedData.node_coords);
@@ -89,11 +99,6 @@ void CeedPADiffusionAssemble(const FiniteElementSpace &fes,
    // Create the operator that builds the quadrature data for the diff operator.
    CeedOperatorCreate(ceed, ceedData.build_qfunc, NULL, NULL,
                       &ceedData.build_oper);
-   CeedTransposeMode lmode = CEED_NOTRANSPOSE;
-   if (mesh_fes->GetOrdering()==Ordering::byVDIM)
-   {
-      lmode = CEED_TRANSPOSE;
-   }
    if (ceedData.coeff_type==CeedCoeff::Grid)
    {
       CeedGridCoeff* ceedCoeff = (CeedGridCoeff*)ceedData.coeff;
@@ -105,15 +110,13 @@ void CeedPADiffusionAssemble(const FiniteElementSpace &fes,
       CeedVectorSetArray(ceedCoeff->coeffVector, CEED_MEM_HOST, CEED_USE_POINTER,
                          ceedCoeff->coeff->GetData());
       CeedOperatorSetField(ceedData.build_oper, "coeff", ceedCoeff->restr,
-                           CEED_NOTRANSPOSE, ceedCoeff->basis, ceedCoeff->coeffVector);
+                           ceedCoeff->basis, ceedCoeff->coeffVector);
    }
-   CeedOperatorSetField(ceedData.build_oper, "dx", ceedData.mesh_restr, lmode,
+   CeedOperatorSetField(ceedData.build_oper, "dx", ceedData.mesh_restr,
                         ceedData.mesh_basis, CEED_VECTOR_ACTIVE);
    CeedOperatorSetField(ceedData.build_oper, "weights", ceedData.mesh_restr_i,
-                        CEED_NOTRANSPOSE,
                         ceedData.mesh_basis, CEED_VECTOR_NONE);
    CeedOperatorSetField(ceedData.build_oper, "rho", ceedData.restr_i,
-                        CEED_NOTRANSPOSE,
                         CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
 
    // Compute the quadrature data for the diff operator.
@@ -134,12 +137,12 @@ void CeedPADiffusionAssemble(const FiniteElementSpace &fes,
 
    // Create the diff operator.
    CeedOperatorCreate(ceed, ceedData.apply_qfunc, NULL, NULL, &ceedData.oper);
-   CeedOperatorSetField(ceedData.oper, "u", ceedData.restr, CEED_NOTRANSPOSE,
-                        ceedData.basis, CEED_VECTOR_ACTIVE);
-   CeedOperatorSetField(ceedData.oper, "rho", ceedData.restr_i, CEED_NOTRANSPOSE,
+   CeedOperatorSetField(ceedData.oper, "u", ceedData.restr, ceedData.basis,
+                        CEED_VECTOR_ACTIVE);
+   CeedOperatorSetField(ceedData.oper, "rho", ceedData.restr_i,
                         CEED_BASIS_COLLOCATED, ceedData.rho);
-   CeedOperatorSetField(ceedData.oper, "v", ceedData.restr, CEED_NOTRANSPOSE,
-                        ceedData.basis, CEED_VECTOR_ACTIVE);
+   CeedOperatorSetField(ceedData.oper, "v", ceedData.restr, ceedData.basis,
+                        CEED_VECTOR_ACTIVE);
 
    CeedVectorCreate(ceed, fes.GetNDofs(), &ceedData.u);
    CeedVectorCreate(ceed, fes.GetNDofs(), &ceedData.v);

fem/libceed/mass.cpp

@@ -40,9 +40,19 @@ void CeedPAMassAssemble(const FiniteElementSpace &fes,
                                      &ceedData.mesh_restr);
    CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);
 
-   CeedElemRestrictionCreateIdentity(ceed, nelem, nqpts,
+   CeedInterlaceMode imode = CEED_NONINTERLACED;
+   if (fes.GetOrdering()==Ordering::byVDIM)
+   {
+      imode = CEED_INTERLACED;
+   }
+   CeedElemRestrictionCreateIdentity(ceed, imode, nelem, nqpts,
                                      nqpts*nelem, 1, &ceedData.restr_i);
-   CeedElemRestrictionCreateIdentity(ceed, nelem, nqpts,
+   CeedInterlaceMode mesh_imode = CEED_NONINTERLACED;
+   if (mesh_fes->GetOrdering()==Ordering::byVDIM)
+   {
+      mesh_imode = CEED_INTERLACED;
+   }
+   CeedElemRestrictionCreateIdentity(ceed, mesh_imode, nelem, nqpts,
                                      nqpts*nelem, 1, &ceedData.mesh_restr_i);
 
    CeedVectorCreate(ceed, mesh->GetNodes()->Size(), &ceedData.node_coords);
@@ -90,11 +100,6 @@ void CeedPAMassAssemble(const FiniteElementSpace &fes,
    // Create the operator that builds the quadrature data for the mass operator.
    CeedOperatorCreate(ceed, ceedData.build_qfunc, NULL, NULL,
                       &ceedData.build_oper);
-   CeedTransposeMode lmode = CEED_NOTRANSPOSE;
-   if (mesh_fes->GetOrdering()==Ordering::byVDIM)
-   {
-      lmode = CEED_TRANSPOSE;
-   }
    if (ceedData.coeff_type==CeedCoeff::Grid)
    {
       CeedGridCoeff* ceedCoeff = (CeedGridCoeff*)ceedData.coeff;
@@ -106,16 +111,13 @@ void CeedPAMassAssemble(const FiniteElementSpace &fes,
       CeedVectorSetArray(ceedCoeff->coeffVector, CEED_MEM_HOST, CEED_USE_POINTER,
                          ceedCoeff->coeff->GetData());
       CeedOperatorSetField(ceedData.build_oper, "coeff", ceedCoeff->restr,
-                           CEED_NOTRANSPOSE,
                            ceedCoeff->basis, ceedCoeff->coeffVector);
    }
-   CeedOperatorSetField(ceedData.build_oper, "dx", ceedData.mesh_restr, lmode,
+   CeedOperatorSetField(ceedData.build_oper, "dx", ceedData.mesh_restr,
                         ceedData.mesh_basis, CEED_VECTOR_ACTIVE);
    CeedOperatorSetField(ceedData.build_oper, "weights", ceedData.mesh_restr_i,
-                        CEED_NOTRANSPOSE,
                         ceedData.mesh_basis, CEED_VECTOR_NONE);
    CeedOperatorSetField(ceedData.build_oper, "rho", ceedData.restr_i,
-                        CEED_NOTRANSPOSE,
                         CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
 
    // Compute the quadrature data for the mass operator.
@@ -132,11 +134,11 @@ void CeedPAMassAssemble(const FiniteElementSpace &fes,
 
    // Create the mass operator.
    CeedOperatorCreate(ceed, ceedData.apply_qfunc, NULL, NULL, &ceedData.oper);
-   CeedOperatorSetField(ceedData.oper, "u", ceedData.restr, CEED_NOTRANSPOSE,
+   CeedOperatorSetField(ceedData.oper, "u", ceedData.restr,
                         ceedData.basis, CEED_VECTOR_ACTIVE);
-   CeedOperatorSetField(ceedData.oper, "rho", ceedData.restr_i, CEED_NOTRANSPOSE,
+   CeedOperatorSetField(ceedData.oper, "rho", ceedData.restr_i,
                         CEED_BASIS_COLLOCATED, ceedData.rho);
-   CeedOperatorSetField(ceedData.oper, "v", ceedData.restr, CEED_NOTRANSPOSE,
+   CeedOperatorSetField(ceedData.oper, "v", ceedData.restr,
                         ceedData.basis, CEED_VECTOR_ACTIVE);
 
    CeedVectorCreate(ceed, fes.GetNDofs(), &ceedData.u);

fem/pnonlinearform.cpp

@@ -46,7 +46,6 @@ double ParNonlinearForm::GetParGridFunctionEnergy(const Vector &x) const
 void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
    NonlinearForm::Mult(x, y); // x --(P)--> aux1 --(A_local)--> aux2
-   Y.MakeRef(aux2, 0); // aux2 contains A_local.P.x
 
    if (fnfi.Size())
    {
@@ -58,6 +57,7 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
       Array<int> vdofs1, vdofs2;
       Vector el_x, el_y;
 
+      aux1.HostReadWrite();
       X.MakeRef(aux1, 0); // aux1 contains P.x
       X.ExchangeFaceNbrData();
       const int n_shared_faces = pmesh->GetNSharedFaces();
@@ -78,13 +78,14 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
          for (int k = 0; k < fnfi.Size(); k++)
          {
             fnfi[k]->AssembleFaceVector(*fe1, *fe2, *tr, el_x, el_y);
-            Y.AddElementVector(vdofs1, el_y.GetData());
+            aux2.AddElementVector(vdofs1, el_y.GetData());
          }
       }
    }
 
-   P->MultTranspose(Y, y);
+   P->MultTranspose(aux2, y);
 
+   y.HostReadWrite();
    for (int i = 0; i < ess_tdof_list.Size(); i++)
    {
       y(ess_tdof_list[i]) = 0.0;

linalg/densemat.cpp

@@ -56,6 +56,9 @@ extern "C" void
 dgesvd_(char *JOBU, char *JOBVT, int *M, int *N, double *A, int *LDA,
         double *S, double *U, int *LDU, double *VT, int *LDVT, double *WORK,
         int *LWORK, int *INFO);
+extern "C" void
+dtrsm_(char *side, char *uplo, char *transa, char *diag, int *m, int *n,
+       double *alpha, double *a, int *lda, double *b, int *ldb);
 #endif
 
 
@@ -2902,22 +2905,36 @@ void DenseMatrix::SetCol(int col, double value)
    }
 }
 
-void DenseMatrix::SetRow(int r, const Vector &row)
+void DenseMatrix::SetRow(int r, const double* row)
 {
+   MFEM_ASSERT(row != nullptr, "supplied row pointer is null");
    for (int j = 0; j < Width(); j++)
    {
       (*this)(r, j) = row[j];
    }
 }
 
-void DenseMatrix::SetCol(int c, const Vector &col)
+void DenseMatrix::SetRow(int r, const Vector &row)
 {
+   MFEM_ASSERT(Width() == row.Size(), "");
+   SetRow(r, row.GetData());
+}
+
+void DenseMatrix::SetCol(int c, const double* col)
+{
+   MFEM_ASSERT(col != nullptr, "supplied column pointer is null");
    for (int i = 0; i < Height(); i++)
    {
       (*this)(i, c) = col[i];
    }
 }
 
+void DenseMatrix::SetCol(int c, const Vector &col)
+{
+   MFEM_ASSERT(Height() == col.Size(), "");
+   SetCol(c, col.GetData());
+}
+
 void DenseMatrix::Threshold(double eps)
 {
    for (int col = 0; col < Width(); col++)
@@ -3060,6 +3077,53 @@ void Add(double alpha, const DenseMatrix &A,
    Add(alpha, A.GetData(), beta, B.GetData(), C);
 }
 
+bool LinearSolve(DenseMatrix& A, double* X, double TOL)
+{
+   MFEM_VERIFY(A.IsSquare(), "A must be a square matrix!");
+   MFEM_ASSERT(A.NumCols() > 0, "supplied matrix, A, is empty!");
+   MFEM_ASSERT(X != nullptr, "supplied vector, X, is null!");
+
+   int N = A.NumCols();
+
+   switch (N)
+   {
+      case 1:
+      {
+         double det = A(0,0);
+         if (std::abs(det) <= TOL) { return false; } // singular
+
+         X[0] /= det;
+         break;
+      }
+      case 2:
+      {
+         double det = A.Det();
+         if (std::abs(det) <= TOL) { return false; } // singular
+
+         double invdet = 1. / det;
+
+         double b0 = X[0];
+         double b1 = X[1];
+
+         X[0] = ( A(1,1)*b0 - A(0,1)*b1) * invdet;
+         X[1] = (-A(1,0)*b0 + A(0,0)*b1) * invdet;
+         break;
+      }
+      default:
+      {
+         // default to LU factorization for the general case
+         Array<int> ipiv(N);
+         LUFactors lu(A.Data(), ipiv);
+
+         if (!lu.Factor(N,TOL)) { return false; } // singular
+
+         lu.Solve(N, 1, X);
+      }
+
+   } // END switch
+
+   return true;
+}
 
 void Mult(const DenseMatrix &b, const DenseMatrix &c, DenseMatrix &a)
 {
@@ -3097,6 +3161,39 @@ void Mult(const DenseMatrix &b, const DenseMatrix &c, DenseMatrix &a)
 #endif
 }
 
+void AddMult_a(double alpha, const DenseMatrix &b, const DenseMatrix &c,
+               DenseMatrix &a)
+{
+   MFEM_ASSERT(a.Height() == b.Height() && a.Width() == c.Width() &&
+               b.Width() == c.Height(), "incompatible dimensions");
+
+#ifdef MFEM_USE_LAPACK
+   static char transa = 'N', transb = 'N';
+   static double beta = 1.0;
+   int m = b.Height(), n = c.Width(), k = b.Width();
+
+   dgemm_(&transa, &transb, &m, &n, &k, &alpha, b.Data(), &m,
+          c.Data(), &k, &beta, a.Data(), &m);
+#else
+   const int ah = a.Height();
+   const int aw = a.Width();
+   const int bw = b.Width();
+   double *ad = a.Data();
+   const double *bd = b.Data();
+   const double *cd = c.Data();
+   for (int j = 0; j < aw; j++)
+   {
+      for (int k = 0; k < bw; k++)
+      {
+         for (int i = 0; i < ah; i++)
+         {
+            ad[i+j*ah] += alpha * bd[i+k*ah] * cd[k+j*bw];
+         }
+      }
+   }
+#endif
+}
+
 void AddMult(const DenseMatrix &b, const DenseMatrix &c, DenseMatrix &a)
 {
    MFEM_ASSERT(a.Height() == b.Height() && a.Width() == c.Width() &&
@@ -3937,12 +4034,12 @@ void AddMult_a_VVt(const double a, const Vector &v, DenseMatrix &VVt)
 }
 
 
-void LUFactors::Factor(int m)
+bool LUFactors::Factor(int m, double TOL)
 {
 #ifdef MFEM_USE_LAPACK
    int info = 0;
    if (m) { dgetrf_(&m, &m, data, &m, ipiv, &info); }
-   MFEM_VERIFY(!info, "LAPACK: error in DGETRF");
+   return info == 0;
 #else
    // compiling without LAPACK
    double *data = this->data;
@@ -3971,8 +4068,13 @@ void LUFactors::Factor(int m)
             }
          }
       }
-      MFEM_ASSERT(data[i+i*m] != 0.0, "division by zero");
-      const double a_ii_inv = 1.0/data[i+i*m];
+
+      if (abs(data[i + i*m]) <= TOL)
+      {
+         return false; // failed
+      }
+
+      const double a_ii_inv = 1.0 / data[i+i*m];
       for (int j = i+1; j < m; j++)
       {
          data[j+i*m] *= a_ii_inv;
@@ -3987,6 +4089,8 @@ void LUFactors::Factor(int m)
       }
    }
 #endif
+
+   return true; // success
 }
 
 double LUFactors::Det(int m) const
@@ -4100,6 +4204,64 @@ void LUFactors::Solve(int m, int n, double *X) const
 #endif
 }
 
+void LUFactors::RightSolve(int m, int n, double *X) const
+{
+   double *x;
+#ifdef MFEM_USE_LAPACK
+   char n_ch = 'N', side = 'R', u_ch = 'U', l_ch = 'L';
+   double alpha = 1.0;
+   if (m > 0 && n > 0)
+   {
+      dtrsm_(&side,&u_ch,&n_ch,&n_ch,&n,&m,&alpha,data,&m,X,&n);
+      dtrsm_(&side,&l_ch,&n_ch,&u_ch,&n,&m,&alpha,data,&m,X,&n);
+   }
+#else
+   // compiling without LAPACK
+   const double *data = this->data;
+   const int *ipiv = this->ipiv;
+
+   // X <- X U^{-1}
+   x = X;
+   for (int k = 0; k < n; k++)
+   {
+      for (int j = 0; j < m; j++)
+      {
+         const double x_j = ( x[j*n] /= data[j+j*m]);
+         for (int i = j+1; i < m; i++)
+         {
+            x[i*n] -= data[j + i*m] * x_j;
+         }
+      }
+      ++x;
+   }
+
+   // X <- X L^{-1}
+   x = X;
+   for (int k = 0; k < n; k++)
+   {
+      for (int j = m-1; j >= 0; j--)
+      {
+         const double x_j = x[j*n];
+         for (int i = 0; i < j; i++)
+         {
+            x[i*n] -= data[j + i*m] * x_j;
+         }
+      }
+      ++x;
+   }
+#endif
+   // X <- X P
+   x = X;
+   for (int k = 0; k < n; k++)
+   {
+      for (int i = 0; i < m; i++)
+      {
+         Swap<double>(x[i*n], x[(ipiv[i]-ipiv_base)*n]);
+      }
+      ++x;
+   }
+}
+
 void LUFactors::GetInverseMatrix(int m, double *X) const
 {
    // A^{-1} = U^{-1} L^{-1} P

linalg/densemat.hpp

@@ -262,9 +262,13 @@ public:
    void GetColumnReference(int c, Vector &col)
    { col.SetDataAndSize(data + c * height, height); }
 
+   void SetRow(int r, const double* row);
    void SetRow(int r, const Vector &row);
+
+   void SetCol(int c, const double* col);
    void SetCol(int c, const Vector &col);
 
+
    /// Set all entries of a row to the specified value.
    void SetRow(int row, double value);
    /// Set all entries of a column to the specified value.
@@ -370,12 +374,32 @@ void Add(double alpha, const double *A,
 void Add(double alpha, const DenseMatrix &A,
          double beta,  const DenseMatrix &B, DenseMatrix &C);
 
+/// @brief Solves the dense linear system, `A * X = B` for `X`
+///
+/// @param [in,out] A the square matrix for the linear system
+/// @param [in,out] X the rhs vector, B, on input, the solution, X, on output.
+/// @param [in] TOL optional fuzzy comparison tolerance. Defaults to 1e-9.
+///
+/// @return status set to true if successful, otherwise, false.
+///
+/// @note This routine may replace the contents of the input Matrix, A, with the
+///       corresponding LU factorization of the matrix. Matrices of size 1x1 and
+///       2x2 are handled explicitly.
+///
+/// @pre A.IsSquare() == true
+/// @pre X != nullptr
+bool LinearSolve(DenseMatrix& A, double* X, double TOL = 1.e-9);
+
 /// Matrix matrix multiplication.  A = B * C.
 void Mult(const DenseMatrix &b, const DenseMatrix &c, DenseMatrix &a);
 
 /// Matrix matrix multiplication.  A += B * C.
 void AddMult(const DenseMatrix &b, const DenseMatrix &c, DenseMatrix &a);
 
+/// Matrix matrix multiplication.  A += alpha * B * C.
+void AddMult_a(double alpha, const DenseMatrix &b, const DenseMatrix &c,
+               DenseMatrix &a);
+
 /** Calculate the adjugate of a matrix (for NxN matrices, N=1,2,3) or the matrix
     adj(A^t.A).A^t for rectangular matrices (2x1, 3x1, or 3x2). This operation
     is well defined even when the matrix is not full rank. */
@@ -470,10 +494,19 @@ public:
 
    LUFactors(double *data_, int *ipiv_) : data(data_), ipiv(ipiv_) { }
 
-   /** Factorize the current data of size (m x m) overwriting it with the LU
-       factors. The factorization is such that L.U = P.A, where A is the
-       original matrix and P is a permutation matrix represented by ipiv. */
-   void Factor(int m);
+   /**
+    * @brief Compute the LU factorization of the current matrix
+    *
+    * Factorize the current matrix of size (m x m) overwriting it with the
+    * LU factors. The factorization is such that L.U = P.A, where A is the
+    * original matrix and P is a permutation matrix represented by ipiv.
+    *
+    * @param [in] m size of the square matrix
+    * @param [in] TOL optional fuzzy comparison tolerance. Defaults to 0.0.
+    *
+    * @return status set to true if successful, otherwise, false.
+    */
+   bool Factor(int m, double TOL = 0.0);
 
    /** Assuming L.U = P.A factored data of size (m x m), compute |A|
        from the diagonal values of U and the permutation information. */
@@ -495,6 +528,10 @@ public:
        for a matrix X of size (m x n). */
    void Solve(int m, int n, double *X) const;
 
+   /** Assuming L.U = P.A factored data of size (m x m), compute X <- X A^{-1},
+       for a matrix X of size (n x m). */
+   void RightSolve(int m, int n, double *X) const;
+
    /// Assuming L.U = P.A factored data of size (m x m), compute X <- A^{-1}.
    void GetInverseMatrix(int m, double *X) const;
 

linalg/matrix.hpp

@@ -42,6 +42,9 @@ public:
    /// Creates a matrix of the given height and width.
    explicit Matrix(int h, int w) : Operator(h, w) { }
 
+   /// Returns whether the matrix is a square matrix.
+   bool IsSquare() const { return (height == width); };
+
    /// Returns reference to a_{ij}.
    virtual double &Elem(int i, int j) = 0;
 

linalg/petsc.cpp

@@ -142,6 +142,7 @@ typedef struct
    mfem::TimeDependentOperator     *op;        // The time-dependent operator
    mfem::PetscBCHandler            *bchandler; // Handling of essential bc
    mfem::Vector                    *work;      // Work vector
+   mfem::Vector                    *work2;     // Work vector
    mfem::Operator::Type            jacType;    // OperatorType for the Jacobian
    enum mfem::PetscODESolver::Type type;
    PetscReal                       cached_shift;
@@ -2166,6 +2167,7 @@ void PetscSolver::CreatePrivateContext()
       ts_ctx->op = NULL;
       ts_ctx->bchandler = NULL;
       ts_ctx->work = NULL;
+      ts_ctx->work2 = NULL;
       ts_ctx->cached_shift = std::numeric_limits<PetscReal>::min();
       ts_ctx->cached_ijacstate = -1;
       ts_ctx->cached_rhsjacstate = -1;
@@ -2190,6 +2192,7 @@ void PetscSolver::FreePrivateContext()
    {
       __mfem_ts_ctx *ts_ctx = (__mfem_ts_ctx *)private_ctx;
       delete ts_ctx->work;
+      delete ts_ctx->work2;
    }
    ierr = PetscFree(private_ctx); CCHKERRQ(PETSC_COMM_SELF,ierr);
 }
@@ -2295,6 +2298,25 @@ void PetscBCHandler::FixResidualBC(const Vector& x, Vector& y)
    }
 }
 
+void PetscBCHandler::Zero(Vector &x)
+{
+   (*this).SetUp(x.Size());
+   for (int i = 0; i < ess_tdof_list.Size(); ++i)
+   {
+      x[ess_tdof_list[i]] = 0.0;
+   }
+}
+
+void PetscBCHandler::ZeroBC(const Vector &x, Vector &y)
+{
+   (*this).SetUp(x.Size());
+   y = x;
+   for (int i = 0; i < ess_tdof_list.Size(); ++i)
+   {
+      y[ess_tdof_list[i]] = 0.0;
+   }
+}
+
 // PetscLinearSolver methods
 
 PetscLinearSolver::PetscLinearSolver(MPI_Comm comm, const std::string &prefix,
@@ -3672,12 +3694,17 @@ static PetscErrorCode __mfem_ts_ifunction(TS ts, PetscReal t, Vec x, Vec xp,
    if (ts_ctx->bchandler)
    {
       // we evaluate the ImplicitMult method with the correct bc
+      // this means the correct time derivative for essential boundary
+      // dofs is zero
       if (!ts_ctx->work) { ts_ctx->work = new mfem::Vector(xx.Size()); }
+      if (!ts_ctx->work2) { ts_ctx->work2 = new mfem::Vector(xx.Size()); }
       mfem::PetscBCHandler *bchandler = ts_ctx->bchandler;
       mfem::Vector* txx = ts_ctx->work;
+      mfem::Vector* txp = ts_ctx->work2;
       bchandler->SetTime(t);
       bchandler->ApplyBC(xx,*txx);
-      op->ImplicitMult(*txx,yy,ff);
+      bchandler->ZeroBC(yy,*txp);
+      op->ImplicitMult(*txx,*txp,ff);
       // and fix the residual (i.e. f_\partial\Omega = u - g(t))
       bchandler->FixResidualBC(xx,ff);
    }

linalg/petsc.hpp

@@ -499,6 +499,12 @@ public:
    /// y = x-g on ess_tdof_list, the rest of y is unchanged
    void FixResidualBC(const Vector& x, Vector& y);
 
+   /// Replace boundary dofs with 0
+   void Zero(Vector &x);
+
+   /// y = x on ess_tdof_list_c and y = 0 on ess_tdof_list
+   void ZeroBC(const Vector &x, Vector &y);
+
 private:
    enum Type bctype;
    bool setup;

linalg/solvers.cpp

@@ -17,6 +17,7 @@
 #include <iomanip>
 #include <algorithm>
 #include <cmath>
+#include <set>
 
 namespace mfem
 {
@@ -802,10 +803,12 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
       return;
    }
 
-   if (print_level>=0)
+   if (print_level == 1)
+   {
       mfem::out << "   Pass : " << setw(2) << 1
                 << "   Iteration : " << setw(3) << 0
                 << "  || r || = " << beta << endl;
+   }
 
    Array<Vector*> v(m+1);
    Array<Vector*> z(m+1);
@@ -861,17 +864,25 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
 
          double resid = fabs(s(i+1));
          MFEM_ASSERT(IsFinite(resid), "resid = " << resid);
-         if (print_level >= 0)
+         if (print_level == 1)
+         {
             mfem::out << "   Pass : " << setw(2) << (j-1)/m+1
                       << "   Iteration : " << setw(3) << j
                       << "  || r || = " << resid << endl;
+         }
 
-         if ( resid <= final_norm)
+         if (resid <= final_norm)
          {
             Update(x, i, H, s, z);
             final_norm = resid;
             final_iter = j;
             converged = 1;
+
+            if (print_level == 2)
+            {
+               mfem::out << "Number of FGMRES iterations: " << final_iter << endl;
+            }
+
             for (i= 0; i<=m; i++)
             {
                if (v[i]) { delete v[i]; }
@@ -881,7 +892,7 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
          }
       }
 
-      if (print_level>=0)
+      if (print_level == 1)
       {
          mfem::out << "Restarting..." << endl;
       }
@@ -892,11 +903,17 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
       subtract(b,r,r);
       beta = Norm(r);
       MFEM_ASSERT(IsFinite(beta), "beta = " << beta);
-      if ( beta <= final_norm)
+      if (beta <= final_norm)
       {
          final_norm = beta;
          final_iter = j;
          converged = 1;
+
+         if (print_level == 2)
+         {
+            mfem::out << "Number of FGMRES iterations: " << final_iter << endl;
+         }
+
          for (i= 0; i<=m; i++)
          {
             if (v[i]) { delete v[i]; }
@@ -912,8 +929,13 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
       if (z[i]) { delete z[i]; }
    }
    converged = 0;
-   return;
 
+   if (print_level >= 0)
+   {
+      mfem::out << "FGMRES: No convergence!" << endl;
+   }
+
+   return;
 }
 
 
@@ -1790,6 +1812,459 @@ slbqp_done:
    }
 }
 
+struct WeightMinHeap
+{
+   const std::vector<double> &w;
+   std::vector<size_t> c;
+   std::vector<int> loc;
+
+   WeightMinHeap(const std::vector<double> &w_) : w(w_)
+   {
+      c.reserve(w.size());
+      loc.resize(w.size());
+      for (size_t i=0; i<w.size(); ++i) { push(i); }
+   }
+
+   size_t percolate_up(size_t pos, double val)
+   {
+      for (; pos > 0 && w[c[(pos-1)/2]] > val; pos = (pos-1)/2)
+      {
+         c[pos] = c[(pos-1)/2];
+         loc[c[(pos-1)/2]] = pos;
+      }
+      return pos;
+   }
+
+   size_t percolate_down(size_t pos, double val)
+   {
+      while (2*pos+1 < c.size())
+      {
+         size_t left = 2*pos+1;
+         size_t right = left+1;
+         size_t tgt;
+         if (right < c.size() && w[c[right]] < w[c[left]]) { tgt = right; }
+         else { tgt = left; }
+         if (w[c[tgt]] < val)
+         {
+            c[pos] = c[tgt];
+            loc[c[tgt]] = pos;
+            pos = tgt;
+         }
+         else
+         {
+            break;
+         }
+      }
+      return pos;
+   }
+
+   void push(size_t i)
+   {
+      double val = w[i];
+      c.push_back(0);
+      size_t pos = c.size()-1;
+      pos = percolate_up(pos, val);
+      c[pos] = i;
+      loc[i] = pos;
+   }
+
+   int pop()
+   {
+      size_t i = c[0];
+      size_t j = c.back();
+      c.pop_back();
+      // Mark as removed
+      loc[i] = -1;
+      if (c.empty()) { return i; }
+      double val = w[j];
+      size_t pos = 0;
+      pos = percolate_down(pos, val);
+      c[pos] = j;
+      loc[j] = pos;
+      return i;
+   }
+
+   void update(size_t i)
+   {
+      size_t pos = loc[i];
+      double val = w[i];
+      pos = percolate_up(pos, val);
+      pos = percolate_down(pos, val);
+      c[pos] = i;
+      loc[i] = pos;
+   }
+
+   bool picked(size_t i)
+   {
+      return loc[i] < 0;
+   }
+};
+
+void MinimumDiscardedFillOrdering(SparseMatrix &C, Array<int> &p)
+{
+   int n = C.Width();
+   // Scale rows by reciprocal of diagonal and take absolute value
+   Vector D;
+   C.GetDiag(D);
+   int *I = C.GetI();
+   int *J = C.GetJ();
+   double *V = C.GetData();
+   for (int i=0; i<n; ++i)
+   {
+      for (int j=I[i]; j<I[i+1]; ++j)
+      {
+         V[j] = abs(V[j]/D[i]);
+      }
+   }
+
+   std::vector<double> w(n, 0.0);
+   // Compute the discarded-fill weights
+   for (int k=0; k<n; ++k)
+   {
+      for (int ii=I[k]; ii<I[k+1]; ++ii)
+      {
+         double C_ki = V[ii];
+         for (int jj=I[k]; jj<I[k+1]; ++jj)
+         {
+            if (jj == ii) { continue; }
+            double C_jk = V[jj];
+            w[k] += pow(C_jk*C_ki, 2);
+         }
+      }
+      w[k] = sqrt(w[k]);
+   }
+
+   WeightMinHeap w_heap(w);
+
+   // Compute ordering
+   p.SetSize(n);
+   for (int i=0; i<n; ++i)
+   {
+      int pi = w_heap.pop();
+      p[n-1-i] = pi;
+      w[pi] = -1;
+      for (int kk=I[pi]; kk<I[pi+1]; ++kk)
+      {
+         int k = J[kk];
+         if (w_heap.picked(k)) { continue; }
+         // Recompute weight
+         w[k] = 0.0;
+         for (int ii=I[k]; ii<I[k+1]; ++ii)
+         {
+            if (w_heap.picked(J[ii])) { continue; }
+            double C_ki = V[ii];
+            for (int jj=I[k]; jj<I[k+1]; ++jj)
+            {
+               if (jj == ii || w_heap.picked(J[jj])) { continue; }
+               double C_jk = V[jj];
+               w[k] += pow(C_jk*C_ki, 2);
+            }
+         }
+         w[k] = sqrt(w[k]);
+         w_heap.update(k);
+      }
+   }
+}
+
+BlockILU::BlockILU(int block_size_,
+                   Reordering reordering_,
+                   int k_fill_)
+   : Solver(0),
+     block_size(block_size_),
+     k_fill(k_fill_),
+     reordering(reordering_)
+{ }
+
+BlockILU::BlockILU(Operator &op,
+                   int block_size_,
+                   Reordering reordering_,
+                   int k_fill_)
+   : BlockILU(block_size_, reordering_, k_fill_)
+{
+   SetOperator(op);
+}
+
+void BlockILU::SetOperator(const Operator &op)
+{
+   const SparseMatrix *A = NULL;
+#ifdef MFEM_USE_MPI
+   const HypreParMatrix *A_par = dynamic_cast<const HypreParMatrix *>(&op);
+   SparseMatrix A_par_diag;
+   if (A_par != NULL)
+   {
+      A_par->GetDiag(A_par_diag);
+      A = &A_par_diag;
+   }
+#endif
+   if (A == NULL)
+   {
+      A = dynamic_cast<const SparseMatrix *>(&op);
+      if (A == NULL)
+      {
+         MFEM_ABORT("BlockILU must be created with a SparseMatrix or HypreParMatrix");
+      }
+   }
+   height = op.Height();
+   width = op.Width();
+   MFEM_ASSERT(A->Finalized(), "Matrix must be finalized.");
+   CreateBlockPattern(*A);
+   Factorize();
+}
+
+void BlockILU::CreateBlockPattern(const SparseMatrix &A)
+{
+   MFEM_VERIFY(k_fill == 0, "Only block ILU(0) is currently supported.");
+   if (A.Height() % block_size != 0)
+   {
+      MFEM_ABORT("BlockILU: block size must evenly divide the matrix size");
+   }
+
+   int nrows = A.Height();
+   const int *I = A.GetI();
+   const int *J = A.GetJ();
+   const double *V = A.GetData();
+   int nnz = 0;
+   int nblockrows = nrows / block_size;
+
+   std::vector<std::set<int>> unique_block_cols(nblockrows);
+
+   for (int iblock = 0; iblock < nblockrows; ++iblock)
+   {
+      for (int bi = 0; bi < block_size; ++bi)
+      {
+         int i = iblock * block_size + bi;
+         for (int k = I[i]; k < I[i + 1]; ++k)
+         {
+            unique_block_cols[iblock].insert(J[k] / block_size);
+         }
+      }
+      nnz += unique_block_cols[iblock].size();
+   }
+
+   if (reordering != Reordering::NONE)
+   {
+      SparseMatrix C(nblockrows, nblockrows);
+      for (int iblock = 0; iblock < nblockrows; ++iblock)
+      {
+         for (int jblock : unique_block_cols[iblock])
+         {
+            for (int bi = 0; bi < block_size; ++bi)
+            {
+               int i = iblock * block_size + bi;
+               for (int k = I[i]; k < I[i + 1]; ++k)
+               {
+                  int j = J[k];
+                  if (j >= jblock * block_size && j < (jblock + 1) * block_size)
+                  {
+                     C.Add(iblock, jblock, V[k]*V[k]);
+                  }
+               }
+            }
+         }
+      }
+      C.Finalize(false);
+      double *CV = C.GetData();
+      for (int i=0; i<C.NumNonZeroElems(); ++i)
+      {
+         CV[i] = sqrt(CV[i]);
+      }
+
+      switch (reordering)
+      {
+         case Reordering::MINIMUM_DISCARDED_FILL:
+            MinimumDiscardedFillOrdering(C, P);
+            break;
+         default:
+            MFEM_ABORT("BlockILU: unknown reordering")
+      }
+   }
+   else
+   {
+      // No reordering: permutation is identity
+      P.SetSize(nblockrows);
+      for (int i=0; i<nblockrows; ++i)
+      {
+         P[i] = i;
+      }
+   }
+
+   // Compute inverse permutation
+   Pinv.SetSize(nblockrows);
+   for (int i=0; i<nblockrows; ++i)
+   {
+      Pinv[P[i]] = i;
+   }
+
+   // Permute columns
+   std::vector<std::vector<int>> unique_block_cols_perminv(nblockrows);
+   for (int i=0; i<nblockrows; ++i)
+   {
+      std::vector<int> &cols = unique_block_cols_perminv[i];
+      for (int j : unique_block_cols[P[i]])
+      {
+         cols.push_back(Pinv[j]);
+      }
+      std::sort(cols.begin(), cols.end());
+   }
+
+   ID.SetSize(nblockrows);
+   IB.SetSize(nblockrows + 1);
+   IB[0] = 0;
+   JB.SetSize(nnz);
+   AB.SetSize(block_size, block_size, nnz);
+   DB.SetSize(block_size, block_size, nblockrows);
+   AB = 0.0;
+   DB = 0.0;
+   ipiv.SetSize(block_size*nblockrows);
+   int counter = 0;
+
+   for (int iblock = 0; iblock < nblockrows; ++iblock)
+   {
+      int iblock_perm = P[iblock];
+      for (int jblock : unique_block_cols_perminv[iblock])
+      {
+         int jblock_perm = P[jblock];
+         if (iblock == jblock)
+         {
+            ID[iblock] = counter;
+         }
+         JB[counter] = jblock;
+         for (int bi = 0; bi < block_size; ++bi)
+         {
+            int i = iblock_perm*block_size + bi;
+            for (int k = I[i]; k < I[i + 1]; ++k)
+            {
+               int j = J[k];
+               if (j >= jblock_perm*block_size && j < (jblock_perm + 1)*block_size)
+               {
+                  int bj = j - jblock_perm*block_size;
+                  double val = V[k];
+                  AB(bi, bj, counter) = val;
+                  // Extract the diagonal
+                  if (iblock == jblock)
+                  {
+                     DB(bi, bj, iblock) = val;
+                  }
+               }
+            }
+         }
+         ++counter;
+      }
+      IB[iblock + 1] = counter;
+   }
+}
+
+void BlockILU::Factorize()
+{
+   int nblockrows = Height()/block_size;
+
+   // Precompute LU factorization of diagonal blocks
+   for (int i=0; i<nblockrows; ++i)
+   {
+      LUFactors factorization(DB.GetData(i), &ipiv[i*block_size]);
+      factorization.Factor(block_size);
+   }
+
+   // Note: we use UseExternalData to extract submatrices from the tensor AB
+   // instead of the DenseTensor call operator, because the call operator does
+   // not allow for two simultaneous submatrix views into the same tensor
+   DenseMatrix A_ik, A_ij, A_kj;
+   // Loop over block rows (starting with second block row)
+   for (int i=1; i<nblockrows; ++i)
+   {
+      // Find all nonzeros to the left of the diagonal in row i
+      for (int kk=IB[i]; kk<IB[i+1]; ++kk)
+      {
+         int k = JB[kk];
+         // Make sure we're still to the left of the diagonal
+         if (k == i) { break; }
+         if (k > i)
+         {
+            MFEM_ABORT("Matrix must be sorted with nonzero diagonal");
+         }
+         LUFactors A_kk_inv(DB.GetData(k), &ipiv[k*block_size]);
+         A_ik.UseExternalData(&AB(0,0,kk), block_size, block_size);
+         // A_ik = A_ik * A_kk^{-1}
+         A_kk_inv.RightSolve(block_size, block_size, A_ik.GetData());
+         // Modify everything to the right of k in row i
+         for (int jj=kk+1; jj<IB[i+1]; ++jj)
+         {
+            int j = JB[jj];
+            if (j <= k) { continue; } // Superfluous because JB is sorted?
+            A_ij.UseExternalData(&AB(0,0,jj), block_size, block_size);
+            for (int ll=IB[k]; ll<IB[k+1]; ++ll)
+            {
+               int l = JB[ll];
+               if (l == j)
+               {
+                  A_kj.UseExternalData(&AB(0,0,ll), block_size, block_size);
+                  // A_ij = A_ij - A_ik*A_kj;
+                  AddMult_a(-1.0, A_ik, A_kj, A_ij);
+                  // If we need to, update diagonal factorization
+                  if (j == i)
+                  {
+                     DB(i) = A_ij;
+                     LUFactors factorization(DB.GetData(i), &ipiv[i*block_size]);
+                     factorization.Factor(block_size);
+                  }
+                  break;
+               }
+            }
+         }
+      }
+   }
+}
+
+void BlockILU::Mult(const Vector &b, Vector &x) const
+{
+   MFEM_ASSERT(height > 0, "BlockILU(0) preconditioner is not constructed");
+   int nblockrows = Height()/block_size;
+   y.SetSize(Height());
+
+   DenseMatrix B;
+   Vector yi, yj, xi, xj;
+   Vector tmp(block_size);
+   // Forward substitute to solve Ly = b
+   // Implicitly, L has identity on the diagonal
+   y = 0.0;
+   for (int i=0; i<nblockrows; ++i)
+   {
+      yi.SetDataAndSize(&y[i*block_size], block_size);
+      for (int ib=0; ib<block_size; ++ib)
+      {
+         yi[ib] = b[ib + P[i]*block_size];
+      }
+      for (int k=IB[i]; k<ID[i]; ++k)
+      {
+         int j = JB[k];
+         const DenseMatrix &L_ij = AB(k);
+         yj.SetDataAndSize(&y[j*block_size], block_size);
+         // y_i = y_i - L_ij*y_j
+         L_ij.AddMult_a(-1.0, yj, yi);
+      }
+   }
+   // Backward substitution to solve Ux = y
+   for (int i=nblockrows-1; i >= 0; --i)
+   {
+      xi.SetDataAndSize(&x[P[i]*block_size], block_size);
+      for (int ib=0; ib<block_size; ++ib)
+      {
+         xi[ib] = y[ib + i*block_size];
+      }
+      for (int k=ID[i]+1; k<IB[i+1]; ++k)
+      {
+         int j = JB[k];
+         const DenseMatrix &U_ij = AB(k);
+         xj.SetDataAndSize(&x[P[j]*block_size], block_size);
+         // x_i = x_i - U_ij*x_j
+         U_ij.AddMult_a(-1.0, xj, xi);
+      }
+      LUFactors A_ii_inv(&DB(0,0,i), &ipiv[i*block_size]);
+      // x_i = D_ii^{-1} x_i
+      A_ii_inv.Solve(block_size, 1, xi);
+   }
+}
+
 #ifdef MFEM_USE_SUITESPARSE
 
 void UMFPackSolver::Init()

linalg/solvers.hpp

@@ -13,7 +13,7 @@
 #define MFEM_SOLVERS
 
 #include "../config/config.hpp"
-#include "operator.hpp"
+#include "densemat.hpp"
 
 #ifdef MFEM_USE_MPI
 #include <mpi.h>
@@ -473,6 +473,106 @@ public:
    virtual void Mult(const Vector &xt, Vector &x) const;
 };
 
+/** Block ILU solver:
+ *  Performs a block ILU(k) approximate factorization with specified block
+ *  size. Currently only k=0 is supported. This is useful as a preconditioner
+ *  for DG-type discretizations, where the system matrix has a natural
+ *  (elemental) block structure.
+ *
+ *  In the case of DG discretizations, the block size should usually be set to
+ *  either ndofs_per_element or vdim*ndofs_per_element (if the finite element
+ *  space has Ordering::byVDIM). The block size must evenly divide the size of
+ *  the matrix.
+ *
+ *  Renumbering the blocks is also supported by specifying a reordering method.
+ *  Currently greedy minimum discarded fill ordering and no reordering are
+ *  supported. Renumbering the blocks can lead to a much better approximate
+ *  factorization.
+ */
+class BlockILU : public Solver
+{
+public:
+
+   /// The reordering method used by the BlockILU factorization.
+   enum class Reordering
+   {
+      MINIMUM_DISCARDED_FILL,
+      NONE
+   };
+
+   /** Create an "empty" BlockILU solver. SetOperator must be called later to
+    *  actually form the factorization
+    */
+   BlockILU(int block_size_,
+            Reordering reordering_ = Reordering::MINIMUM_DISCARDED_FILL,
+            int k_fill_ = 0);
+
+   /** Create a block ILU approximate factorization for the matrix @a op.
+    *  @a op should be of type either SparseMatrix or HypreParMatrix. In the
+    *  case that @a op is a HypreParMatrix, the ILU factorization is performed
+    *  on the diagonal blocks of the parallel decomposition.
+    */
+   BlockILU(Operator &op, int block_size_ = 1,
+            Reordering reordering_ = Reordering::MINIMUM_DISCARDED_FILL,
+            int k_fill_ = 0);
+
+   /** Perform the block ILU factorization for the matrix @a op.
+    *  As in the constructor, @a op must either be a SparseMatrix or
+    *  HypreParMatrix
+    */
+   void SetOperator(const Operator &op);
+
+   /// Solve the system `LUx = b`, where `L` and `U` are the block ILU factors.
+   void Mult(const Vector &b, Vector &x) const;
+
+   /** Get the I array for the block CSR representation of the factorization.
+    *  Similar to SparseMatrix::GetI(). Mostly used for testing.
+    */
+   int *GetBlockI() { return IB.GetData(); }
+
+   /** Get the J array for the block CSR representation of the factorization.
+    *  Similar to SparseMatrix::GetJ(). Mostly used for testing.
+    */
+   int *GetBlockJ() { return JB.GetData(); }
+
+   /** Get the data array for the block CSR representation of the factorization.
+    *  Similar to SparseMatrix::GetData(). Mostly used for testing.
+    */
+   double *GetBlockData() { return AB.Data(); }
+
+private:
+   /// Set up the block CSR structure corresponding to a sparse matrix @a A
+   void CreateBlockPattern(const class SparseMatrix &A);
+
+   /// Perform the block ILU factorization
+   void Factorize();
+
+   int block_size;
+
+   /// Fill level for block ILU(k) factorizations. Only k=0 is supported.
+   int k_fill;
+
+   Reordering reordering;
+
+   /// Temporary vector used in the Mult() function.
+   mutable Vector y;
+
+   /// Permutation and inverse permutation vectors for the block reordering.
+   Array<int> P, Pinv;
+
+   /** Block CSR storage of the factorization. The block upper triangular part
+    *  stores the U factor. The L factor implicitly has identity on the diagonal
+    *  blocks, and the rest of L is given by the strictly block lower triangular
+    *  part.
+    */
+   Array<int> IB, ID, JB;
+   DenseTensor AB;
+
+   /// DB(i) stores the LU factorization of the i'th diagonal block
+   mutable DenseTensor DB;
+   /// Pivot arrays for the LU factorizations given by #DB
+   mutable Array<int> ipiv;
+};
 
 #ifdef MFEM_USE_SUITESPARSE
 

makefile

@@ -117,7 +117,7 @@ EXAMPLE_SUBDIRS = sundials petsc pumi hiop ginkgo
 EXAMPLE_DIRS := examples $(addprefix examples/,$(EXAMPLE_SUBDIRS))
 EXAMPLE_TEST_DIRS := examples
 
-MINIAPP_SUBDIRS = common electromagnetics meshing performance tools nurbs gslib
+MINIAPP_SUBDIRS = common electromagnetics meshing performance tools toys nurbs gslib
 MINIAPP_DIRS := $(addprefix miniapps/,$(MINIAPP_SUBDIRS))
 MINIAPP_TEST_DIRS := $(filter-out %/common,$(MINIAPP_DIRS))
 MINIAPP_USE_COMMON := $(addprefix miniapps/,electromagnetics tools)

mesh/mesh.cpp

@@ -1131,6 +1131,11 @@ void Mesh::AddVertex(const double *x)
    NumOfVertices++;
 }
 
+void Mesh::AddSegment(const int *vi, int attr)
+{
+   elements[NumOfElements++] = new Segment(vi, attr);
+}
+
 void Mesh::AddTri(const int *vi, int attr)
 {
    elements[NumOfElements++] = new Triangle(vi, attr);

mesh/mesh.hpp

@@ -490,6 +490,7 @@ public:
    Element *NewElement(int geom);
 
    void AddVertex(const double *);
+   void AddSegment(const int *vi, int attr = 1);
    void AddTri(const int *vi, int attr = 1);
    void AddTriangle(const int *vi, int attr = 1);
    void AddQuad(const int *vi, int attr = 1);
@@ -555,7 +556,7 @@ public:
        Mesh vertices or nodes are set. */
    virtual void Finalize(bool refine = false, bool fix_orientation = false);
 
-   void SetAttributes();
+   virtual void SetAttributes();
 
 #ifdef MFEM_USE_GECKO
    /** This is our integration with the Gecko library.  This will call the

mesh/nurbs.cpp

@@ -13,6 +13,7 @@
 #include "../fem/fem.hpp"
 #include "../general/text.hpp"
 
+#include <fstream>
 #include <algorithm>
 #if defined(_MSC_VER) && (_MSC_VER < 1800)
 #include <float.h>
@@ -128,6 +129,33 @@ void KnotVector::Print(std::ostream &out) const
    knot.Print(out, knot.Size());
 }
 
+
+void KnotVector::PrintFunctions(std::ostream &out, int samples) const
+{
+   Vector shape(Order+1);
+
+   double x, dx = 1.0/double (samples - 1);
+
+   for (int i = 0; i <GetNE() ; i++)
+   {
+      for (int j = 0; j <samples; j++)
+      {
+         x =j*dx;
+         out<< x + i;
+
+         CalcShape ( shape, i, x);
+         for (int d = 0; d < Order+1; d++) { out<<"\t"<<shape[d]; }
+
+         CalcDShape ( shape, i, x);
+         for (int d = 0; d < Order+1; d++) { out<<"\t"<<shape[d]; }
+
+         CalcD2Shape ( shape, i, x);
+         for (int d = 0; d < Order+1; d++) { out<<"\t"<<shape[d]; }
+         out<<endl;
+      }
+   }
+}
+
 // Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
 void KnotVector::CalcShape(Vector &shape, int i, double xi) const
 {
@@ -211,6 +239,109 @@ void KnotVector::CalcDShape(Vector &grad, int i, double xi) const
    }
 }
 
+// Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
+void KnotVector::CalcDnShape(Vector &gradn, int n, int i, double xi) const
+{
+   int    p = Order, rk, pk, j1, j2,r,j,k;
+   int    ip = (i >= 0) ? (i + p) : (-1 - i + p);
+   double u = getKnotLocation((i >= 0) ? xi : 1. - xi, ip);
+   double temp, saved, d;
+   double a[2][MaxOrder+1],ndu[MaxOrder+1][MaxOrder+1], left[MaxOrder+1],
+          right[MaxOrder+1];
+
+#ifdef MFEM_DEBUG
+   if (p > MaxOrder)
+   {
+      mfem_error("KnotVector::CalcDnShape : Order > MaxOrder!");
+   }
+#endif
+
+   ndu[0][0] = 1.0;
+   for (j = 1; j <= p; j++)
+   {
+      left[j] = u - knot(ip-j+1);
+      right[j] = knot(ip+j)- u;
+
+      saved = 0.0;
+      for (r = 0; r < j; r++)
+      {
+         ndu[j][r] = right[r+1] + left[j-r];
+         temp = ndu[r][j-1]/ndu[j][r];
+         ndu[r][j] = saved + right[r+1]*temp;
+         saved = left[j-r]*temp;
+      }
+      ndu[j][j] = saved;
+   }
+
+   for (r = 0; r <= p; r++)
+   {
+      int s1 = 0;
+      int s2 = 1;
+      a[0][0] = 1.0;
+      for (k = 1; k <= n; k++)
+      {
+         d = 0.0;
+         rk = r-k;
+         pk = p-k;
+         if (r >= k)
+         {
+            a[s2][0] = a[s1][0]/ndu[pk+1][rk];
+            d = a[s2][0]*ndu[rk][pk];
+         }
+
+         if (rk >= -1)
+         {
+            j1 = 1;
+         }
+         else
+         {
+            j1 = -rk;
+         }
+
+         if (r-1<= pk)
+         {
+            j2 = k-1;
+         }
+         else
+         {
+            j2 = p-r;
+         }
+
+         for (j = j1; j <= j2; j++)
+         {
+            a[s2][j] = (a[s1][j] - a[s1][j-1])/ndu[pk+1][rk+j];
+            d += a[s2][j]*ndu[rk+j][pk];
+         }
+
+         if (r <= pk)
+         {
+            a[s2][k] = - a[s1][k-1]/ndu[pk+1][r];
+            d += a[s2][j]*ndu[rk+j][pk];
+         }
+         gradn[r] = d;
+         j = s1;
+         s1 = s2;
+         s2 = j;
+      }
+   }
+
+   if (i >= 0)
+   {
+      u = (knot(ip+1) - knot(ip));
+   }
+   else
+   {
+      u = (knot(ip) - knot(ip+1));
+   }
+
+   temp = p*u;
+   for (k = 1; k <= n-1; k++) { temp *= (p-k)*u; }
+
+   for (j = 0; j <= p; j++) { gradn[j] *= temp; }
+
+}
+
+
 int KnotVector::findKnotSpan(double u) const
 {
    int low, mid, high;
@@ -273,7 +404,6 @@ void KnotVector::Difference(const KnotVector &kv, Vector &diff) const
    }
 }
 
-
 void NURBSPatch::init(int dim_)
 {
    Dim = dim_;
@@ -1633,6 +1763,19 @@ void NURBSExtension::PrintCharacteristics(std::ostream &out) const
    out << endl;
 }
 
+void NURBSExtension::PrintFunctions(const char *basename, int samples) const
+{
+   std::ofstream out;
+   for (int i = 0; i < NumOfKnotVectors; i++)
+   {
+      std::ostringstream filename;
+      filename << basename<<"_"<<i<<".dat";
+      out.open(filename.str().c_str());
+      knotVectors[i]->PrintFunctions(out,samples);
+      out.close();
+   }
+}
+
 void NURBSExtension::InitDofMap()
 {
    master.SetSize(0);
@@ -1672,6 +1815,8 @@ void NURBSExtension::ConnectBoundaries()
    }
 
    // Finalize
+   if (el_dof) { delete el_dof; }
+   if (bel_dof) { delete bel_dof; }
    GenerateElementDofTable();
    GenerateBdrElementDofTable();
 }

mesh/nurbs.hpp

@@ -61,8 +61,11 @@ public:
 
    int findKnotSpan(double u) const;
 
-   void CalcShape (Vector &shape, int i, double xi) const;
-   void CalcDShape(Vector &grad,  int i, double xi) const;
+   void CalcShape  (Vector &shape, int i, double xi) const;
+   void CalcDShape (Vector &grad,  int i, double xi) const;
+   void CalcDnShape(Vector &gradn, int n, int i, double xi) const;
+   void CalcD2Shape(Vector &grad2, int i, double xi) const
+   { CalcDnShape(grad2, 2, i, xi); }
 
    void Difference(const KnotVector &kv, Vector &diff) const;
    void UniformRefinement(Vector &newknots) const;
@@ -74,6 +77,8 @@ public:
 
    void Print(std::ostream &out) const;
 
+   void PrintFunctions(std::ostream &out, int samples=11) const;
+
    /// Destroys KnotVector
    ~KnotVector() { }
 
@@ -329,6 +334,7 @@ public:
    // Print functions
    void Print(std::ostream &out) const;
    void PrintCharacteristics(std::ostream &out) const;
+   void PrintFunctions(const char *filename, int samples=11) const;
 
    // Meta data functions
    int Dimension() const { return patchTopo->Dimension(); }

mesh/pmesh.cpp

@@ -1292,6 +1292,66 @@ void ParMesh::Finalize(bool refine, bool fix_orientation)
    FinalizeParTopo();
 }
 
+void ParMesh::DistributeAttributes(Array<int> &attr)
+{
+   // Determine the largest attribute number across all processors
+   int max_attr = attr.Max();
+   int glb_max_attr = -1;
+   MPI_Allreduce(&max_attr, &glb_max_attr, 1, MPI_INT, MPI_MAX, MyComm);
+
+   // Create marker arrays to indicate which attributes are present
+   // assuming attribute numbers are in the range [1,glb_max_attr].
+   bool * attr_marker = new bool[glb_max_attr];
+   bool * glb_attr_marker = new bool[glb_max_attr];
+   for (int i=0; i<glb_max_attr; i++)
+   {
+      attr_marker[i] = false;
+   }
+   for (int i=0; i<attr.Size(); i++)
+   {
+      attr_marker[attr[i] - 1] = true;
+   }
+   MPI_Allreduce(attr_marker, glb_attr_marker, glb_max_attr,
+                 MPI_C_BOOL, MPI_LOR, MyComm);
+   delete [] attr_marker;
+
+   // Translate from the marker array to a unique, sorted list of attributes
+   Array<int> glb_attr;
+   glb_attr.SetSize(glb_max_attr);
+   glb_attr = glb_max_attr;
+   int o = 0;
+   for (int i=0; i<glb_max_attr; i++)
+   {
+      if (glb_attr_marker[i])
+      {
+         glb_attr[o++] = i + 1;
+      }
+   }
+   delete [] glb_attr_marker;
+
+   glb_attr.Sort();
+   glb_attr.Unique();
+   glb_attr.Copy(attr);
+}
+
+void ParMesh::SetAttributes()
+{
+   // Determine the attributes occurring in local interior and boundary elements
+   Mesh::SetAttributes();
+
+   DistributeAttributes(bdr_attributes);
+   if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0)
+   {
+      MFEM_WARNING("Non-positive boundary element attributes found!");
+   }
+
+   DistributeAttributes(attributes);
+   if (attributes.Size() > 0 && attributes[0] <= 0)
+   {
+      MFEM_WARNING("Non-positive element attributes found!");
+   }
+}
+
 void ParMesh::GroupEdge(int group, int i, int &edge, int &o)
 {
    int sedge = group_sedge.GetRow(group-1)[i];

mesh/pmesh.hpp

@@ -193,6 +193,8 @@ protected:
    void BuildSharedVertMapping(int nvert, const Table* vert_element,
                                const Array<int> &vert_global_local);
 
+   /// Ensure that bdr_attributes and attributes agree across processors
+   void DistributeAttributes(Array<int> &attr);
 
 public:
    /** Copy constructor. Performs a deep copy of (almost) all data, so that the
@@ -223,6 +225,8 @@ public:
 
    virtual void Finalize(bool refine = false, bool fix_orientation = false);
 
+   virtual void SetAttributes();
+
    MPI_Comm GetComm() const { return MyComm; }
    int GetNRanks() const { return NRanks; }
    int GetMyRank() const { return MyRank; }

miniapps/CMakeLists.txt

@@ -19,5 +19,6 @@ add_subdirectory(electromagnetics)
 add_subdirectory(meshing)
 add_subdirectory(performance)
 add_subdirectory(tools)
+add_subdirectory(toys)
 add_subdirectory(nurbs)
 add_subdirectory(gslib)

miniapps/common/mesh_extras.cpp

@@ -116,6 +116,83 @@ ElementMeshStream::ElementMeshStream(Element::Type e)
 
 }
 
+void
+MergeMeshNodes(Mesh * mesh, int logging)
+{
+   int dim  = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   double h_min, h_max, k_min, k_max;
+   mesh->GetCharacteristics(h_min, h_max, k_min, k_max);
+
+   // Set tolerance for merging vertices
+   double tol = 1.0e-8 * h_min;
+
+   if ( logging > 0 )
+      cout << "Euler Number of Initial Mesh:  "
+           << ((dim==3)?mesh->EulerNumber() :
+               ((dim==2)?mesh->EulerNumber2D() :
+                mesh->GetNV() - mesh->GetNE())) << endl;
+
+   vector<int> v2v(mesh->GetNV());
+
+   Vector vd(sdim);
+
+   for (int i = 0; i < mesh->GetNV(); i++)
+   {
+      Vector vi(mesh->GetVertex(i), sdim);
+
+      v2v[i] = -1;
+
+      for (int j = 0; j < i; j++)
+      {
+         Vector vj(mesh->GetVertex(j), sdim);
+         add(vi, -1.0, vj, vd);
+
+         if ( vd.Norml2() < tol )
+         {
+            v2v[i] = j;
+            break;
+         }
+      }
+      if ( v2v[i] < 0 ) { v2v[i] = i; }
+   }
+
+   // renumber elements
+   for (int i = 0; i < mesh->GetNE(); i++)
+   {
+      Element *el = mesh->GetElement(i);
+      int *v = el->GetVertices();
+      int nv = el->GetNVertices();
+      for (int j = 0; j < nv; j++)
+      {
+         v[j] = v2v[v[j]];
+      }
+   }
+   // renumber boundary elements
+   for (int i = 0; i < mesh->GetNBE(); i++)
+   {
+      Element *el = mesh->GetBdrElement(i);
+      int *v = el->GetVertices();
+      int nv = el->GetNVertices();
+      for (int j = 0; j < nv; j++)
+      {
+         v[j] = v2v[v[j]];
+      }
+   }
+
+   mesh->RemoveUnusedVertices();
+
+   if ( logging > 0 )
+   {
+      cout << "Euler Number of Final Mesh:    "
+           << ((dim==3) ? mesh->EulerNumber() :
+               ((dim==2) ? mesh->EulerNumber2D() :
+                mesh->GetNV() - mesh->GetNE()))
+           << endl;
+   }
+}
+
 } // namespace common
 
 } // namespace mfem

miniapps/common/mesh_extras.hpp

@@ -27,6 +27,9 @@ public:
    ElementMeshStream(Element::Type e);
 };
 
+/// Merges vertices which lie at the same location
+void MergeMeshNodes(Mesh * mesh, int logging);
+
 } // namespace common
 
 } // namespace mfem

miniapps/electromagnetics/makefile

@@ -90,6 +90,10 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 volta-test-par: volta
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
 	-maxit 2 -dbcs 1 -dbcg -ds '0.0 0.0 0.0 0.2 8.0')
+volta-1-test-par: volta
+	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
+	-maxit 2 -m ../../data/square-disc.mesh \
+	-dbcs '1 2 3 4 5 6 7 8' -dbcv '0 0 0 0 1 1 1 1')
 tesla-test-par: tesla
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
 	-maxit 2 -cr '0 0 -0.2 0 0 0.2 0.2 0.4 1')

miniapps/electromagnetics/volta_solver.cpp

@@ -89,10 +89,7 @@ VoltaSolver::VoltaSolver(ParMesh & pmesh, int order,
    ess_bdr_ = 0;   // Deselect all outer surfaces
    for (int i=0; i<dbcs_->Size(); i++)
    {
-      if ((*dbcs_)[i] <= ess_bdr_.Size())
-      {
-         ess_bdr_[(*dbcs_)[i]-1] = 1;
-      }
+      ess_bdr_[(*dbcs_)[i]-1] = 1;
    }
 
    // Setup various coefficients

miniapps/meshing/mesh-explorer.cpp

@@ -118,6 +118,115 @@ Mesh *read_par_mesh(int np, const char *mesh_prefix)
    return mesh;
 }
 
+// Given a 3D mesh, produce a 2D mesh consisting of its boundary elements.
+Mesh *skin_mesh(Mesh *mesh)
+{
+   // Determine mapping from vertex to boundary vertex
+   Array<int> v2v(mesh->GetNV());
+   v2v = -1;
+   for (int i = 0; i < mesh->GetNBE(); i++)
+   {
+      Element *el = mesh->GetBdrElement(i);
+      int *v = el->GetVertices();
+      int nv = el->GetNVertices();
+      for (int j = 0; j < nv; j++)
+      {
+         v2v[v[j]] = 0;
+      }
+   }
+   int nbvt = 0;
+   for (int i = 0; i < v2v.Size(); i++)
+   {
+      if (v2v[i] == 0)
+      {
+         v2v[i] = nbvt++;
+      }
+   }
+
+   // Create a new mesh for the boundary
+   Mesh * bmesh = new Mesh(mesh->Dimension() - 1, nbvt, mesh->GetNBE(),
+                           0, mesh->SpaceDimension());
+
+   // Copy vertices to the boundary mesh
+   nbvt = 0;
+   for (int i = 0; i < v2v.Size(); i++)
+   {
+      if (v2v[i] >= 0)
+      {
+         double *c = mesh->GetVertex(i);
+         bmesh->AddVertex(c);
+         nbvt++;
+      }
+   }
+
+   // Copy elements to the boundary mesh
+   int bv[4];
+   for (int i = 0; i < mesh->GetNBE(); i++)
+   {
+      Element *el = mesh->GetBdrElement(i);
+      int *v = el->GetVertices();
+      int nv = el->GetNVertices();
+
+      for (int j = 0; j < nv; j++)
+      {
+         bv[j] = v2v[v[j]];
+      }
+
+      switch (el->GetGeometryType())
+      {
+         case Geometry::SEGMENT:
+            bmesh->AddSegment(bv, el->GetAttribute());
+            break;
+         case Geometry::TRIANGLE:
+            bmesh->AddTriangle(bv, el->GetAttribute());
+            break;
+         case Geometry::SQUARE:
+            bmesh->AddQuad(bv, el->GetAttribute());
+            break;
+         default:
+            break; /// This should not happen
+      }
+
+   }
+   bmesh->FinalizeTopology();
+
+   // Copy GridFunction describing nodes if present
+   if (mesh->GetNodes())
+   {
+      FiniteElementSpace *fes = mesh->GetNodes()->FESpace();
+      const FiniteElementCollection *fec = fes->FEColl();
+      if (dynamic_cast<const H1_FECollection*>(fec))
+      {
+         FiniteElementCollection *fec_copy =
+            FiniteElementCollection::New(fec->Name());
+         FiniteElementSpace *fes_copy =
+            new FiniteElementSpace(*fes, bmesh, fec_copy);
+         GridFunction *bdr_nodes = new GridFunction(fes_copy);
+         bdr_nodes->MakeOwner(fec_copy);
+
+         bmesh->NewNodes(*bdr_nodes, true);
+
+         Array<int> vdofs;
+         Array<int> bvdofs;
+         Vector v;
+         for (int i=0; i<mesh->GetNBE(); i++)
+         {
+            fes->GetBdrElementVDofs(i, vdofs);
+            mesh->GetNodes()->GetSubVector(vdofs, v);
+
+            fes_copy->GetElementVDofs(i, bvdofs);
+            bdr_nodes->SetSubVector(bvdofs, v);
+         }
+      }
+      else
+      {
+         cout << "\nDiscontinuous nodes not yet supported" << endl;
+      }
+   }
+
+   return bmesh;
+}
+
 int main (int argc, char *argv[])
 {
    int np = 0;
@@ -150,6 +259,7 @@ int main (int argc, char *argv[])
    args.PrintOptions(cout);
 
    Mesh *mesh;
+   Mesh *bdr_mesh = NULL;
    if (np <= 0)
    {
       mesh = new Mesh(mesh_file, 1, refine);
@@ -165,6 +275,7 @@ int main (int argc, char *argv[])
    int dim  = mesh->Dimension();
    int sdim = mesh->SpaceDimension();
 
+   FiniteElementCollection *bdr_attr_fec = NULL;
    FiniteElementCollection *attr_fec;
    if (dim == 2)
    {
@@ -172,6 +283,7 @@ int main (int argc, char *argv[])
    }
    else
    {
+      bdr_attr_fec = new Const2DFECollection;
       attr_fec = new Const3DFECollection;
    }
 
@@ -528,9 +640,12 @@ int main (int argc, char *argv[])
       if (mk == 'm' || mk == 'b' || mk == 'e' || mk == 'v' || mk == 'h' ||
           mk == 'k' || mk == 'p')
       {
+         Array<int> bdr_part;
          Array<int> part(mesh->GetNE());
+         FiniteElementSpace *bdr_attr_fespace = NULL;
          FiniteElementSpace *attr_fespace =
             new FiniteElementSpace(mesh, attr_fec);
+         GridFunction bdr_attr;
          GridFunction attr(attr_fespace);
 
          if (mk == 'm')
@@ -541,7 +656,28 @@ int main (int argc, char *argv[])
             }
          }
 
-         if (mk == 'b' || mk == 'v')
+         if (mk == 'b')
+         {
+            if (dim == 3)
+            {
+               delete bdr_mesh;
+               bdr_mesh = skin_mesh(mesh);
+               bdr_attr_fespace =
+                  new FiniteElementSpace(bdr_mesh, bdr_attr_fec);
+               bdr_part.SetSize(bdr_mesh->GetNE());
+               bdr_attr.SetSpace(bdr_attr_fespace);
+               for (int i = 0; i < bdr_mesh->GetNE(); i++)
+               {
+                  bdr_part[i] = (bdr_attr(i) = bdr_mesh->GetAttribute(i)) - 1;
+               }
+            }
+            else
+            {
+               attr = 1.0;
+            }
+         }
+
+         if (mk == 'v')
          {
             attr = 1.0;
          }
@@ -760,10 +896,18 @@ int main (int argc, char *argv[])
             else
             {
                sol_sock << "fem3d_gf_data_keys\n";
-               if (mk == 'b' || mk == 'v' || mk == 'h' || mk == 'k')
+               if (mk == 'v' || mk == 'h' || mk == 'k')
                {
                   mesh->Print(sol_sock);
                }
+               else if (mk == 'b')
+               {
+                  bdr_mesh->Print(sol_sock);
+                  bdr_attr.Save(sol_sock);
+                  sol_sock << "mcaaA";
+                  // Switch to a discrete color scale
+                  sol_sock << "pppppp" << "pppppp" << "pppppp";
+               }
                else
                {
                   // NURBS meshes do not support PrintWithPartitioning
@@ -780,15 +924,18 @@ int main (int argc, char *argv[])
                      mesh->PrintWithPartitioning(part, sol_sock);
                   }
                }
-               attr.Save(sol_sock);
-               sol_sock << "maaA";
-               if (mk == 'v')
+               if (mk != 'b')
                {
-                  sol_sock << "aa";
-               }
-               else
-               {
-                  sol_sock << "\n";
+                  attr.Save(sol_sock);
+                  sol_sock << "maaA";
+                  if (mk == 'v')
+                  {
+                     sol_sock << "aa";
+                  }
+                  else
+                  {
+                     sol_sock << "\n";
+                  }
                }
             }
             sol_sock << flush;
@@ -799,6 +946,7 @@ int main (int argc, char *argv[])
                  << vishost << ':' << visport << endl;
          }
          delete attr_fespace;
+         delete bdr_attr_fespace;
       }
 
       if (mk == 'S')
@@ -832,7 +980,9 @@ int main (int argc, char *argv[])
 
    }
 
+   delete bdr_attr_fec;
    delete attr_fec;
+   delete bdr_mesh;
    delete mesh;
    return 0;
 }

miniapps/meshing/shaper.cpp

@@ -19,6 +19,10 @@
 // for example in the case when the interface is too complex to describe without
 // local refinement. Both conforming and non-conforming refinements are supported.
 //
+// Two additional versions of this miniapp can be found in the miniapps/toys
+// directory: Mandel uses the Shaper algorithm for fractal visualization, while
+// Mondrian convert an image to an AMR mesh suitable for MFEM computations.
+//
 // Compile with: make shaper
 //
 // Sample runs:  shaper

miniapps/nurbs/CMakeLists.txt

@@ -9,8 +9,20 @@
 # terms of the GNU Lesser General Public License (as published by the Free
 # Software Foundation) version 2.1 dated February 1999.
 
+configure_file(${PROJECT_SOURCE_DIR}/miniapps/nurbs/square-nurbs.mesh
+               ${PROJECT_BINARY_DIR}/miniapps/nurbs/square-nurbs.mesh
+               COPYONLY)
+
+configure_file(${PROJECT_SOURCE_DIR}/miniapps/nurbs/cube-nurbs.mesh
+               ${PROJECT_BINARY_DIR}/miniapps/nurbs/cube-nurbs.mesh
+               COPYONLY)
+
+configure_file(${PROJECT_SOURCE_DIR}/miniapps/nurbs/pipe-nurbs-2d.mesh
+               ${PROJECT_BINARY_DIR}/miniapps/nurbs/pipe-nurbs-2d.mesh
+               COPYONLY)
+
 add_mfem_miniapp(nurbs_ex1
-  MAIN ex1.cpp
+  MAIN nurbs_ex1.cpp
   LIBRARIES mfem)
 
 add_test(NAME nurbs_ex1_ser
@@ -20,9 +32,14 @@ add_test(NAME nurbs_ex1_per_ser
   COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
   -m ../../data/beam-hex-nurbs.mesh -pm 1 -ps 2)
 
+add_test(NAME nurbs_ex1_lap_ser
+  COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
+  -m pipe-nurbs-2d.mesh -o 2 -no-ibp)
+
+
 if (MFEM_USE_MPI)
   add_mfem_miniapp(nurbs_ex1p
-    MAIN ex1p.cpp
+    MAIN nurbs_ex1p.cpp
     LIBRARIES mfem)
 
   add_test(NAME nurbs_ex1p_np=4
@@ -30,8 +47,13 @@ if (MFEM_USE_MPI)
     ${MPIEXEC_PREFLAGS} $<TARGET_FILE:nurbs_ex1p> -no-vis
     ${MPIEXEC_POSTFLAGS})
 
+  add_test(NAME nurbs_ex1p_lap_np=4
+    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+    ${MPIEXEC_PREFLAGS} $<TARGET_FILE:nurbs_ex1p> -no-vis -m pipe-nurbs-2d.mesh -o 2 -no-ibp
+    ${MPIEXEC_POSTFLAGS})
+
   add_mfem_miniapp(nurbs_ex11p
-    MAIN ex11p.cpp
+    MAIN nurbs_ex11p.cpp
     LIBRARIES mfem)
 
   add_test(NAME nurbs_ex11p_np=4

miniapps/nurbs/cube-nurbs.mesh

@@ -0,0 +1,73 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+3
+
+elements
+1
+1 5 0 1 2 3 4 5 6 7
+
+boundary
+6
+1 3 0 1 2 3
+1 3 4 5 6 7
+1 3 0 1 5 4
+1 3 1 2 6 5
+1 3 2 3 7 6
+1 3 3 0 4 7
+
+edges
+12
+0 0 1
+0 3 2
+0 4 5
+0 7 6
+1 0 3
+1 1 2
+1 4 7
+1 5 6
+2 0 4
+2 1 5
+2 2 6
+2 3 7
+
+vertices
+8
+
+knotvectors
+3
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+weights
+1
+1
+1
+1
+1
+1
+1
+1
+
+FiniteElementSpace
+FiniteElementCollection: NURBS1
+VDim: 3
+Ordering: 1
+
+0 0 0
+1 0 0
+1 1 0
+0 1 0
+0 0 1
+1 0 1
+1 1 1
+0 1 1

miniapps/nurbs/makefile

@@ -21,8 +21,8 @@ CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_MINIAPPS = ex1
-PAR_MINIAPPS = ex1p ex11p
+SEQ_MINIAPPS = nurbs_ex1
+PAR_MINIAPPS = nurbs_ex1p nurbs_ex11p
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
 else

miniapps/nurbs/nurbs_ex1.cpp

@@ -1,25 +1,14 @@
 //                          MFEM Example 1 - NURBS Version
 //
-// Compile with: make ex1
+// Compile with: make nurbs_ex1
 //
-// Sample runs:  ex1 -m ../../data/square-disc.mesh
-//               ex1 -m ../../data/star.mesh
-//               ex1 -m ../../data/escher.mesh
-//               ex1 -m ../../data/fichera.mesh
-//               ex1 -m ../../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../../data/star-surf.mesh
-//               ex1 -m ../../data/square-disc-surf.mesh
-//               ex1 -m ../../data/inline-segment.mesh
-//               ex1 -m ../../data/amr-quad.mesh
-//               ex1 -m ../../data/amr-hex.mesh
-//               ex1 -m ../../data/fichera-amr.mesh
-//               ex1 -m ../../data/mobius-strip.mesh
-//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
-//               ex1 -m ../../data/beam-hex-nurbs.mesh -pm 1 -ps 2
+// Sample runs:  nurbs_ex1 -m square-nurbs.mesh -o 2 -no-ibp
+//               nurbs_ex1 -m cube-nurbs.mesh -o 2 -no-ibp
+//               nurbs_ex1 -m pipe-nurbs-2d.mesh -o 2 -no-ibp
+//               nurbs_ex1 -m ../../data/square-disc-nurbs.mesh -o -1
+//               nurbs_ex1 -m ../../data/disc-nurbs.mesh -o -1
+//               nurbs_ex1 -m ../../data/pipe-nurbs.mesh -o -1
+//               nurbs_ex1 -m ../../data/beam-hex-nurbs.mesh -pm 1 -ps 2
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -43,6 +32,93 @@
 using namespace std;
 using namespace mfem;
 
+
+/** Class for integrating the bilinear form a(u,v) := (Q Laplace u, v) where Q
+    can be a scalar coefficient. */
+class Diffusion2Integrator: public BilinearFormIntegrator
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   Vector shape,laplace;
+#endif
+   Coefficient *Q;
+
+public:
+   /// Construct a diffusion integrator with coefficient Q = 1
+   Diffusion2Integrator() { Q = NULL; }
+
+   /// Construct a diffusion integrator with a scalar coefficient q
+   Diffusion2Integrator (Coefficient &q) : Q(&q) { }
+
+   /** Given a particular Finite Element
+       computes the element stiffness matrix elmat. */
+   virtual void AssembleElementMatrix(const FiniteElement &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat)
+   {
+      int nd = el.GetDof();
+      int dim = el.GetDim();
+      double w;
+
+#ifdef MFEM_THREAD_SAFE
+      Vector shape[nd];
+      Vector laplace(nd);
+#else
+      shape.SetSize(nd);
+      laplace.SetSize(nd);
+#endif
+      elmat.SetSize(nd);
+
+      const IntegrationRule *ir = IntRule;
+      if (ir == NULL)
+      {
+         int order;
+         if (el.Space() == FunctionSpace::Pk)
+         {
+            order = 2*el.GetOrder() - 2;
+         }
+         else
+         {
+            order = 2*el.GetOrder() + dim - 1;
+         }
+
+         if (el.Space() == FunctionSpace::rQk)
+         {
+            ir = &RefinedIntRules.Get(el.GetGeomType(),order);
+         }
+         else
+         {
+            ir = &IntRules.Get(el.GetGeomType(),order);
+         }
+      }
+
+      elmat = 0.0;
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         Trans.SetIntPoint(&ip);
+         w = -ip.weight * Trans.Weight();
+
+         el.CalcShape(ip, shape);
+         el.CalcPhysLaplacian(Trans, laplace);
+
+         if (Q)
+         {
+            w *= Q->Eval(Trans, ip);
+         }
+
+         for (int j = 0; j < nd; j++)
+         {
+            for (int i = 0; i < nd; i++)
+            {
+               elmat(i, j) += w*shape(i)*laplace(j);
+            }
+         }
+      }
+   }
+
+};
+
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options.
@@ -52,6 +128,7 @@ int main(int argc, char *argv[])
    Array<int> slave(0);
    bool static_cond = false;
    bool visualization = 1;
+   bool ibp = 1;
    Array<int> order(1);
    order[0] = 1;
 
@@ -67,6 +144,9 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) or -1 for"
                   " isoparametric space.");
+   args.AddOption(&ibp, "-ibp", "--ibp", "-no-ibp",
+                  "--no-ibp",
+                  "Selects the standard weak form (IBP) or the nonstandard (NO-IBP).");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -92,7 +172,7 @@ int main(int argc, char *argv[])
    //    elements.
    {
       int ref_levels =
-         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+         (int)floor(log(5000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();
@@ -160,10 +240,36 @@ int main(int argc, char *argv[])
       fec = new H1_FECollection(abs(order[0]), dim);
       own_fec = 1;
    }
+
    FiniteElementSpace *fespace = new FiniteElementSpace(mesh, NURBSext, fec);
    cout << "Number of finite element unknowns: "
         << fespace->GetTrueVSize() << endl;
 
+   if (!ibp)
+   {
+      if (!mesh->NURBSext)
+      {
+         cout << "No integration by parts requires a NURBS mesh."<< endl;
+         return 2;
+      }
+      if (mesh->NURBSext->GetNP()>1)
+      {
+         cout << "No integration by parts requires a NURBS mesh, with only 1 patch."<<
+              endl;
+         cout << "A C_1 discretisation is required."<< endl;
+         cout << "Currently only C_0 multipatch coupling implemented."<< endl;
+         return 3;
+      }
+      if (order[0]<2)
+      {
+         cout << "No integration by parts requires at least quadratic NURBS."<< endl;
+         cout << "A C_1 discretisation is required."<< endl;
+         return 4;
+      }
+   }
+
+
+
    // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
    //    In this example, the boundary conditions are defined by marking all
    //    the boundary attributes from the mesh as essential (Dirichlet) and
@@ -200,7 +306,14 @@ int main(int argc, char *argv[])
    //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //    domain integrator.
    BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (ibp)
+   {
+      a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   }
+   else
+   {
+      a->AddDomainIntegrator(new Diffusion2Integrator(one));
+   }
 
    // 9. Assemble the bilinear form and the corresponding linear system,
    //    applying any necessary transformations such as: eliminating boundary
@@ -216,7 +329,7 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A.Height() << endl;
 
 #ifndef MFEM_USE_SUITESPARSE
-   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   // 10. Define a simple Jacobi preconditioner and use it to
    //     solve the system A X = B with PCG.
    GSSmoother M(A);
    PCG(A, M, B, X, 1, 200, 1e-12, 0.0);

miniapps/nurbs/nurbs_ex11p.cpp

@@ -1,24 +1,24 @@
 //                    MFEM Example 11 - Parallel NURBS Version
 //
-// Compile with: make ex11p
+// Compile with: make nurbs_ex11p
 //
-// Sample runs:  mpirun -np 4 ex11p -m ../../data/square-disc.mesh
-//               mpirun -np 4 ex11p -m ../../data/star.mesh
-//               mpirun -np 4 ex11p -m ../../data/escher.mesh
-//               mpirun -np 4 ex11p -m ../../data/fichera.mesh
-//               mpirun -np 4 ex11p -m ../../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex11p -m ../../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex11p -m ../../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex11p -m ../../data/disc-nurbs.mesh -o -1 -n 20
-//               mpirun -np 4 ex11p -m ../../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex11p -m ../../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex11p -m ../../data/star-surf.mesh
-//               mpirun -np 4 ex11p -m ../../data/square-disc-surf.mesh
-//               mpirun -np 4 ex11p -m ../../data/inline-segment.mesh
-//               mpirun -np 4 ex11p -m ../../data/amr-quad.mesh
-//               mpirun -np 4 ex11p -m ../../data/amr-hex.mesh
-//               mpirun -np 4 ex11p -m ../../data/mobius-strip.mesh -n 8
-//               mpirun -np 4 ex11p -m ../../data/klein-bottle.mesh -n 10
+// Sample runs:  mpirun -np 4 nurbs_ex11p -m ../../data/square-disc.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/star.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/escher.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/fichera.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 nurbs_ex11p -m ../../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 nurbs_ex11p -m ../../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 nurbs_ex11p -m ../../data/disc-nurbs.mesh -o -1 -n 20
+//               mpirun -np 4 nurbs_ex11p -m ../../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 nurbs_ex11p -m ../../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 nurbs_ex11p -m ../../data/star-surf.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/square-disc-surf.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/inline-segment.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/amr-quad.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/amr-hex.mesh
+//               mpirun -np 4 nurbs_ex11p -m ../../data/mobius-strip.mesh -n 8
+//               mpirun -np 4 nurbs_ex11p -m ../../data/klein-bottle.mesh -n 10
 //
 // Description:  This example code demonstrates the use of MFEM to solve the
 //               eigenvalue problem -Delta u = lambda u with homogeneous

miniapps/nurbs/nurbs_ex1p.cpp

@@ -1,25 +1,31 @@
 //                     MFEM Example 1 - Parallel NURBS Version
 //
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../../data/star.mesh
-//               mpirun -np 4 ex1p -m ../../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh -o -1 -sc
+// Compile with: make nurbs_ex1p
 //
+// Sample runs:  mpirun -np 4 nurbs_ex1p -m ../../data/square-disc.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/star.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/escher.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/fichera.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 nurbs_ex1p -m ../../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 nurbs_ex1p -m ../../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 nurbs_ex1p -m ../../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 nurbs_ex1p -m ../../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 nurbs_ex1p -m ../../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 nurbs_ex1p -m ../../data/star-surf.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/square-disc-surf.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/inline-segment.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/amr-quad.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/amr-hex.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/mobius-strip.mesh
+//               mpirun -np 4 nurbs_ex1p -m ../../data/mobius-strip.mesh -o -1 -sc
+//               mpirun -np 4 nurbs_ex1p -m ../../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 nurbs_ex1p -m ../../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 nurbs_ex1p -m ../../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 nurbs_ex1p -m square-nurbs.mesh -o 2 -no-ibp
+//               mpirun -np 4 nurbs_ex1p -m cube-nurbs.mesh -o 2 -no-ibp
+//               mpirun -np 4 nurbs_ex1p -m pipe-nurbs-2d.mesh -o 2 -no-ibp
+
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
 //               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
@@ -41,6 +47,91 @@
 
 using namespace std;
 using namespace mfem;
+/** Class for integrating the bilinear form a(u,v) := (Q Laplace u, v) where Q
+    can be a scalar coefficient. */
+class Diffusion2Integrator: public BilinearFormIntegrator
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   Vector shape,laplace;
+#endif
+   Coefficient *Q;
+
+public:
+   /// Construct a diffusion integrator with coefficient Q = 1
+   Diffusion2Integrator() { Q = NULL; }
+
+   /// Construct a diffusion integrator with a scalar coefficient q
+   Diffusion2Integrator (Coefficient &q) : Q(&q) { }
+
+   /** Given a particular Finite Element
+       computes the element stiffness matrix elmat. */
+   virtual void AssembleElementMatrix(const FiniteElement &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat)
+   {
+      int nd = el.GetDof();
+      int dim = el.GetDim();
+      double w;
+
+#ifdef MFEM_THREAD_SAFE
+      Vector shape[nd];
+      Vector laplace(nd);
+#else
+      shape.SetSize(nd);
+      laplace.SetSize(nd);
+#endif
+      elmat.SetSize(nd);
+
+      const IntegrationRule *ir = IntRule;
+      if (ir == NULL)
+      {
+         int order;
+         if (el.Space() == FunctionSpace::Pk)
+         {
+            order = 2*el.GetOrder() - 2;
+         }
+         else
+         {
+            order = 2*el.GetOrder() + dim - 1;
+         }
+
+         if (el.Space() == FunctionSpace::rQk)
+         {
+            ir = &RefinedIntRules.Get(el.GetGeomType(),order);
+         }
+         else
+         {
+            ir = &IntRules.Get(el.GetGeomType(),order);
+         }
+      }
+
+      elmat = 0.0;
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         Trans.SetIntPoint(&ip);
+         w = -ip.weight * Trans.Weight();
+
+         el.CalcShape(ip, shape);
+         el.CalcPhysLaplacian(Trans, laplace);
+
+         if (Q)
+         {
+            w *= Q->Eval(Trans, ip);
+         }
+
+         for (int j = 0; j < nd; j++)
+         {
+            for (int i = 0; i < nd; i++)
+            {
+               elmat(i, j) += w*shape(i)*laplace(j);
+            }
+         }
+      }
+   }
+
+};
 
 int main(int argc, char *argv[])
 {
@@ -56,6 +147,7 @@ int main(int argc, char *argv[])
    order[0] = 1;
    bool static_cond = false;
    bool visualization = 1;
+   bool ibp = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -63,6 +155,9 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) or -1 for"
                   " isoparametric space.");
+   args.AddOption(&ibp, "-ibp", "--ibp", "-no-ibp",
+                  "--no-ibp",
+                  "Selects the standard weak form (IBP) or the nonstandard (NO-IBP).");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -107,6 +202,7 @@ int main(int argc, char *argv[])
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
+   if (!pmesh->NURBSext)
    {
       int par_ref_levels = 2;
       for (int l = 0; l < par_ref_levels; l++)
@@ -165,6 +261,29 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
+   if (!ibp)
+   {
+      if (!pmesh->NURBSext)
+      {
+         cout << "No integration by parts requires a NURBS mesh."<< endl;
+         return 2;
+      }
+      if (pmesh->NURBSext->GetNP()>1)
+      {
+         cout << "No integration by parts requires a NURBS mesh, with only 1 patch."<<
+              endl;
+         cout << "A C_1 discretisation is required."<< endl;
+         cout << "Currently only C_0 multipatch coupling implemented."<< endl;
+         return 3;
+      }
+      if (order[0]<2)
+      {
+         cout << "No integration by parts requires at least quadratic NURBS."<< endl;
+         cout << "A C_1 discretisation is required."<< endl;
+         return 4;
+      }
+   }
+
    // 7. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined
    //    by marking all the boundary attributes from the mesh as essential
@@ -195,7 +314,14 @@ int main(int argc, char *argv[])
    //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //     domain integrator.
    ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (ibp)
+   {
+      a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   }
+   else
+   {
+      a->AddDomainIntegrator(new Diffusion2Integrator(one));
+   }
 
    // 11. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel

miniapps/tools/display-basis.cpp

@@ -24,6 +24,8 @@
 // Sample runs:  display-basis
 //               display_basis -e 2 -b 3 -o 3
 //               display-basis -e 5 -b 1 -o 1
+//               display-basis -e 3 -b 7 -o 3
+//               display-basis -e 3 -b 7 -o 5 -only 16
 
 #include "mfem.hpp"
 #include "../common/mfem-common.hpp"
@@ -104,7 +106,7 @@ string mapTypeStr(int mType);
 int update_basis(vector<socketstream*> & sock, const VisWinLayout & vwl,
                  Element::Type e, char bType, int bOrder, int mType,
                  Deformation::DefType dType, const DeformationData & defData,
-                 bool visualization);
+                 bool visualization, int &onlySome);
 
 int main(int argc, char *argv[])
 {
@@ -127,6 +129,7 @@ int main(int argc, char *argv[])
    DeformationData defData;
 
    bool visualization = true;
+   int onlySome = -1;
 
    vector<socketstream*> sock;
 
@@ -137,7 +140,7 @@ int main(int argc, char *argv[])
    args.AddOption(&bInt, "-b", "--basis-type",
                   "Basis Function Type (0-H1, 1-Nedelec, 2-Raviart-Thomas, "
                   "3-L2, 4-Fixed Order Cont.,\n\t5-Gaussian Discontinuous (2D),"
-                  " 6-Crouzeix-Raviart)");
+                  " 6-Crouzeix-Raviart, 7-Serendipity)");
    args.AddOption(&bOrder, "-o", "--order", "Basis function order");
    args.AddOption(&vwl.nx, "-nx", "--num-win-x",
                   "Number of Viz windows in X");
@@ -150,6 +153,8 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&onlySome, "-only", "--onlySome",
+                  "Only view 10 dofs, starting with the specified one.");
    args.Parse();
    if (!args.Good())
    {
@@ -186,6 +191,9 @@ int main(int argc, char *argv[])
       case 6:
          bType = 'c';
          break;
+      case 7:
+         bType = 's';
+         break;
       default:
          bType = 'h';
    }
@@ -203,7 +211,7 @@ int main(int argc, char *argv[])
          cout << "Map Type:              " << mapTypeStr(mType) << endl;
       }
       if ( update_basis(sock, vwl, eType, bType, bOrder, mType,
-                        dType, defData, visualization) )
+                        dType, defData, visualization, onlySome) )
       {
          cerr << "Invalid combination of basis info (try again)" << endl;
       }
@@ -218,7 +226,7 @@ int main(int argc, char *argv[])
            "e) Change Element Type\n"
            "b) Change Basis Type\n";
       if ( bType == 'h' || bType == 'p' || bType == 'n' || bType == 'r' ||
-           bType == 'l' || bType == 'f' || bType == 'g' )
+           bType == 'l' || bType == 'f' || bType == 'g' || bType == 's')
       {
          cout << "o) Change Basis Order\n";
       }
@@ -299,6 +307,7 @@ int main(int argc, char *argv[])
          cout << "p) H1 Positive Finite Element\n";
          if ( elemIs2D(eType) || elemIs3D(eType) )
          {
+            cout << "s) H1 Serendipity Finite Element\n";
             cout << "n) Nedelec Finite Element\n";
             cout << "r) Raviart-Thomas Finite Element\n";
          }
@@ -314,11 +323,11 @@ int main(int argc, char *argv[])
          }
          cout << "enter new basis type --> " << flush;
          cin >> bChar;
-         if ( bChar == 'h' || bChar == 'p' || bChar == 'l' || bChar == 'f' ||
-              ((bChar == 'n' || bChar == 'r') &&
-               (elemIs2D(eType) || elemIs3D(eType))) ||
-              (bChar == 'c' && (elemIs1D(eType) || elemIs2D(eType))) ||
-              (bChar == 'g' && elemIs2D(eType)))
+         if (bChar == 'h' || bChar == 'p' || bChar == 'l' || bChar == 'f' ||
+             bChar == 's' ||
+             ((bChar == 'n' || bChar == 'r') && (elemIs2D(eType) || elemIs3D(eType))) ||
+             (bChar == 'c' && (elemIs1D(eType) || elemIs2D(eType))) ||
+             (bChar == 'g' && elemIs2D(eType)))
          {
             bType = bChar;
             if ( bType == 'h' )
@@ -329,6 +338,10 @@ int main(int argc, char *argv[])
             {
                mType = FiniteElement::VALUE;
             }
+            else if (bType == 's')
+            {
+               mType = FiniteElement::VALUE;
+            }
             else if ( bType == 'n' )
             {
                mType = FiniteElement::H_CURL;
@@ -395,7 +408,7 @@ int main(int argc, char *argv[])
       {
          int oInt = 1;
          int oMin = ( bType == 'h' || bType == 'p' || bType == 'n' ||
-                      bType == 'f' || bType == 'g')?1:0;
+                      bType == 'f' || bType == 'g' || bType == 's')?1:0;
          int oMax = -1;
          switch (bType)
          {
@@ -545,6 +558,8 @@ basisTypeStr(char bType)
          return "Continuous (H1)";
       case 'p':
          return "Continuous Positive (H1)";
+      case 's':
+         return "Continuous Serendipity (H1)";
       case 'n':
          return "Nedelec";
       case 'r':
@@ -573,7 +588,8 @@ bool
 basisIs2D(char bType)
 {
    return bType == 'h' || bType == 'p' || bType == 'n' || bType == 'r' ||
-          bType == 'l' || bType == 'c' || bType == 'f' || bType == 'g';
+          bType == 'l' || bType == 'c' || bType == 'f' || bType == 'g' ||
+          bType == 's';
 }
 
 bool
@@ -683,7 +699,7 @@ int
 update_basis(vector<socketstream*> & sock,  const VisWinLayout & vwl,
              Element::Type e, char bType, int bOrder, int mType,
              Deformation::DefType dType, const DeformationData & defData,
-             bool visualization)
+             bool visualization, int &onlySome)
 {
    bool vec = false;
 
@@ -716,6 +732,17 @@ update_basis(vector<socketstream*> & sock,  const VisWinLayout & vwl,
          FEC = new H1Pos_FECollection(bOrder, dim);
          vec = false;
          break;
+      case 's':
+         if (bOrder == 1)
+         {
+            FEC = new H1_FECollection(bOrder, dim);
+         }
+         else
+         {
+            FEC = new H1Ser_FECollection(bOrder, dim);
+         }
+         vec = false;
+         break;
       case 'n':
          FEC = new ND_FECollection(bOrder, dim);
          vec = true;
@@ -818,8 +845,23 @@ update_basis(vector<socketstream*> & sock,  const VisWinLayout & vwl,
       ref++;
    }
 
-   for (int i=0; i<ndof; i++)
+   int stopAt = ndof;
+   if (ndof > 25 && onlySome == -1)
    {
+      cout << endl;
+      cout << "There are more than 25 windows to open.\n"
+           << "Only showing Dofs 1-10 to avoid crashing.\n"
+           << "Use the option -only N to show Dofs N to N+9 instead.\n";
+      onlySome = 1;
+   }
+   for (int i = 0; i < stopAt; i++)
+   {
+      if (i ==0 && onlySome > 0 && onlySome <ndof)
+      {
+         i = onlySome-1;
+         stopAt = min(ndof,onlySome+9);
+      }
+
       ostringstream oss;
       oss << "DoF " << i + 1;
       if (visualization)

miniapps/toys/CMakeLists.txt

@@ -0,0 +1,36 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+add_mfem_miniapp(toy_automata
+    MAIN automata.cpp LIBRARIES mfem)
+
+add_mfem_miniapp(toy_life
+    MAIN life.cpp LIBRARIES mfem)
+
+add_mfem_miniapp(toy_mandel
+    MAIN mandel.cpp LIBRARIES mfem)
+
+add_mfem_miniapp(toy_rubik
+    MAIN rubik.cpp
+    ${MFEM_MINIAPPS_COMMON_HEADERS}
+    LIBRARIES mfem mfem-common)
+
+add_mfem_miniapp(toy_snake
+    MAIN snake.cpp
+    ${MFEM_MINIAPPS_COMMON_HEADERS}
+    LIBRARIES mfem mfem-common)
+
+add_mfem_miniapp(toy_lissajous
+    MAIN lissajous.cpp LIBRARIES mfem)
+
+add_mfem_miniapp(toy_mondrian
+    MAIN mondrian.cpp LIBRARIES mfem)
+

miniapps/toys/automata.cpp

@@ -0,0 +1,202 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+//
+//         ----------------------------------------------------
+//         Automata Miniapp:  Model of simple cellular automata
+//         ----------------------------------------------------
+//
+// This miniapp implements a one dimensional elementary cellular automata
+// as described in: mathworld.wolfram.com/ElementaryCellularAutomaton.html
+//
+// This miniapp shows a completely unnecessary use of the finite element
+// method to simply display binary data (but it's fun to play with).
+//
+// Compile with: make automata
+//
+// Sample runs: automata
+//              automata -r 110 -ns 32
+//              automata -r 30 -ns 96
+
+#include "mfem.hpp"
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <bitset>
+#include <vector>
+
+using namespace std;
+using namespace mfem;
+
+void PrintRule(bitset<8> & r);
+void ApplyRule(vector<bool> * b[], bitset<8> & r, int ns, int s);
+void ProjectStep(const vector<bool> & b, GridFunction & x, int ns, int s);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int ns = 16;
+   int  r = 90;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ns, "-ns", "--num-steps",
+                  "Number of steps of the 1D cellular automaton.");
+   args.AddOption(&r, "-r", "--rule",
+                  "Elementary cellular automaton rule [0-255].");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Build a rectangular mesh of quadrilateral elements nearly twice
+   //    as wide as it is high.
+   Mesh *mesh = new Mesh(2 * ns - 1, ns, Element::QUADRILATERAL,
+                         0, 2 * ns - 1, ns, false);
+
+   // 3. Define a finite element space on the mesh. Here we use discontinuous
+   //    Lagrange finite elements of order zero i.e. piecewise constant basis
+   //    functions.
+   FiniteElementCollection *fec = new L2_FECollection(0, 2);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   // 4. Initialize a pair of bit arrays to store two rows in the evolution
+   //    of our cellular automaton.
+   int len = 2 * ns - 1;
+
+   vector<bool> * vbp[2];
+   vector<bool> vb0(len);
+   vector<bool> vb1(len);
+
+   vbp[0] = &vb0;
+   vbp[1] = &vb1;
+
+   for (int i=0; i<len; i++)
+   {
+      vb0[i] = false;
+      vb1[i] = false;
+   }
+   vb0[ns-1] = true;
+
+   // 5. Define the vector x as a finite element grid function corresponding
+   //    to fespace which will be used to visualize the cellular automata.
+   //    Initialize x with initial condition of zero, which indicates a
+   //    "white" or "off" cell in our automaton.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 6. Open a socket to GLVis to visualize the automaton.
+   socketstream sol_sock;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sol_sock.open(vishost, visport);
+   }
+
+   // 7. Create the rule as a bitset and display it for the user
+   bitset<8> rbs = r;
+   PrintRule(rbs);
+
+   // Transfer the current row of the automaton to the vector x.
+   ProjectStep(*vbp[0], x, ns, 0);
+
+   // 8. Apply the rule iteratively
+   cout << endl << "Applying rule..." << flush;
+   for (int s=1; s<ns; s++)
+   {
+      // Compute the next row from the current row
+      ApplyRule(vbp, rbs, ns, s);
+
+      // Transfer the new row of the automaton to the vector x.
+      ProjectStep(*vbp[1], x, ns, s);
+
+      // Swap bit arrays
+      std::swap(vbp[0], vbp[1]);
+
+      // 9. Send the solution by socket to a GLVis server.
+      if (visualization)
+      {
+         sol_sock << "solution\n" << *mesh << x << flush;
+         {
+            static int once = 1;
+            if (once)
+            {
+               sol_sock << "keys Ajl\n";
+               sol_sock << "view 0 180\n";
+               sol_sock << "zoom 2.2\n";
+               sol_sock << "palette 24\n";
+               once = 0;
+            }
+         }
+      }
+   }
+   cout << "done." << endl;
+
+   // 10. Save the mesh and the final state of the automaton. This output can be
+   //     viewed later using GLVis: "glvis -m automata.mesh -g automata.gf".
+   ofstream mesh_ofs("automata.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("automata.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 11. Free the used memory.
+   delete fespace;
+   delete fec;
+   delete mesh;
+
+   return 0;
+}
+
+bool Rule(bitset<8> & r, bool b0, bool b1, bool b2)
+{
+   return r[(b0 ? 1 : 0) + (b1 ? 2 : 0) + (b2 ? 4 : 0)];
+}
+
+void PrintRule(bitset<8> & r)
+{
+   cout << endl << "Rule:" << endl;
+   for (int i=7; i>=0; i--)
+   {
+      cout << " " << i/4 << (i/2)%2 << i%2;
+   }
+   cout << endl;
+   for (int i=7; i>=0; i--)
+   {
+      cout << "  " << Rule(r,i%2,(i/2)%2,i/4) << " ";
+   }
+   cout << endl;
+}
+
+void ApplyRule(vector<bool> * b[], bitset<8> & r, int ns, int s)
+{
+   for (int i=0; i<2*ns-1; i++)
+   {
+      int i0 = (i + 2 * ns - 2) % (2 * ns - 1);
+      int i2 = (i + 1) % (2 * ns - 1);
+      (*b[1])[i] = Rule(r, (*b[0])[i0], (*b[0])[i], (*b[0])[i2]);
+   }
+}
+
+void ProjectStep(const vector<bool> & b, GridFunction & x, int ns, int s)
+{
+   for (int i=0; i<2*ns-1; i++)
+   {
+      x[s*(2*ns-1)+i] = (double)b[i];
+   }
+}

miniapps/toys/life.cpp

@@ -0,0 +1,444 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+//
+//               ----------------------------------------
+//               Life Miniapp:  Model of the Game of Life
+//               ----------------------------------------
+//
+// This miniapp implements Conway's Game of Life. A few simple starting
+// positions are available as well as a random initial state. The game will
+// terminate only if two successive iterations are identical.
+//
+// See the output of 'life -h' for more options.
+//
+// Compile with: make life
+//
+// Sample runs: life
+//              life -nx 30
+//              life -nx 100 -ny 100 -r 0.3
+//              life -g '2 3 0'
+//              life -b '10 10 0' -g '2 2 0'
+//              life -b '10 10 1' -g '2 2 0'
+//              life -sp '8 10 0 1 1 1 2 1 1 1'
+//              life -nx 30 -sp '11 11 1 1 1 1 1 1 1 1 2
+//                                     1 0 1 1 1 1 0 1 2
+//                                     1 1 1 1 1 1 1 1'
+
+#include "mfem.hpp"
+#include <algorithm>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <bitset>
+#include <vector>
+
+using namespace std;
+using namespace mfem;
+
+bool GameStep(vector<bool> * b[], int nx, int ny);
+void ProjectStep(const vector<bool> & b, GridFunction & x, int n);
+
+bool InitSketchPad(vector<bool> & b, int nx, int ny, const Array<int> & params);
+bool InitBlinker(vector<bool> & b, int nx, int ny, const Array<int> & params);
+bool InitGlider(vector<bool> & b, int nx, int ny, const Array<int> & params);
+bool InitMFEM(vector<bool> & b, int nx, int ny);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int nx = 20;
+   int ny = 20;
+   int rs = -1;
+   double r = -1.0;
+   Array<int> sketch_pad_params(0);
+   Array<int> blinker_params(0);
+   Array<int> glider_params(0);
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&nx, "-nx", "--num-elems-x",
+                  "Number of elements in the x direction.");
+   args.AddOption(&ny, "-ny", "--num-elems-y",
+                  "Number of elements in the y direction.");
+   args.AddOption(&r, "-r", "--random-fraction",
+                  "Fraction of randomly chosen live cells.");
+   args.AddOption(&rs, "-rs", "--random-seed",
+                  "Seed for the random number generator.");
+   args.AddOption(&sketch_pad_params, "-sp", "--sketch-pad",
+                  "Specify the starting coordinates and values on a grid"
+                  " of cells.  The values can be 0, 1, or 2.  Where 0 and 1"
+                  " indicate cells that are off or on and 2 represents a"
+                  " newline character.");
+   args.AddOption(&blinker_params, "-b", "--blinker",
+                  "Specify the starting coordinates and orientation (0 or 1)"
+                  " of the blinker. Multiple blinkers can be specified as "
+                  "'x0 y0 o0 x1 y1 o1 ...'.");
+   args.AddOption(&glider_params, "-g", "--glider",
+                  "Specify the starting coordinates and "
+                  "orientation (0,1,2, or 3) of the glider. "
+                  "Multiple gliders can be specified as "
+                  "'x0 y0 o0 x1 y1 o1 ...'.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Build a rectangular mesh of quadrilateral elements.
+   Mesh *mesh = new Mesh(nx, ny, Element::QUADRILATERAL, 0, nx, ny, false);
+
+   // 3. Define a finite element space on the mesh. Here we use discontinuous
+   //    Lagrange finite elements of order zero i.e. piecewise constant basis
+   //    functions.
+   FiniteElementCollection *fec = new L2_FECollection(0, 2);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+   // 4. Initialize a pair of bit arrays to store two copies of the
+   //    playing field.
+   int len = nx * ny;
+
+   vector<bool> * vbp[2];
+   vector<bool> vb0(len);
+   vector<bool> vb1(len);
+
+   vbp[0] = &vb0;
+   vbp[1] = &vb1;
+
+   if ( r > 0.0 )
+   {
+      unsigned int seed;
+      if ( rs < 0 )
+      {
+         srand(time(NULL));
+         seed = (unsigned int)rand();
+      }
+      else
+      {
+         seed = (unsigned int)rs;
+      }
+      cout << "Using random seed:  " << seed << endl;
+      srand(seed);
+   }
+
+   bool init = false;
+   if (r > 0)
+   {
+      for (int i=0; i<len; i++)
+      {
+         double rv = double(rand()) / RAND_MAX;
+         vb0[i] = (rv <= r);
+         vb1[i] = false;
+      }
+   }
+   else
+   {
+      for (int i=0; i<len; i++)
+      {
+         vb0[i] = false;
+      }
+   }
+   if ( sketch_pad_params.Size() > 2 )
+   {
+      init = InitSketchPad(vb0, nx, ny, sketch_pad_params);
+   }
+   if ( blinker_params.Size() > 0 && (blinker_params.Size() % 3 == 0 ) )
+   {
+      init = InitBlinker(vb0, nx, ny, blinker_params);
+   }
+   if ( glider_params.Size() > 0 && (glider_params.Size() % 3 == 0 ) )
+   {
+      init = InitGlider(vb0, nx, ny, glider_params);
+   }
+   if (!init)
+   {
+      init = InitMFEM(vb0, nx, ny);
+   }
+
+   // 5. Define the vector x as a finite element grid function corresponding
+   //    to fespace which will be used to visualize the playing field.
+   //    Initialize x with the starting layout set above.
+   GridFunction x(fespace);
+
+   ProjectStep(*vbp[0], x, len);
+
+   // 6. Open a socket to GLVis
+   socketstream sol_sock;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sol_sock.open(vishost, visport);
+   }
+
+   // 7. Apply the rule iteratively
+   cout << endl << "Running the Game of Life..." << flush;
+
+   bool is_good = true;
+   bool is_stable = false;
+   while ( is_good && visualization && !is_stable )
+   {
+      is_stable = GameStep(vbp, nx, ny);
+      ProjectStep(*vbp[1], x, len);
+
+      // Swap bit arrays
+      std::swap(vbp[0], vbp[1]);
+
+      // 8. Send the solution by socket to a GLVis server.
+      is_good = sol_sock.good();
+
+      if (visualization && is_good )
+      {
+         sol_sock << "solution\n" << *mesh << x << flush;
+         {
+            static int once = 1;
+            if (once)
+            {
+               sol_sock << "keys Ajlm\n";
+               sol_sock << "view 0 0\n";
+               sol_sock << "zoom 1.9\n";
+               sol_sock << "palette 24\n";
+               once = 0;
+            }
+            if (is_stable)
+            {
+               sol_sock << "valuerange 0 1\n";
+            }
+         }
+      }
+   }
+   cout << "done." << endl;
+
+   // 9. Save the mesh and the final state of the game. This output can be
+   //    viewed later using GLVis: "glvis -m life.mesh -g life.gf".
+   ofstream mesh_ofs("life.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("life.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 10. Free the used memory.
+   delete fespace;
+   delete fec;
+   delete mesh;
+
+   return 0;
+}
+
+inline int index(int i, int j, int nx, int ny)
+{
+   return ((j + ny) % ny) * nx + ((i + nx) % nx);
+}
+
+bool GameStep(vector<bool> * b[], int nx, int ny)
+{
+   bool is_stable = true;
+   for (int j=0; j<ny; j++)
+   {
+      for (int i=0; i<nx; i++)
+      {
+         int c =
+            (int)(*b[0])[index(i+0,j+0,nx,ny)] +
+            (int)(*b[0])[index(i+1,j+0,nx,ny)] +
+            (int)(*b[0])[index(i+1,j+1,nx,ny)] +
+            (int)(*b[0])[index(i+0,j+1,nx,ny)] +
+            (int)(*b[0])[index(i-1,j+1,nx,ny)] +
+            (int)(*b[0])[index(i-1,j+0,nx,ny)] +
+            (int)(*b[0])[index(i-1,j-1,nx,ny)] +
+            (int)(*b[0])[index(i+0,j-1,nx,ny)] +
+            (int)(*b[0])[index(i+1,j-1,nx,ny)];
+         switch (c)
+         {
+            case 3:
+               (*b[1])[index(i,j,nx,ny)] = true;
+               break;
+            case 4:
+               (*b[1])[index(i,j,nx,ny)] = (*b[0])[index(i,j,nx,ny)];
+               break;
+            default:
+               (*b[1])[index(i,j,nx,ny)] = false;
+               break;
+         }
+         is_stable &= (*b[1])[index(i,j,nx,ny)] == (*b[0])[index(i,j,nx,ny)];
+      }
+   }
+   return is_stable;
+}
+
+void ProjectStep(const vector<bool> & b, GridFunction & x, int n)
+{
+   for (int i=0; i<n; i++)
+   {
+      x[i] = (double)b[i];
+   }
+}
+
+bool InitBlinker(vector<bool> & b, int nx, int ny, const Array<int> & params)
+{
+   for (int i=0; i<params.Size()/3; i++)
+   {
+      int cx   = params[3 * i + 0];
+      int cy   = params[3 * i + 1];
+      int ornt = params[3 * i + 2];
+
+      switch (ornt % 2)
+      {
+         case 0:
+            b[index(cx+0,cy+1,nx,ny)] = true;
+            b[index(cx+0,cy+0,nx,ny)] = true;
+            b[index(cx+0,cy-1,nx,ny)] = true;
+            break;
+         case 1:
+            b[index(cx+1,cy+0,nx,ny)] = true;
+            b[index(cx+0,cy+0,nx,ny)] = true;
+            b[index(cx-1,cy+0,nx,ny)] = true;
+            break;
+      }
+   }
+   return true;
+}
+
+bool InitGlider(vector<bool> & b, int nx, int ny, const Array<int> & params)
+{
+   for (int i=0; i<params.Size()/3; i++)
+   {
+      int cx   = params[3 * i + 0];
+      int cy   = params[3 * i + 1];
+      int ornt = params[3 * i + 2];
+
+      switch (ornt % 4)
+      {
+         case 0:
+            b[index(cx-1,cy+0,nx,ny)] = true;
+            b[index(cx+0,cy+1,nx,ny)] = true;
+            b[index(cx+1,cy-1,nx,ny)] = true;
+            b[index(cx+1,cy+0,nx,ny)] = true;
+            b[index(cx+1,cy+1,nx,ny)] = true;
+            break;
+         case 1:
+            b[index(cx+0,cy-1,nx,ny)] = true;
+            b[index(cx-1,cy+0,nx,ny)] = true;
+            b[index(cx-1,cy+1,nx,ny)] = true;
+            b[index(cx+0,cy+1,nx,ny)] = true;
+            b[index(cx+1,cy+1,nx,ny)] = true;
+            break;
+         case 2:
+            b[index(cx+1,cy+0,nx,ny)] = true;
+            b[index(cx+0,cy-1,nx,ny)] = true;
+            b[index(cx-1,cy-1,nx,ny)] = true;
+            b[index(cx-1,cy+0,nx,ny)] = true;
+            b[index(cx-1,cy+1,nx,ny)] = true;
+            break;
+         case 3:
+            b[index(cx+0,cy+1,nx,ny)] = true;
+            b[index(cx+1,cy+0,nx,ny)] = true;
+            b[index(cx-1,cy-1,nx,ny)] = true;
+            b[index(cx+0,cy-1,nx,ny)] = true;
+            b[index(cx+1,cy-1,nx,ny)] = true;
+            break;
+      }
+   }
+   return true;
+}
+
+bool InitSketchPad(vector<bool> & b, int nx, int ny, const Array<int> & params)
+{
+   int cx   = params[0];
+   int cy   = params[1];
+
+   int ox = 0;
+   int oy = 0;
+
+   for (int i=2; i<params.Size(); i++)
+   {
+      if ( params[i]/2 == 1 )
+      {
+         ox = 0;
+         oy--;
+      }
+      else
+      {
+         b[index(cx+ox,cy+oy,nx,ny)] = (bool)params[i];
+         ox++;
+      }
+   }
+   return true;
+}
+
+bool InitMFEM(vector<bool> & b, int nx, int ny)
+{
+   int ox = 0;
+   int oy = 0;
+   int wx = (nx >= 23) ? 23 : 5;
+   int hy = (ny >= 7) ? 7 : 5;
+
+   if (wx == 23)
+   {
+      // Write out "MFEM"
+      ox = (nx - 23) / 2;
+      oy = (ny - hy) / 2;
+
+      for (int j=0; j<hy; j++)
+      {
+         b[index(ox +  0, oy+j,nx,ny)] = true;
+         b[index(ox +  4, oy+j,nx,ny)] = true;
+         b[index(ox +  6, oy+j,nx,ny)] = true;
+         b[index(ox + 12, oy+j,nx,ny)] = true;
+         b[index(ox + 18, oy+j,nx,ny)] = true;
+         b[index(ox + 22, oy+j,nx,ny)] = true;
+      }
+      for (int i=1; i<5; i++)
+      {
+         b[index(ox +  6 + i, oy + hy - 1,nx,ny)] = true;
+         b[index(ox + 12 + i, oy +  0,nx,ny)] = true;
+         b[index(ox + 12 + i, oy + hy - 1,nx,ny)] = true;
+      }
+      for (int i=1; i<4; i++)
+      {
+         b[index(ox +  6 + i, oy +  hy/2,nx,ny)] = true;
+         b[index(ox + 12 + i, oy +  hy/2,nx,ny)] = true;
+      }
+      b[index(ox + 1, oy +  hy - 2,nx,ny)] = true;
+      b[index(ox + 2, oy +  hy - 3,nx,ny)] = true;
+      b[index(ox + 3, oy +  hy - 2,nx,ny)] = true;
+
+      b[index(ox + 19, oy +  hy - 2,nx,ny)] = true;
+      b[index(ox + 20, oy +  hy - 3,nx,ny)] = true;
+      b[index(ox + 21, oy +  hy - 2,nx,ny)] = true;
+   }
+   else if (wx == 5)
+   {
+      // Create a single 'M'
+      ox = (nx - 5) / 2;
+      oy = (ny - hy) / 2;
+
+      for (int j=0; j<hy; j++)
+      {
+         b[index(ox +  0, oy+j,nx,ny)] = true;
+         b[index(ox +  4, oy+j,nx,ny)] = true;
+      }
+      b[index(ox + 1, oy +  hy - 2,nx,ny)] = true;
+      b[index(ox + 2, oy +  hy - 3,nx,ny)] = true;
+      b[index(ox + 3, oy +  hy - 2,nx,ny)] = true;
+   }
+   else
+   {
+      // Set a single pixel
+      b[index(nx/2,ny/2,nx,ny)] = true;
+   }
+
+   return true;
+}

miniapps/toys/lissajous.cpp

@@ -0,0 +1,201 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+//
+//               ---------------------------------------------
+//               Lissajous Miniapp:  Spinning optical illusion
+//               ---------------------------------------------
+//
+// This miniapp generates two different Lissajous curves in 3D which appear to
+// spin vertically and/or horizontally, even though the net motion is the same.
+// Based on the 2019 Illusion of the year "Dual Axis Illusion" by Frank Force,
+// see http://illusionoftheyear.com/2019/12/dual-axis-illusion.
+//
+// Compile with: make lissajous
+//
+// Sample runs:  lissajous
+//               lissajous -a 5 -b 4
+//               lissajous -a 4 -b 3 -delta -90
+//               lissajous -o 8 -nx 3 -ny 3
+//               lissajous -a 11 -b 10 -o 4
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double u_function(const Vector &x);
+void lissajous_trans_v(const Vector &x, Vector &p);
+void lissajous_trans_h(const Vector &x, Vector &p);
+
+// Default Lissajous curve parameters
+double a = 3.0;
+double b = 2.0;
+double delta = 90;
+
+int main(int argc, char *argv[])
+{
+   int nx = 32;
+   int ny = 3;
+   int order = 2;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&nx, "-nx", "--num-elements-x",
+                  "Number of elements in x-direction.");
+   args.AddOption(&ny, "-ny", "--num-elements-y",
+                  "Number of elements in y-direction.");
+   args.AddOption(&order, "-o", "--mesh-order",
+                  "Order (polynomial degree) of the mesh elements.");
+   args.AddOption(&a, "-a", "--x-frequency",
+                  "Frequency of the x-component.");
+   args.AddOption(&b, "-b", "--y-frequency",
+                  "Frequency of the y-component.");
+   args.AddOption(&delta, "-delta", "--x-phase",
+                  "Phase angle of the x-component.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   delta *= M_PI / 180.0; // convert to radians
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream soutv, south;
+
+   {
+      Mesh mesh(nx, ny, Element::QUADRILATERAL, 1, 2*M_PI, 2*M_PI);
+      mesh.SetCurvature(order, true, 3, Ordering::byVDIM);
+      mesh.Transform(lissajous_trans_v);
+
+      H1_FECollection fec(order, 3);
+      FiniteElementSpace fes(&mesh, &fec);
+      GridFunction u(&fes);
+      FunctionCoefficient ufc(u_function);
+      u.ProjectCoefficient(ufc);
+
+      if (visualization)
+      {
+         soutv.open(vishost, visport);
+         soutv << "solution\n" << mesh << u;
+         soutv << "keys 'ARRj" << std::string(90, '7')  << "'\n";
+         soutv << "palette 17 zoom 1.65 subdivisions 32 0\n";
+         soutv << "window_title 'V' window_geometry 0 0 500 500\n";
+         soutv << flush;
+      }
+   }
+
+   {
+      Mesh mesh(nx, ny, Element::QUADRILATERAL, 1, 2*M_PI, 2*M_PI);
+      mesh.SetCurvature(order, true, 3, Ordering::byVDIM);
+      mesh.Transform(lissajous_trans_h);
+
+      H1_FECollection fec(order, 3);
+      FiniteElementSpace fes(&mesh, &fec);
+      GridFunction u(&fes);
+      FunctionCoefficient ufc(u_function);
+      u.ProjectCoefficient(ufc);
+
+      if (visualization)
+      {
+         south.open(vishost, visport);
+         south << "solution\n" << mesh << u;
+         south << "keys 'ARRj'\n";
+         south << "palette 17 zoom 1.65 subdivisions 32 0\n";
+         south << "window_title 'H' window_geometry 500 0 500 500\n";
+         south << flush;
+      }
+
+      ofstream mesh_ofs("lissajous.mesh");
+      mesh_ofs.precision(8);
+      mesh.Print(mesh_ofs);
+      ofstream sol_ofs("lissajous.gf");
+      sol_ofs.precision(8);
+      u.Save(sol_ofs);
+   }
+
+   soutv << "keys '.0" << std::string(b, '0') << "'\n" << flush;
+   south << "keys '.0" << std::string(a, '0') << "'\n" << flush;
+
+   cout << "Which direction(s) are the two curves spinning in?\n";
+
+   return 0;
+}
+
+// Simple function to project to help identify the spinning
+double u_function(const Vector &x)
+{
+   return x[2];
+}
+
+// Tubular Lissajous curve with the given parameters (a, b, theta)
+void lissajous_trans(const Vector &x, Vector &p,
+                     double a, double b, double delta)
+{
+   p.SetSize(3);
+
+   double phi = x[0];
+   double theta = x[1];
+   double t = phi;
+
+   double A = b; // Scaling of the curve along the x-axis
+   double B = a; // Scaling of the curve along the y-axis
+
+   // Lissajous curve on a 3D cylinder
+   p[0] = B*cos(b*t);
+   p[1] = B*sin(b*t); // Y
+   p[2] = A*sin(a*t + delta); // X
+
+   // Turn the curve into a tubular surface
+   {
+      // tubular radius
+      double R = 0.02*(A+B);
+
+      // normal to the cylinder at p(t)
+      double normal[3] = { cos(b*t), sin(b*t), 0 };
+
+      // tangent to the curve, dp/dt(t)
+      // double tangent[3] = { -b*B*sin(b*t), b*B*cos(b*t), A*a*cos(a*t+delta) };
+
+      // normalized cross product of tangent and normal at p(t)
+      double cn = 1e-128;
+      double cross[3] = { A*a*sin(b*t)*cos(a*t+delta), -A*a*cos(b*t)*cos(a*t+delta), b*B };
+      for (int i = 0; i < 3; i++) { cn += cross[i]*cross[i]; }
+      for (int i = 0; i < 3; i++) { cross[i] /= sqrt(cn); }
+
+      // create a tubular surface of radius R around the curve p(t), in the plane
+      // orthogonal to the tangent (with basis given by normal and cross)
+      for (int i = 0; i < 3; i++)
+      {
+         p[i] += R * (cos(theta)*normal[i] + sin(theta)*cross[i]);
+      }
+   }
+}
+
+// Vertically spinning curve
+void lissajous_trans_v(const Vector &x, Vector &p)
+{
+   return lissajous_trans(x, p, a, b, delta);
+}
+
+// Horizontally spinning curve
+void lissajous_trans_h(const Vector &x, Vector &p)
+{
+   return lissajous_trans(x, p, b, a, delta);
+}

miniapps/toys/makefile

@@ -0,0 +1,98 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/toys/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+# Include defaults.mk to get XLINKER
+DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk
+include $(DEFAULTS_MK)
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_MINIAPPS = automata life mandel rubik snake lissajous mondrian
+PAR_MINIAPPS =
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all lib-common clean clean-build clean-exec
+.PRECIOUS: %.o
+
+COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common
+
+# If MFEM_SHARED is set, add the ../common rpath
+COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
+   $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\
+   $(MFEM_BUILD_DIR)/miniapps/common))
+
+all: $(MINIAPPS)
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+# Toys that depend on lib-common
+snake rubik: %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK) | lib-common
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) -o $@ $@.o $(COMMON_LIB) $(MFEM_LIBS)
+
+# Rule for building lib-common
+lib-common:
+	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test-file,$<, $(RUN_MPI), Toys miniapp,$(<).mesh)
+%-test-seq: %
+	@$(call mfem-test-file,$<,, Toys miniapp,$(<).mesh)
+snake-test-seq: snake
+	@$(call mfem-test-file,$<,, Toys miniapp,$(<)-init.mesh)
+rubik-test-seq: rubik
+	@$(call mfem-test-file,$<,, Toys miniapp,$(<)-init.mesh)
+
+# Testing: Specific execution options
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_MINIAPPS) $(PAR_MINIAPPS)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f snake-init.mesh snake-user.mesh snake-joined.mesh snake-c*.mesh
+	@rm -f automata.gf automata.mesh rubik-init.mesh mandel.mesh
+	@rm -f life.gf life.mesh lissajous.mesh lissajous.gf mondrian.mesh

miniapps/toys/mandel.cpp

@@ -0,0 +1,253 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+//
+//             ----------------------------------------------
+//             Mandel Miniapp: Fractal visualization with AMR
+//             ----------------------------------------------
+//
+// This miniapp is a specialized version of the Shaper miniapp to the Mandelbrot
+// set. It provides a light-hearted example of AMR and GLVis integration.
+//
+// Compile with: make mandel
+//
+// Sample runs:  mandel
+//               mandel -m ../../data/inline-tri.mesh
+//               mandel -m ../../data/star-mixed-p2.mesh -a -ncl -1 -sd 4
+//               mandel -m ../../data/klein-bottle.mesh
+//               mandel -m ../../data/inline-hex.mesh
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace mfem;
+using namespace std;
+
+// Given a point x, return its material id as an integer. The ids should be
+// positive. If the point is exactly on the interface, return 0.
+//
+// In this particular miniapp, the material value is based on the number of
+// iterations for the point from the definition of the Mandelbrot set.
+int material(Vector &x, Vector &xmin, Vector &xmax)
+{
+   // Rescaling to [0,1]^sdim
+   for (int i = 0; i < x.Size(); i++)
+   {
+      x(i) = (x(i)-xmin(i))/(xmax(i)-xmin(i));
+   }
+   x(0) -= 0.1;
+   double col = x(0), row = x(1);
+   {
+      int width = 1080, height = 1080;
+      col *= width;
+      row *= height;
+      double c_re = (col - width/2)*4.0/width;
+      double c_im = (row - height/2)*4.0/width;
+      double x = 0, y = 0;
+      int iteration = 0, maxit = 10000;
+      while (x*x+y*y <= 4 && iteration < maxit)
+      {
+         double x_new = x*x - y*y + c_re;
+         y = 2*x*y + c_im;
+         x = x_new;
+         iteration++;
+      }
+      if (iteration < maxit)
+      {
+         return iteration%10+2;
+      }
+      else
+      {
+         return 1;
+      }
+   }
+}
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int sd = 2;
+   int nclimit = 1;
+   bool aniso = false;
+   bool visualization = 1;
+
+   // Parse command line
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Input mesh file to shape materials in.");
+   args.AddOption(&sd, "-sd", "--sub-divisions",
+                  "Number of element subdivisions for interface detection.");
+   args.AddOption(&nclimit, "-ncl", "--nc-limit",
+                  "Level of hanging nodes allowed (-1 = unlimited).");
+   args.AddOption(&aniso, "-a", "--aniso", "-i", "--iso",
+                  "Enable anisotropic refinement of quads and hexes.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good()) { args.PrintUsage(cout); return 1; }
+   args.PrintOptions(cout);
+
+   // Read initial mesh, get dimensions and bounding box
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+   Vector xmin, xmax;
+   mesh.GetBoundingBox(xmin, xmax);
+
+   // Increase the mesh resolution
+   for (int l = 0; l < 3; l++) { mesh.UniformRefinement(); }
+
+   // NURBS meshes don't support non-conforming refinement for now
+   if (mesh.NURBSext) { mesh.SetCurvature(2); }
+
+   // Anisotropic refinement not supported for simplex meshes.
+   if (mesh.MeshGenerator() & 1) { aniso = false; }
+
+   // Mesh attributes will be visualized as piece-wise constants
+   L2_FECollection attr_fec(0, dim);
+   FiniteElementSpace attr_fespace(&mesh, &attr_fec);
+   GridFunction attr(&attr_fespace);
+
+   // GLVis server to visualize to
+   socketstream sol_sock;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sol_sock.open(vishost, visport);
+      sol_sock.precision(8);
+   }
+
+   // Shaping loop
+   for (int iter = 0; 1; iter++)
+   {
+      Array<Refinement> refs;
+      for (int i = 0; i < mesh.GetNE(); i++)
+      {
+         bool refine = false;
+
+         // Sample materials in each element using "sd" sub-divisions
+         Vector pt;
+         Geometry::Type geom = mesh.GetElementBaseGeometry(i);
+         ElementTransformation *T = mesh.GetElementTransformation(i);
+         RefinedGeometry *RefG = GlobGeometryRefiner.Refine(geom, sd, 1);
+         IntegrationRule &ir = RefG->RefPts;
+
+         // Refine any element where different materials are detected. A more
+         // sophisticated logic can be implemented here -- e.g. don't refine
+         // the interfaces between certain materials.
+         Array<int> mat(ir.GetNPoints());
+         double matsum = 0.0;
+         for (int j = 0; j < ir.GetNPoints(); j++)
+         {
+            T->Transform(ir.IntPoint(j), pt);
+            int m = material(pt, xmin, xmax);
+            mat[j] = m;
+            matsum += m;
+            if ((int)matsum != m*(j+1))
+            {
+               refine = true;
+            }
+         }
+
+         // Set the element attribute as the "average". Other choices are
+         // possible here too, e.g. attr(i) = mat;
+         attr(i) = round(matsum/ir.GetNPoints());
+
+         // Mark the element for refinement
+         if (refine)
+         {
+            int type = 7;
+            if (aniso)
+            {
+               // Determine the XYZ bitmask for anisotropic refinement.
+               int dx = 0, dy = 0, dz = 0;
+               const int s = sd+1;
+               if (dim == 2)
+               {
+                  for (int j = 0; j <= sd; j++)
+                     for (int i = 0; i < sd; i++)
+                     {
+                        dx += abs(mat[j*s + i+1] - mat[j*s + i]);
+                        dy += abs(mat[(i+1)*s + j] - mat[i*s + j]);
+                     }
+               }
+               else if (dim == 3)
+               {
+                  for (int k = 0; k <= sd; k++)
+                     for (int j = 0; j <= sd; j++)
+                        for (int i = 0; i < sd; i++)
+                        {
+                           dx += abs(mat[(k*s + j)*s + i+1] - mat[(k*s + j)*s + i]);
+                           dy += abs(mat[(k*s + i+1)*s + j] - mat[(k*s + i)*s + j]);
+                           dz += abs(mat[((i+1)*s + j)*s + k] - mat[(i*s + j)*s + k]);
+                        }
+               }
+               type = 0;
+               const int tol = mat.Size() / 10;
+               if (dx > tol) { type |= 1; }
+               if (dy > tol) { type |= 2; }
+               if (dz > tol) { type |= 4; }
+               if (!type) { type = 7; } // because of tol
+            }
+
+            refs.Append(Refinement(i, type));
+         }
+      }
+
+      // Visualization
+      if (visualization)
+      {
+         sol_sock << "solution\n" << mesh << attr;
+         if (iter == 0 && sdim == 2)
+         {
+            sol_sock << "keys 'RjlppppppppppppppA*************'\n";
+         }
+         if (iter == 0 && sdim == 3)
+         {
+            sol_sock << "keys 'YYYYYYYYYXXXXXXXmA********8888888pppttt";
+            if (dim == 3) { sol_sock << "iiM"; }
+            sol_sock << "'\n";
+         }
+         sol_sock << flush;
+      }
+
+      // Ask the user if we should continue refining
+      cout << "Iteration " << iter+1 << ": mesh has " << mesh.GetNE() <<
+           " elements. \n";
+      if ((iter+1) % 4 == 0)
+      {
+         if (!visualization) { break; }
+         char yn;
+         cout << "Continue shaping? --> ";
+         cin >> yn;
+         if (yn == 'n' || yn == 'q') { break; }
+      }
+
+      // Perform refinement, update spaces and grid functions
+      mesh.GeneralRefinement(refs, -1, nclimit);
+      attr_fespace.Update();
+      attr.Update();
+   }
+
+   // Set element attributes in the mesh object before saving
+   for (int i = 0; i < mesh.GetNE(); i++)
+   {
+      mesh.SetAttribute(i, attr(i));
+   }
+   mesh.SetAttributes();
+
+   // Save the final mesh
+   ofstream mesh_ofs("mandel.mesh");
+   mesh_ofs.precision(8);
+   mesh.Print(mesh_ofs);
+}

miniapps/toys/mondrian.cpp

@@ -0,0 +1,365 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+//
+//           -------------------------------------------------
+//           Mondrian Miniapp: Convert an image to an AMR mesh
+//           -------------------------------------------------
+//
+// This miniapp is a specialized version of the Shaper miniapp that converts an
+// input image to an AMR mesh. It allows the fast approximate meshing of any
+// domain for which there is an image.
+//
+// The input to image should be in 8-bit grayscale PGM format. You can use a
+// number of image manipulation tools, such as GIMP (gimp.org) and ImageMagick's
+// convert utility (imagemagick.org/script/convert.php) to convert your image to
+// this format as a pre-processing step, e.g.:
+//
+//   /usr/bin/convert australia.svg -compress none -depth 8 australia.pgm
+//
+// Compile with: make mondrian
+//
+// Sample runs:  mondrian -i australia.pgm
+//               mondrian -i australia.pgm -m ../../data/inline-tri.mesh
+//               mondrian -i australia.pgm -m ../../data/disc-nurbs.mesh
+//               mondrian -i australia.pgm -sd 3 -a -ncl -1
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace mfem;
+using namespace std;
+
+// Simple class to parse portable graymap format (PGM) image files, see
+// http://netpbm.sourceforge.net/doc/pgm.html
+class ParsePGM
+{
+public:
+   ParsePGM(const char *filename);
+   ~ParsePGM();
+
+   int Height() const { return N; }
+   int Width() const { return M; }
+
+   int operator()(int i, int j) const
+   { return int((pgm8) ? pgm8[M*i+j] : pgm16[M*i+j]); }
+
+private:
+   int M, N;
+   int depth;
+
+   char *pgm8;
+   unsigned short int *pgm16;
+
+   void ReadMagicNumber(istream &in);
+   void ReadComments(istream &in);
+   void ReadDimensions(istream &in);
+   void ReadDepth(istream &in);
+   void ReadPGM(istream &in);
+};
+
+// Given a point x, return its "material" specification defined by the grayscale
+// pixel values from the pgm image using NC different colors.
+int material(const ParsePGM &pgm, int NC,
+             Vector &x, Vector &xmin, Vector &xmax);
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   const char *img_file = "australia.pgm";
+   int sd = 2;
+   int nclimit = 1;
+   int ncolors = 3;
+   bool aniso = false;
+   bool visualization = 1;
+
+   // Parse command line
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Input mesh file to shape materials in.");
+   args.AddOption(&img_file, "-i", "--img",
+                  "Input image.");
+   args.AddOption(&sd, "-sd", "--sub-divisions",
+                  "Number of element subdivisions for interface detection.");
+   args.AddOption(&nclimit, "-ncl", "--nc-limit",
+                  "Level of hanging nodes allowed (-1 = unlimited).");
+   args.AddOption(&ncolors, "-nc", "--num-colors",
+                  "Number of colors considered (1-256, based on binning).");
+   args.AddOption(&aniso, "-a", "--aniso", "-i", "--iso",
+                  "Enable anisotropic refinement of quads and hexes.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good()) { args.PrintUsage(cout); return 1; }
+   args.PrintOptions(cout);
+
+   // Read the image
+   ParsePGM pgm(img_file);
+
+   // Read initial mesh, get dimensions and bounding box
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+   Vector xmin, xmax;
+   mesh.GetBoundingBox(xmin, xmax);
+
+   // NURBS meshes don't support non-conforming refinement for now
+   if (mesh.NURBSext) { mesh.SetCurvature(2); }
+
+   // Anisotropic refinement not supported for simplex meshes.
+   if (mesh.MeshGenerator() & 1) { aniso = false; }
+
+   // Mesh attributes will be visualized as piece-wise constants
+   L2_FECollection attr_fec(0, dim);
+   FiniteElementSpace attr_fespace(&mesh, &attr_fec);
+   GridFunction attr(&attr_fespace);
+
+   // GLVis server to visualize to
+   socketstream sol_sock;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sol_sock.open(vishost, visport);
+      sol_sock.precision(8);
+   }
+
+   // Shaping loop
+   for (int iter = 0; 1; iter++)
+   {
+      Array<Refinement> refs;
+      for (int i = 0; i < mesh.GetNE(); i++)
+      {
+         bool refine = false;
+
+         // Sample materials in each element using "sd" sub-divisions
+         Vector pt;
+         Geometry::Type geom = mesh.GetElementBaseGeometry(i);
+         ElementTransformation *T = mesh.GetElementTransformation(i);
+         RefinedGeometry *RefG = GlobGeometryRefiner.Refine(geom, sd, 1);
+         IntegrationRule &ir = RefG->RefPts;
+
+         // Refine any element where different materials are detected. A more
+         // sophisticated logic can be implemented here -- e.g. don't refine
+         // the interfaces between certain materials.
+         Array<int> mat(ir.GetNPoints());
+         double matsum = 0.0;
+         for (int j = 0; j < ir.GetNPoints(); j++)
+         {
+            T->Transform(ir.IntPoint(j), pt);
+            int m = material(pgm, 256/ncolors, pt, xmin, xmax);
+            mat[j] = m;
+            matsum += m;
+            if ((int)matsum != m*(j+1))
+            {
+               refine = true;
+            }
+         }
+
+         // Set the element attribute as the "average". Other choices are
+         // possible here too, e.g. attr(i) = mat;
+         attr(i) = round(matsum/ir.GetNPoints());
+
+         // Mark the element for refinement
+         if (refine)
+         {
+            int type = 7;
+            if (aniso)
+            {
+               // Determine the XYZ bitmask for anisotropic refinement.
+               int dx = 0, dy = 0, dz = 0;
+               const int s = sd+1;
+               if (dim == 2)
+               {
+                  for (int j = 0; j <= sd; j++)
+                     for (int i = 0; i < sd; i++)
+                     {
+                        dx += abs(mat[j*s + i+1] - mat[j*s + i]);
+                        dy += abs(mat[(i+1)*s + j] - mat[i*s + j]);
+                     }
+               }
+               else if (dim == 3)
+               {
+                  for (int k = 0; k <= sd; k++)
+                     for (int j = 0; j <= sd; j++)
+                        for (int i = 0; i < sd; i++)
+                        {
+                           dx += abs(mat[(k*s + j)*s + i+1] - mat[(k*s + j)*s + i]);
+                           dy += abs(mat[(k*s + i+1)*s + j] - mat[(k*s + i)*s + j]);
+                           dz += abs(mat[((i+1)*s + j)*s + k] - mat[(i*s + j)*s + k]);
+                        }
+               }
+               type = 0;
+               const int tol = mat.Size() / 10;
+               if (dx > tol) { type |= 1; }
+               if (dy > tol) { type |= 2; }
+               if (dz > tol) { type |= 4; }
+               if (!type) { type = 7; } // because of tol
+            }
+
+            refs.Append(Refinement(i, type));
+         }
+      }
+
+      // Visualization
+      if (visualization)
+      {
+         sol_sock << "solution\n" << mesh << attr;
+         if (iter == 0 && sdim == 2)
+         {
+            sol_sock << "keys 'RjlmpppppppppppppA*************'\n";
+         }
+         if (iter == 0 && sdim == 3)
+         {
+            sol_sock << "keys 'YYYYYYYYYXXXXXXXmA********8888888pppttt";
+            if (dim == 3) { sol_sock << "iiM"; }
+            sol_sock << "'\n";
+         }
+         sol_sock << flush;
+      }
+
+      // Ask the user if we should continue refining
+      cout << "Iteration " << iter+1 << ": mesh has " << mesh.GetNE() <<
+           " elements. \n";
+      if ((iter+1) % 3 == 0)
+      {
+         if (!visualization) { break; }
+         char yn;
+         cout << "Continue shaping? --> ";
+         cin >> yn;
+         if (yn == 'n' || yn == 'q') { break; }
+      }
+
+      // Perform refinement, update spaces and grid functions
+      mesh.GeneralRefinement(refs, -1, nclimit);
+      attr_fespace.Update();
+      attr.Update();
+   }
+
+   // Set element attributes in the mesh object before saving
+   for (int i = 0; i < mesh.GetNE(); i++)
+   {
+      mesh.SetAttribute(i, attr(i));
+   }
+   mesh.SetAttributes();
+
+   // Save the final mesh
+   ofstream mesh_ofs("mondrian.mesh");
+   mesh_ofs.precision(8);
+   mesh.Print(mesh_ofs);
+}
+
+ParsePGM::ParsePGM(const char *filename)
+   : M(-1), N(-1), depth(-1), pgm8(NULL), pgm16(NULL)
+{
+   ifstream in(filename);
+   if (!in)
+   {
+      // Abort with an error message
+      MFEM_ABORT("Image file not found: " << filename << '\n');
+   }
+
+   ReadMagicNumber(in);
+   ReadDimensions(in);
+   ReadDepth(in);
+   ReadPGM(in);
+
+   in.close();
+}
+
+ParsePGM::~ParsePGM()
+{
+   if (pgm8  != NULL) { delete [] pgm8; }
+   if (pgm16 != NULL) { delete [] pgm16; }
+}
+
+void ParsePGM::ReadMagicNumber(istream &in)
+{
+   char c;
+   int p;
+   in >> c >> p; // Read magic number which should be P2 or P5
+   MFEM_VERIFY(c == 'P' && (p == 2 || p == 5),
+               "Invalid PGM file! Unrecognized magic number\""
+               << c << p << "\".");
+   ReadComments(in);
+}
+
+void ParsePGM::ReadComments(istream &in)
+{
+   string buf;
+   in >> std::ws; // absorb any white space
+   while (in.peek() == '#')
+   {
+      std::getline(in,buf);
+   }
+   in >> std::ws; // absorb any white space
+}
+
+void ParsePGM::ReadDimensions(istream &in)
+{
+   in >> M;
+   ReadComments(in);
+   in >> N;
+   ReadComments(in);
+}
+
+void ParsePGM::ReadDepth(istream &in)
+{
+   in >> depth;
+   ReadComments(in);
+}
+
+void ParsePGM::ReadPGM(istream &in)
+{
+   if (depth < 16)
+   {
+      pgm8 = new char[M*N];
+   }
+   else
+   {
+      pgm16 = new unsigned short int[M*N];
+   }
+
+   if (pgm8)
+   {
+      for (int i=0; i<M*N; i++)
+      {
+         in >> pgm8[i];
+      }
+   }
+   else
+   {
+      for (int i=0; i<M*N; i++)
+      {
+         in >> pgm16[i];
+      }
+   }
+}
+
+int material(const ParsePGM &pgm, int NC, Vector &x, Vector &xmin, Vector &xmax)
+{
+   // Rescaling to [0,1]^sdim
+   for (int i = 0; i < x.Size(); i++)
+   {
+      x(i) = (x(i)-xmin(i))/(xmax(i)-xmin(i));
+   }
+
+   int M = pgm.Width();
+   int N = pgm.Height();
+
+   int i = x(1)*N, j = x(0)*M;
+   if (i == N) { i = N-1; }
+   if (j == M) { j = M-1; }
+   i = N-1-i;
+
+   return pgm(i,j)/NC+1;
+}

miniapps/toys/snake.cpp

@@ -0,0 +1,468 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+//
+//           ----------------------------------------------------
+//           Snake Miniapp:  Model of the Rubik's Snake_TM Puzzle
+//           ----------------------------------------------------
+//
+// This miniapp provides a light-hearted example of mesh manipulation and
+// GLVis integration.
+//
+// The Rubik's Snake a.k.a. Twist is a simple tool for experimenting with
+// geometric shapes in 3D. It consists of 24 triangular prisms attached in
+// a row so that neighboring wedges can rotate against each other but cannot
+// be separated. An astonishing variety of different configurations can be
+// reached. Enjoy!
+//
+// Compile with: make snake
+//
+// Sample runs: snake
+//              snake -c 6
+
+#include "mfem.hpp"
+#include "../common/mesh_extras.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+
+static int joint_ = 0;
+static int notch_ = 0;
+static int step_  = 0;
+static int nstep_ = 6;
+
+static double cosa_ = cos(0.5 * M_PI / nstep_);
+static double sina_ = sin(0.5 * M_PI / nstep_);
+
+/** Pre-programmed configurations (feel free to add your own).
+
+    Each configuration must be 23 integers long corresponding to the 23 joints
+    making up the Snake_TM puzzle. The values can be 0-3 indicating how far to
+    rotate the joint in the clockwise direction when looking along the snake
+    from the starting (lower) end. The values 0, 1, 2, and 3 correspond to
+    angles of 0, 90, 180, and 270 degrees respectively.
+*/
+static int conf[][23] =
+{
+   /* 0 - Ball       */ {
+      3,1,3,3,1,3,1,1,
+      3,1,3,3,1,3,1,1,
+      3,1,3,3,1,3,1
+   },
+   /* 1 - Triangle   */ {
+      1,0,0,0,0,0,0,3,
+      1,0,0,0,0,0,0,3,
+      1,0,0,0,0,0,0
+   },
+   /* 2 - Hexagon    */ {
+      0,0,1,0,0,0,0,3,
+      0,0,1,0,0,0,0,3,
+      0,0,1,0,0,0,0
+   },
+   /* 3 - Snow Flake */ {
+      1,1,1,1,3,3,3,3,
+      1,1,1,1,3,3,3,3,
+      1,1,1,1,3,3,3
+   },
+   /* 4 - Spiral     */ {
+      2,1,2,1,2,1,2,1,
+      2,1,2,1,2,1,2,1,
+      2,1,2,1,2,1,2
+   },
+   /* 5 - Zig-zag    */ {
+      3,3,3,1,1,1,3,3,
+      3,1,1,1,3,3,3,1,
+      1,1,3,3,3,1,1
+   },
+   /* 6 - Cobra      */ {
+      2,0,0,2,1,3,0,2,
+      0,2,3,0,1,3,2,1,
+      1,2,3,1,0,3,0
+   },
+   /* 7 - Serenity  */ {
+      3,2,3,2,1,2,0,2,
+      0,2,3,2,1,2,1,2,
+      0,2,3,2,1,2,0
+   },
+   /* 8 - Pinwheel  */ {
+      3,2,1,0,2,3,3,2,
+      1,0,2,3,3,2,1,0,
+      2,3,3,2,1,0,2
+   },
+   /* 9 - Crane     */ {
+      0,0,3,2,0,2,2,0,
+      0,2,0,0,0,2,2,0,
+      0,0,3,0,0,0,2
+   },
+   /* 10 - Snake     */ {
+      0,1,0,1,0,1,0,1,
+      0,1,3,1,0,1,0,1,
+      0,1,0,1,0,1,2
+   },
+   /* 11 - Sculpture */ {
+      0,2,0,2,2,0,3,0,
+      2,2,0,1,0,2,2,0,
+      3,0,2,2,0,2,0
+   },
+   /* 12 - Angles    */ {
+      0,2,0,2,2,0,3,0,
+      2,2,0,0,0,2,2,0,
+      3,0,2,2,0,2,0
+   },
+};
+static int NUM_CONFIGURATIONS = 13;
+
+void trans(const int * conf, Mesh & mesh);
+
+bool anim_step(const int * conf, Mesh & mesh);
+
+int main(int argc, char *argv[])
+{
+   int cfg = -1;
+   bool anim = true;
+   bool user = false;
+   bool visualization = true;
+
+   Array<int> myConf(0);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&cfg, "-c", "--configuration",
+                  "Select one of 13 pre-programmed configurations: 0-12");
+   args.AddOption(&myConf, "-u", "--user-cfg",
+                  "User defined configuration consisting of "
+                  "23 joint positions defined by the integers 0, 1, 2, or 3.");
+   args.AddOption(&anim, "-anim", "--animation", "-no-anim",
+                  "--no-animation",
+                  "Enable or disable GLVis animation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // Test for a user supplied configuration
+   if (myConf.Size() > 0)
+   {
+      user = true;
+
+      if (myConf.Size() != 23)
+      {
+         MFEM_ABORT("Invalid user-defined configuration of length "
+                    << myConf.Size());
+      }
+   }
+
+   // Test for a pre-programmed configuration
+   if (!user && cfg >=0 && cfg < NUM_CONFIGURATIONS)
+   {
+      myConf.SetSize(23);
+      myConf.Assign(conf[cfg]);
+   }
+
+   // Validate the configuration if it has been set
+   for (int i=0; i<myConf.Size(); i++)
+   {
+      if (myConf[i] < 0 || myConf[i] > 3)
+      {
+         MFEM_ABORT("Invalid entry \"" << myConf[i]
+                    << "\"in configuration at position " << i);
+      }
+   }
+
+   if (!visualization) { anim = false; }
+
+   // Define an empty mesh
+   Mesh mesh(3, 6 * 24, 24);
+
+   // Add vertices for 24 elements
+   double c[9];
+   int v[6];
+   for (int i=0; i<12; i++)
+   {
+      // Add vertices for a pair of elements
+      // First Upward-facing wedge
+      c[0] = i-6; c[1] = 0.0; c[2] = i-6;
+      c[3] = i-6; c[4] = 0.0; c[5] = i-5;
+      c[6] = i-5; c[7] = 0.0; c[8] = i-5;
+
+      mesh.AddVertex(&c[0]);
+      mesh.AddVertex(&c[3]);
+      mesh.AddVertex(&c[6]);
+
+      c[1] = 1.0; c[4] = 1.0; c[7] = 1.0;
+      mesh.AddVertex(&c[0]);
+      mesh.AddVertex(&c[3]);
+      mesh.AddVertex(&c[6]);
+
+      for (int j=0; j<6; j++) { v[j] = 12 * i + j; }
+      mesh.AddWedge(v);
+
+      // Next Downward-facing wedge
+      c[0] = i-6; c[1] = 0.0; c[2] = i-5;
+      c[3] = i-5; c[4] = 0.0; c[5] = i-4;
+      c[6] = i-5; c[7] = 0.0; c[8] = i-5;
+
+      mesh.AddVertex(&c[0]);
+      mesh.AddVertex(&c[3]);
+      mesh.AddVertex(&c[6]);
+
+      c[1] = 1.0; c[4] = 1.0; c[7] = 1.0;
+      mesh.AddVertex(&c[0]);
+      mesh.AddVertex(&c[3]);
+      mesh.AddVertex(&c[6]);
+
+      for (int j=0; j<6; j++) { v[j] = 12 * i + j + 6; }
+      mesh.AddWedge(v);
+   }
+
+   mesh.FinalizeTopology();
+
+   // Paint elements with alternating colors
+   FiniteElementCollection *fec = new L2_FECollection(0, 3, 1);
+   FiniteElementSpace fespace(&mesh, fec);
+   GridFunction color(&fespace);
+
+   for (int i=0; i<24; i++) { color[i] = (i%2)?1.0:-1.0; }
+
+   // Output the initial mesh to a file
+   {
+      ostringstream oss;
+      oss << "snake-init.mesh";
+      ofstream ofs(oss.str().c_str());
+      ofs.precision(8);
+      mesh.Print(ofs);
+      ofs.close();
+   }
+
+   // Jump to final configuration if no animation is needed
+   if (myConf.Size() > 0 && !anim) { trans(myConf.GetData(), mesh); }
+
+   // Output the resulting mesh to GLVis
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << color << "keys Am\n"
+               << "palette 22\n" << "valuerange -1.5 1\n"
+               << "autoscale off\n" << flush;
+
+      // Animate the twists of the selected configuration
+      if (myConf.Size() > 0 && anim)
+      {
+         sol_sock << "pause\n" << flush;
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+
+         while (anim_step(myConf.GetData(), mesh))
+         {
+            static int turn = 0;
+            sol_sock << "solution\n" << mesh << color;
+            if (turn++ % 2 == 0)
+            {
+               sol_sock << "pause\n";
+            }
+            sol_sock << flush;
+         }
+      }
+      sol_sock << "autoscale on\n" << "valuerange -1.5 1\n" << flush;
+   }
+
+   // Join the elements together to form a connected mesh
+   MergeMeshNodes(&mesh, 1);
+
+   // Output the resulting mesh to a file
+   {
+      ostringstream oss;
+      if (user)
+      {
+         oss << "snake-user.mesh";
+      }
+      else if (cfg >= 0)
+      {
+         oss << "snake-c" << cfg << ".mesh";
+      }
+      else
+      {
+         oss << "snake-joined.mesh";
+      }
+      ofstream ofs(oss.str().c_str());
+      ofs.precision(8);
+      mesh.Print(ofs);
+      ofs.close();
+   }
+
+   // Clean up and exit
+   return 0;
+}
+
+void
+rotate(double * x)
+{
+   if (notch_ == 0) { return; }
+
+   double cent[3];
+   Vector cVec(cent,3);
+   Vector xVec(x,3);
+
+   if (joint_%2 == 0)
+   {
+      cVec[0] = -5.5 + joint_ / 2; cVec[1] = 0.5; cVec[2] = 0.0;
+      xVec.Add(-1.0, cVec);
+      switch (notch_)
+      {
+         case 1:
+            swap(xVec[0], xVec[1]);
+            xVec[0] *= -1.0;
+            break;
+         case 2:
+            xVec[0] *= -1.0;
+            xVec[1] *= -1.0;
+            break;
+         case 3:
+            swap(xVec[0], xVec[1]);
+            xVec[1] *= -1.0;
+            break;
+      }
+      xVec.Add(1.0, cVec);
+   }
+   else
+   {
+      cVec[0] = 0.0; cVec[1] = 0.5; cVec[2] = -4.5 + joint_ / 2;
+      xVec.Add(-1.0, cVec);
+      switch (notch_)
+      {
+         case 1:
+            swap(xVec[1], xVec[2]);
+            xVec[1] *= -1.0;
+            break;
+         case 2:
+            xVec[1] *= -1.0;
+            xVec[2] *= -1.0;
+            break;
+         case 3:
+            swap(xVec[1], xVec[2]);
+            xVec[2] *= -1.0;
+            break;
+      }
+      xVec.Add(1.0, cVec);
+   }
+}
+
+void
+trans(const int * conf, Mesh & mesh)
+{
+   for (int i=0; i<23; i++)
+   {
+      joint_ = i;
+      notch_ = conf[i];
+
+      if (notch_ != 0)
+      {
+         for (int k=0; k<6*(i+1); k++)
+         {
+            rotate(mesh.GetVertex(k));
+         }
+      }
+   }
+}
+
+void
+rotate_step(double * x)
+{
+   if (notch_ == 0) { return; }
+
+   double cent[3], y[3];
+   Vector cVec(cent,3);
+   Vector xVec(x,3);
+   Vector yVec(y,3);
+
+   if (joint_%2 == 0)
+   {
+      cVec[0] = -5.5 + joint_ / 2; cVec[1] = 0.5; cVec[2] = 0.0;
+      xVec.Add(-1.0, cVec);
+      switch (notch_)
+      {
+         case 1:
+            yVec[0] = cosa_ * xVec[0] - sina_ * xVec[1];
+            yVec[1] = sina_ * xVec[0] + cosa_ * xVec[1];
+            yVec[2] = xVec[2];
+            break;
+         case 2:
+            yVec[0] = cosa_ * xVec[0] - sina_ * xVec[1];
+            yVec[1] = sina_ * xVec[0] + cosa_ * xVec[1];
+            yVec[2] = xVec[2];
+            break;
+         case 3:
+            yVec[0] =  cosa_ * xVec[0] + sina_ * xVec[1];
+            yVec[1] = -sina_ * xVec[0] + cosa_ * xVec[1];
+            yVec[2] = xVec[2];
+            break;
+      }
+      add(yVec, 1.0, cVec, xVec);
+   }
+   else
+   {
+      cVec[0] = 0.0; cVec[1] = 0.5; cVec[2] = -4.5 + joint_ / 2;
+      xVec.Add(-1.0, cVec);
+      switch (notch_)
+      {
+         case 1:
+            yVec[0] = xVec[0];
+            yVec[1] = cosa_ * xVec[1] - sina_ * xVec[2];
+            yVec[2] = sina_ * xVec[1] + cosa_ * xVec[2];
+            break;
+         case 2:
+            yVec[0] = xVec[0];
+            yVec[1] = cosa_ * xVec[1] - sina_ * xVec[2];
+            yVec[2] = sina_ * xVec[1] + cosa_ * xVec[2];
+            break;
+         case 3:
+            yVec[0] = xVec[0];
+            yVec[1] =  cosa_ * xVec[1] + sina_ * xVec[2];
+            yVec[2] = -sina_ * xVec[1] + cosa_ * xVec[2];
+            break;
+      }
+      add(yVec, 1.0, cVec, xVec);
+   }
+}
+
+bool
+anim_step(const int * conf, Mesh & mesh)
+{
+   if (notch_ == 2 && step_ == 2 * nstep_) { joint_++; step_ = 0; }
+   if (notch_ != 2 && step_ == nstep_) { joint_++; step_ = 0; }
+   if (joint_ == 23) { return false; }
+   notch_ = conf[joint_];
+
+   if (notch_ == 0)
+   {
+      step_ = nstep_;
+      return true;
+   }
+   else
+   {
+      for (int k=0; k<6*(joint_+1); k++)
+      {
+         rotate_step(mesh.GetVertex(k));
+      }
+   }
+   step_++;
+   return true;
+}

tests/unit/CMakeLists.txt

@@ -23,6 +23,7 @@ set(UNIT_TESTS_SRCS
   linalg/test_blockMatrix.cpp
   linalg/test_complex_operator.cpp
   linalg/test_densematrix.cpp
+  linalg/test_ilu.cpp
   linalg/test_ode.cpp
   mesh/test_mesh.cpp
   fem/test_1d_bilininteg.cpp
@@ -36,6 +37,8 @@ set(UNIT_TESTS_SRCS
   fem/test_inversetransform.cpp
   fem/test_lin_interp.cpp
   fem/test_linear_fes.cpp
+  fem/test_pa_coeff.cpp
+  fem/test_pa_kernels.cpp
   fem/test_quadraturefunc.cpp
   )
 

tests/unit/fem/test_pa_coeff.cpp

@@ -9,8 +9,8 @@
 // terms of the GNU Lesser General Public License (as published by the Free
 // Software Foundation) version 2.1 dated February 1999.
 
-#include "catch.hpp"
 #include "mfem.hpp"
+#include "catch.hpp"
 
 using namespace mfem;
 

tests/unit/fem/test_pa_kernels.cpp

@@ -353,4 +353,97 @@ TEST_CASE("PA Vector Diffusion", "[PartialAssembly], [VectorPA]")
    }
 }
 
+//test convection
+int dimension;
+
+// Velocity coefficient
+void velocity_function(const Vector &x, Vector &v)
+{
+
+   if (dimension == 2)
+   {
+      v(0) = sqrt(2./3.); v(1) = sqrt(1./3.);
+   }
+
+   if (dimension == 3)
+   {
+      v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
+   }
+
+}
+
+//Basic unit test for convection
+TEST_CASE("PA Convection")
+{
+
+   for (dimension = 2; dimension < 4; ++dimension)
+   {
+
+      for (int imesh = 0; imesh<2; ++imesh)
+      {
+
+         const char *mesh_file;
+         if (dimension == 2)
+         {
+
+            switch (imesh)
+            {
+               case 0: mesh_file = "../../data/periodic-square.mesh"; break;
+               case 1: mesh_file = "../../data/amr-quad.mesh"; break;
+            }
+         }
+
+         if (dimension == 3)
+         {
+            switch (imesh)
+            {
+               case 0: mesh_file = "../../data/periodic-cube.mesh"; break;
+               case 1: mesh_file = "../../data/amr-hex.mesh"; break;
+            }
+         }
+
+         Mesh *mesh = new Mesh(mesh_file, 1, 1);
+         for (int order = 1; order < 5; ++order)
+         {
+
+            H1_FECollection *fec = new H1_FECollection(order, dimension);
+            FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+
+            BilinearForm k(fespace);
+            BilinearForm pak(fespace); //Partial assembly version of k
+
+            VectorFunctionCoefficient velocity(dimension, velocity_function);
+
+            k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+            pak.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+
+            int skip_zeros = 0;
+            k.Assemble(skip_zeros);
+            k.Finalize(skip_zeros);
+
+            pak.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+            pak.Assemble();
+
+            Vector x(k.Size());
+            Vector y(k.Size()), y_pa(k.Size());
+
+            for (int i=0; i<x.Size(); ++i) {x(i) = i/10.0;};
+
+            pak.Mult(x,y_pa);
+            k.Mult(x,y);
+
+            y_pa -= y;
+            double pa_error =- y_pa.Norml2();
+            std::cout << "ConvectionIntegrator:"
+                      << " dim = " << dimension
+                      << ", conforming = " << imesh
+                      << ", order = " << order
+                      << ", PA error = " << pa_error << std::endl;
+            REQUIRE(fabs(pa_error) < 1.e-12);
+         }//order loop
+      }//mesh loop
+   }//dimension loop
+
+}//test case
+
 }// namespace pa_kernels

tests/unit/linalg/test_densematrix.cpp

@@ -14,6 +14,69 @@
 
 using namespace mfem;
 
+TEST_CASE("DenseMatrix LinearSolve methods",
+          "[DenseMatrix]")
+{
+   SECTION("singular_system")
+   {
+      constexpr int N = 3;
+
+      DenseMatrix A(N);
+      A.SetRow(0, 0.0);
+      A.SetRow(1, 0.0);
+      A.SetRow(2, 0.0);
+
+      double X[3];
+
+      REQUIRE_FALSE(LinearSolve(A,X));
+   }
+
+   SECTION("1x1_system")
+   {
+      constexpr int N = 1;
+      DenseMatrix A(N);
+      A(0,0) = 2;
+
+      double X[1] = { 12 };
+
+      REQUIRE(LinearSolve(A,X));
+      REQUIRE(X[0] == Approx(6));
+   }
+
+   SECTION("2x2_system")
+   {
+      constexpr int N = 2;
+
+      DenseMatrix A(N);
+      A(0,0) = 2.0; A(0,1) = 1.0;
+      A(1,0) = 3.0; A(1,1) = 4.0;
+
+      double X[2] = { 1, 14 };
+
+      REQUIRE(LinearSolve(A,X));
+      REQUIRE(X[0] == Approx(-2));
+      REQUIRE(X[1] == Approx(5));
+   }
+
+   SECTION("3x3_system")
+   {
+      constexpr int N = 3;
+
+      DenseMatrix A(N);
+      A(0,0) = 4; A(0,1) =  5; A(0,2) = -2;
+      A(1,0) = 7; A(1,1) = -1; A(1,2) =  2;
+      A(2,0) = 3; A(2,1) =  1; A(2,2) =  4;
+
+      double X[3] = { -14, 42, 28 };
+
+      REQUIRE(LinearSolve(A,X));
+      REQUIRE(X[0] == Approx(4));
+      REQUIRE(X[1] == Approx(-4));
+      REQUIRE(X[2] == Approx(5));
+   }
+
+}
+
 TEST_CASE("DenseMatrix A*B^T methods",
           "[DenseMatrix]")
 {
@@ -51,13 +114,13 @@ TEST_CASE("DenseMatrix A*B^T methods",
       MultABt(A, B, C);
       C.Add(-1.0, Cexact);
 
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
 
       Mult(A, Bt, Cexact);
       MultABt(A, B, C);
       C.Add(-1.0, Cexact);
 
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
    }
    SECTION("MultADBt")
    {
@@ -74,7 +137,7 @@ TEST_CASE("DenseMatrix A*B^T methods",
       MultADBt(A, D, B, C);
       C.Add(-1.0, Cexact);
 
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
    }
    SECTION("AddMultABt")
    {
@@ -91,12 +154,12 @@ TEST_CASE("DenseMatrix A*B^T methods",
       AddMultABt(A, B, C);
       C.Add(-1.0, Cexact);
 
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
 
       MultABt(A, B, C);
       C *= -1.0;
       AddMultABt(A, B, C);
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
    }
    SECTION("AddMultADBt")
    {
@@ -116,18 +179,18 @@ TEST_CASE("DenseMatrix A*B^T methods",
       AddMultADBt(A, D, B, C);
       C.Add(-1.0, Cexact);
 
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
 
       MultADBt(A, D, B, C);
       C *= -1.0;
       AddMultADBt(A, D, B, C);
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
 
       DData[0] = 1.0; DData[1] = 1.0; DData[2] = 1.0;
       MultABt(A, B, C);
       C *= -1.0;
       AddMultADBt(A, D, B, C);
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
    }
    SECTION("AddMult_a_ABt")
    {
@@ -146,12 +209,39 @@ TEST_CASE("DenseMatrix A*B^T methods",
       AddMult_a_ABt(a, A, B, C);
       C.Add(-1.0, Cexact);
 
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
 
       MultABt(A, B, C);
       AddMult_a_ABt(-1.0, A, B, C);
 
-      REQUIRE( C.MaxMaxNorm() < tol );
+      REQUIRE(C.MaxMaxNorm() < tol);
    }
 }
 
+
+TEST_CASE("LUFactors RightSolve", "[DenseMatrix]")
+{
+   double tol = 1e-12;
+
+   // Zero on diagonal forces non-trivial pivot
+   double AData[9] = { 0.0, 0.0, 3.0, 2.0, 2.0, 2.0, 2.0, 0.0, 4.0 };
+   double BData[6] = { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 };
+   int ipiv[3];
+
+   DenseMatrix A(AData, 3, 3);
+   DenseMatrix B(BData, 2, 3);
+
+   DenseMatrixInverse Af1(A);
+   DenseMatrix Ainv;
+   Af1.GetInverseMatrix(Ainv);
+
+   LUFactors Af2(AData, ipiv);
+   Af2.Factor(3);
+
+   DenseMatrix C(2,3);
+   Mult(B, Ainv, C);
+   Af2.RightSolve(3, 2, B.GetData());
+   C -= B;
+
+   REQUIRE(C.MaxMaxNorm() < tol);
+}

tests/unit/linalg/test_ilu.cpp

@@ -0,0 +1,171 @@
+// Copyright (c) 2019, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "catch.hpp"
+#include "mfem.hpp"
+
+using namespace mfem;
+
+TEST_CASE("ILU Structure", "[ILU]")
+{
+   int N = 5;
+   int Nb = 3;
+   int nnz_blocks = 11;
+
+   // Submatrix of size Nb x Nb
+   DenseMatrix Ab(Nb, Nb);
+
+   // Matrix with N x N blocks of size Nb x Nb
+   SparseMatrix A(N * Nb, N * Nb);
+   // Create a SparseMatrix that has a block structure looking like
+   //    {{1, 1, 0, 0, 1},
+   //     {0, 1, 0, 1, 1},
+   //     {0, 0, 1, 0, 0},
+   //     {0, 1, 0, 1, 0},
+   //     {1, 0, 0, 0, 1}}
+   // Where 1 represents a block of size Nb x Nb that is non zero.
+
+   // Lexicographical pattern
+   int p[] =
+   {
+      1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1,
+      0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1
+   };
+
+   Array<int> pattern(p, N * N);
+   int counter = 1;
+   for (int i = 0; i < N; ++i)
+   {
+      for (int j = 0; j < N; ++j)
+      {
+         if (pattern[N * i + j] == 1)
+         {
+            Array<int> rows, cols;
+
+            for (int ii = 0; ii < Nb; ++ii)
+            {
+               rows.Append(i * Nb + ii);
+               cols.Append(j * Nb + ii);
+            }
+
+            Vector Ab_data(Ab.GetData(), Nb * Nb);
+            Ab_data.Randomize(++counter);
+            A.SetSubMatrix(rows, cols, Ab);
+         }
+      }
+   }
+
+   A.Finalize();
+
+   SECTION("Create block pattern from SparseMatrix")
+   {
+      BlockILU ilu(A, Nb, BlockILU::Reordering::NONE);
+
+      int *IB = ilu.GetBlockI();
+      int *JB = ilu.GetBlockJ();
+
+      int nnz_count = 0;
+
+      for (int i = 0; i < N; ++i)
+      {
+         for (int k = IB[i]; k < IB[i + 1]; ++k)
+         {
+            int j = JB[k];
+            // Check if the non zero block is expected
+            REQUIRE(pattern[i * N + j] == 1);
+            nnz_count++;
+         }
+      }
+      // Check if the number of expected non zero blocks matches
+      REQUIRE(nnz_count == nnz_blocks);
+   }
+}
+
+TEST_CASE("ILU Factorization", "[ILU]")
+{
+   SparseMatrix A(6, 6);
+
+   A.Set(0,0,1);
+   A.Set(0,1,2);
+   A.Set(0,2,3);
+   A.Set(0,3,4);
+   A.Set(0,4,5);
+   A.Set(0,5,6);
+
+   A.Set(1,0,7);
+   A.Set(1,1,8);
+   A.Set(1,2,9);
+   A.Set(1,3,1);
+   A.Set(1,4,2);
+   A.Set(1,5,3);
+
+   A.Set(2,0,4);
+   A.Set(2,1,5);
+   A.Set(2,2,6);
+   A.Set(2,3,7);
+
+   A.Set(3,0,8);
+   A.Set(3,1,9);
+   A.Set(3,2,1);
+   A.Set(3,3,2);
+
+   A.Set(4,0,3);
+   A.Set(4,1,4);
+   A.Set(4,4,5);
+   A.Set(4,5,6);
+
+   A.Set(5,0,7);
+   A.Set(5,1,8);
+   A.Set(5,4,9);
+   A.Set(5,5,1);
+
+   A.Finalize();
+
+   BlockILU ilu(A, 2, BlockILU::Reordering::MINIMUM_DISCARDED_FILL);
+
+   DenseTensor AB;
+   AB.UseExternalData(ilu.GetBlockData(), 2, 2, 7);
+
+   REQUIRE(AB(0,0,0) == Approx(6.0));
+   REQUIRE(AB(1,0,0) == Approx(1.0));
+   REQUIRE(AB(0,1,0) == Approx(7.0));
+   REQUIRE(AB(1,1,0) == Approx(2.0));
+
+   REQUIRE(AB(0,0,1) == Approx(4.0));
+   REQUIRE(AB(1,0,1) == Approx(8.0));
+   REQUIRE(AB(0,1,1) == Approx(5.0));
+   REQUIRE(AB(1,1,1) == Approx(9.0));
+
+   REQUIRE(AB(0,0,2) == Approx(0.4));
+   REQUIRE(AB(1,0,2) == Approx(3.4));
+   REQUIRE(AB(0,1,2) == Approx(0.6));
+   REQUIRE(AB(1,1,2) == Approx(-11.4));
+
+   REQUIRE(AB(0,0,3) == Approx(-5.4));
+   REQUIRE(AB(1,0,3) == Approx(84.6));
+   REQUIRE(AB(0,1,3) == Approx(-5.4));
+   REQUIRE(AB(1,1,3) == Approx(93.6));
+
+   REQUIRE(AB(0,0,4) == Approx(5.0));
+   REQUIRE(AB(1,0,4) == Approx(2.0));
+   REQUIRE(AB(0,1,4) == Approx(6.0));
+   REQUIRE(AB(1,1,4) == Approx(3.0));
+
+   REQUIRE(AB(0,0,5) == Approx(32.0/27.0));
+   REQUIRE(AB(1,0,5) == Approx(4.0/9.0));
+   REQUIRE(AB(0,1,5) == Approx(1.0/9.0));
+   REQUIRE(AB(1,1,5) == Approx(1.0/9.0));
+
+   REQUIRE(AB(0,0,6) == Approx(-31.0/27.0));
+   REQUIRE(AB(1,0,6) == Approx(59.0/9.0));
+   REQUIRE(AB(0,1,6) == Approx(-13.0/9.0));
+   REQUIRE(AB(1,1,6) == Approx(-2.0));
+}