more temporary memory in tmop

use temp mem in tmop+pa

.gitignore

@@ -253,6 +253,15 @@ tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
+tests/unit/tmop_tests_*
+tests/unit/ptmop_tests_*
+tests/unit/cube.mesh
+tests/unit/star.mesh
+tests/unit/blade.mesh
+tests/unit/square01.mesh
+tests/unit/toroid-hex.mesh
+tests/unit/beam-hex-nurbs.mesh
+tests/unit/square-disc-nurbs.mesh
 
 # Test script output
 tests/scripts/*.err

CHANGELOG

@@ -50,6 +50,11 @@ Performance improvements
 Improved GPU capabilities
 -------------------------
 - Added support for Chebyshev accelerated polynomial smoother on GPU.
+- The TMOP mesh optimization algorithms were extended to GPU:
+   - QualityMetric #1, #2 and #7 are available in 2D, #302, #303 and #321 in 3D
+   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+   - Kernels for normalization and limiting have been added
+   - The AdvectorCG now also support AssemblyLevel::PARTIAL
 
 - Optimized AMD/HIP kernel support.
 

config/defaults.mk

@@ -138,6 +138,7 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
+MFEM_USE_CAMP          = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 
@@ -341,9 +342,9 @@ GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
 GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
-# CUDA library configuration (currently not needed)
+# CUDA library configuration
 CUDA_OPT =
-CUDA_LIB =
+CUDA_LIB = -lcusparse
 
 # HIP library configuration (currently not needed)
 HIP_OPT =
@@ -372,6 +373,11 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
 
+# CAMP library configuration
+CAMP_DIR = @MFEM_DIR@/../camp
+CAMP_OPT = -I$(CAMP_DIR)/include
+CAMP_LIB = -L$(CAMP_DIR)/lib
+
 # If YES, enable some informational messages
 VERBOSE = NO
 

fem/CMakeLists.txt

@@ -50,10 +50,43 @@ set(SRCS
   fespacehierarchy.cpp
   nonlininteg_vectorconvection.cpp
   quadinterpolator.cpp
+  quadinterpolator_det.cpp
+  quadinterpolator_eval_by_nodes.cpp
+  quadinterpolator_eval_by_vdim.cpp
+  quadinterpolator_grad_by_nodes.cpp
+  quadinterpolator_grad_by_vdim.cpp
+  quadinterpolator_grad_phys_by_nodes.cpp
+  quadinterpolator_grad_phys_by_vdim.cpp
   quadinterpolator_face.cpp
   restriction.cpp
   staticcond.cpp
   tmop.cpp
+  tmop_pa.cpp
+  tmop_pa_h2d.cpp
+  tmop_pa_h2d_c0.cpp
+  tmop_pa_h2m.cpp
+  tmop_pa_h2m_c0.cpp
+  tmop_pa_h2s.cpp
+  tmop_pa_h2s_c0.cpp
+  tmop_pa_h3d.cpp
+  tmop_pa_h3d_c0.cpp
+  tmop_pa_h3m.cpp
+  tmop_pa_h3m_c0.cpp
+  tmop_pa_h3s.cpp
+  tmop_pa_h3s_c0.cpp
+  tmop_pa_jp2.cpp
+  tmop_pa_jp3.cpp
+  tmop_pa_jt2_tc.cpp
+  tmop_pa_jt3_datc.cpp
+  tmop_pa_jt3_tc.cpp
+  tmop_pa_p2.cpp
+  tmop_pa_p2_c0.cpp
+  tmop_pa_p3.cpp
+  tmop_pa_p3_c0.cpp
+  tmop_pa_w2.cpp
+  tmop_pa_w2_c0.cpp
+  tmop_pa_w3.cpp
+  tmop_pa_w3_c0.cpp
   tmop_tools.cpp
   gslib.cpp
   transfer.cpp
@@ -83,7 +116,10 @@ set(HDRS
   nonlinearform_ext.hpp
   nonlininteg.hpp
   quadinterpolator.hpp
+  quadinterpolator_eval.hpp
   quadinterpolator_face.hpp
+  quadinterpolator_grad.hpp
+  quadinterpolator_grad_phys.hpp
   restriction.hpp
   fespacehierarchy.hpp
   staticcond.hpp
@@ -96,6 +132,7 @@ set(HDRS
   tfespace.hpp
   tintrules.hpp
   tmop.hpp
+  tmop_pa.hpp
   tmop_tools.hpp
   gslib.hpp
   transfer.hpp

fem/bilininteg_convection_pa.cpp

@@ -13,6 +13,10 @@
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
 
+#include "restriction.hpp"
+#include "tmop_pa.hpp"
+#include "../linalg/kernels.hpp"
+
 using namespace std;
 
 namespace mfem
@@ -68,47 +72,53 @@ static void PAConvectionSetup3D(const int Q1D,
                                 const double alpha,
                                 Vector &op)
 {
-   const int NQ = Q1D*Q1D*Q1D;
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
+   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
    const bool const_v = vel.Size() == 3;
-   auto V =
-      const_v ? Reshape(vel.Read(), 3,1,1) : Reshape(vel.Read(), 3,NQ,NE);
-   auto y = Reshape(op.Write(), NQ, 3, NE);
-   MFEM_FORALL(e, NE,
+   const auto V = const_v ?
+                  Reshape(vel.Read(), 3,1,1,1,1) :
+                  Reshape(vel.Read(), 3,Q1D,Q1D,Q1D,NE);
+   auto y = Reshape(op.Write(), Q1D,Q1D,Q1D,3,NE);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
    {
-      for (int q = 0; q < NQ; ++q)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double w = alpha * W[q];
-         const double v0 = const_v ? V(0,0,0) : V(0,q,e);
-         const double v1 = const_v ? V(1,0,0) : V(1,q,e);
-         const double v2 = const_v ? V(2,0,0) : V(2,q,e);
-         const double wx = w * v0;
-         const double wy = w * v1;
-         const double wz = w * v2;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-         // q . J^{-1} = q . adj(J)
-         y(q,0,e) =  wx * A11 + wy * A12 + wz * A13;
-         y(q,1,e) =  wx * A21 + wy * A22 + wz * A23;
-         y(q,2,e) =  wx * A31 + wy * A32 + wz * A33;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               const double J11 = J(qx,qy,qz,0,0,e);
+               const double J12 = J(qx,qy,qz,0,1,e);
+               const double J13 = J(qx,qy,qz,0,2,e);
+               const double J21 = J(qx,qy,qz,1,0,e);
+               const double J22 = J(qx,qy,qz,1,1,e);
+               const double J23 = J(qx,qy,qz,1,2,e);
+               const double J31 = J(qx,qy,qz,2,0,e);
+               const double J32 = J(qx,qy,qz,2,1,e);
+               const double J33 = J(qx,qy,qz,2,2,e);
+               const double w = alpha * W(qx,qy,qz);
+               const double v0 = const_v ? V(0,0,0,0,0) : V(0,qx,qy,qz,e);
+               const double v1 = const_v ? V(1,0,0,0,0) : V(1,qx,qy,qz,e);
+               const double v2 = const_v ? V(2,0,0,0,0) : V(2,qx,qy,qz,e);
+               const double wx = w * v0;
+               const double wy = w * v1;
+               const double wz = w * v2;
+               // adj(J)
+               const double A11 = (J22 * J33) - (J23 * J32);
+               const double A12 = (J32 * J13) - (J12 * J33);
+               const double A13 = (J12 * J23) - (J22 * J13);
+               const double A21 = (J31 * J23) - (J21 * J33);
+               const double A22 = (J11 * J33) - (J13 * J31);
+               const double A23 = (J21 * J13) - (J11 * J23);
+               const double A31 = (J21 * J32) - (J31 * J22);
+               const double A32 = (J31 * J12) - (J11 * J32);
+               const double A33 = (J11 * J22) - (J12 * J21);
+               // q . J^{-1} = q . adj(J)
+               y(qx,qy,qz,0,e) = wx * A11 + wy * A12 + wz * A13;
+               y(qx,qy,qz,1,e) = wx * A21 + wy * A22 + wz * A23;
+               y(qx,qy,qz,2,e) = wx * A31 + wy * A32 + wz * A33;
+            }
+         }
       }
    });
 }
@@ -184,8 +194,8 @@ void PAConvectionApply2D(const int ne,
             Gu[dy][qx] = 0.0;
             for (int dx = 0; dx < D1D; ++dx)
             {
-               const double bx  = B(qx,dx);
-               const double gx  = G(qx,dx);
+               const double bx = B(qx,dx);
+               const double gx = G(qx,dx);
                const double x = u[dy][dx];
                Bu[dy][qx] += bx * x;
                Gu[dy][qx] += gx * x;
@@ -202,8 +212,8 @@ void PAConvectionApply2D(const int ne,
             BGu[qy][qx] = 0.0;
             for (int dy = 0; dy < D1D; ++dy)
             {
-               const double bx  = B(qy,dy);
-               const double gx  = G(qy,dy);
+               const double bx = B(qy,dy);
+               const double gx = G(qy,dy);
                GBu[qy][qx] += gx * Bu[dy][qx];
                BGu[qy][qx] += bx * Gu[dy][qx];
             }
@@ -232,7 +242,7 @@ void PAConvectionApply2D(const int ne,
             BDGu[dy][qx] = 0.0;
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               const double w  = Bt(dy,qy);
+               const double w = Bt(dy,qy);
                BDGu[dy][qx] += w * DGu[qy][qx];
             }
          }
@@ -244,7 +254,7 @@ void PAConvectionApply2D(const int ne,
             double BBDGu = 0.0;
             for (int qx = 0; qx < Q1D; ++qx)
             {
-               const double w  = Bt(dx,qx);
+               const double w = Bt(dx,qx);
                BBDGu += w * BDGu[dy][qx];
             }
             y(dx,dy,e) += BBDGu;
@@ -310,7 +320,7 @@ void SmemPAConvectionApply2D(const int ne,
             {
                const double bx = B(qx,dx);
                const double gx = G(qx,dx);
-               const double x  = u[tidz][dy][dx];
+               const double x = u[tidz][dy][dx];
                Bu[tidz][dy][qx] += bx * x;
                Gu[tidz][dy][qx] += gx * x;
             }
@@ -327,8 +337,8 @@ void SmemPAConvectionApply2D(const int ne,
             BGu[tidz][qy][qx] = 0.0;
             for (int dy = 0; dy < D1D; ++dy)
             {
-               const double bx  = B(qy,dy);
-               const double gx  = G(qy,dy);
+               const double bx = B(qy,dy);
+               const double gx = G(qy,dy);
                GBu[tidz][qy][qx] += gx * Bu[tidz][dy][qx];
                BGu[tidz][qy][qx] += bx * Gu[tidz][dy][qx];
             }
@@ -359,7 +369,7 @@ void SmemPAConvectionApply2D(const int ne,
             BDGu[tidz][dy][qx] = 0.0;
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               const double w  = Bt(dy,qy);
+               const double w = Bt(dy,qy);
                BDGu[tidz][dy][qx] += w * DGu[tidz][qy][qx];
             }
          }
@@ -372,7 +382,7 @@ void SmemPAConvectionApply2D(const int ne,
             double BBDGu = 0.0;
             for (int qx = 0; qx < Q1D; ++qx)
             {
-               const double w  = Bt(dx,qx);
+               const double w = Bt(dx,qx);
                BBDGu += w * BDGu[tidz][dy][qx];
             }
             y(dx,dy,e) += BBDGu;
@@ -436,8 +446,8 @@ void PAConvectionApply3D(const int ne,
                Gu[dz][dy][qx] = 0.0;
                for (int dx = 0; dx < D1D; ++dx)
                {
-                  const double bx  = B(qx,dx);
-                  const double gx  = G(qx,dx);
+                  const double bx = B(qx,dx);
+                  const double gx = G(qx,dx);
                   const double x = u[dz][dy][dx];
                   Bu[dz][dy][qx] += bx * x;
                   Gu[dz][dy][qx] += gx * x;
@@ -459,8 +469,8 @@ void PAConvectionApply3D(const int ne,
                BGu[dz][qy][qx] = 0.0;
                for (int dy = 0; dy < D1D; ++dy)
                {
-                  const double bx  = B(qy,dy);
-                  const double gx  = G(qy,dy);
+                  const double bx = B(qy,dy);
+                  const double gx = G(qy,dy);
                   BBu[dz][qy][qx] += bx * Bu[dz][dy][qx];
                   GBu[dz][qy][qx] += gx * Bu[dz][dy][qx];
                   BGu[dz][qy][qx] += bx * Gu[dz][dy][qx];
@@ -482,8 +492,8 @@ void PAConvectionApply3D(const int ne,
                BBGu[qz][qy][qx] = 0.0;
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  const double bx  = B(qz,dz);
-                  const double gx  = G(qz,dz);
+                  const double bx = B(qz,dz);
+                  const double gx = G(qz,dz);
                   GBBu[qz][qy][qx] += gx * BBu[dz][qy][qx];
                   BGBu[qz][qy][qx] += bx * GBu[dz][qy][qx];
                   BBGu[qz][qy][qx] += bx * BGu[dz][qy][qx];
@@ -521,7 +531,7 @@ void PAConvectionApply3D(const int ne,
                BDGu[dz][qy][qx] = 0.0;
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  const double w  = Bt(dz,qz);
+                  const double w = Bt(dz,qz);
                   BDGu[dz][qy][qx] += w * DGu[qz][qy][qx];
                }
             }
@@ -537,7 +547,7 @@ void PAConvectionApply3D(const int ne,
                BBDGu[dz][dy][qx] = 0.0;
                for (int qy = 0; qy < Q1D; ++qy)
                {
-                  const double w  = Bt(dy,qy);
+                  const double w = Bt(dy,qy);
                   BBDGu[dz][dy][qx] += w * BDGu[dz][qy][qx];
                }
             }
@@ -552,7 +562,7 @@ void PAConvectionApply3D(const int ne,
                double BBBDGu = 0.0;
                for (int qx = 0; qx < Q1D; ++qx)
                {
-                  const double w  = Bt(dx,qx);
+                  const double w = Bt(dx,qx);
                   BBBDGu += w * BBDGu[dz][dy][qx];
                }
                y(dx,dy,dz,e) += BBBDGu;
@@ -625,8 +635,8 @@ void SmemPAConvectionApply3D(const int ne,
                double Gu_ = 0.0;
                for (int dx = 0; dx < D1D; ++dx)
                {
-                  const double bx  = B(qx,dx);
-                  const double gx  = G(qx,dx);
+                  const double bx = B(qx,dx);
+                  const double gx = G(qx,dx);
                   const double x = u[dz][dy][dx];
                   Bu_ += bx * x;
                   Gu_ += gx * x;
@@ -651,8 +661,8 @@ void SmemPAConvectionApply3D(const int ne,
                double BGu_ = 0.0;
                for (int dy = 0; dy < D1D; ++dy)
                {
-                  const double bx  = B(qy,dy);
-                  const double gx  = G(qy,dy);
+                  const double bx = B(qy,dy);
+                  const double gx = G(qy,dy);
                   BBu_ += bx * Bu[dz][dy][qx];
                   GBu_ += gx * Bu[dz][dy][qx];
                   BGu_ += bx * Gu[dz][dy][qx];
@@ -678,8 +688,8 @@ void SmemPAConvectionApply3D(const int ne,
                double BBGu_ = 0.0;
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  const double bx  = B(qz,dz);
-                  const double gx  = G(qz,dz);
+                  const double bx = B(qz,dz);
+                  const double gx = G(qz,dz);
                   GBBu_ += gx * BBu[dz][qy][qx];
                   BGBu_ += bx * GBu[dz][qy][qx];
                   BBGu_ += bx * BGu[dz][qy][qx];
@@ -721,7 +731,7 @@ void SmemPAConvectionApply3D(const int ne,
                double BDGu_ = 0.0;
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  const double w  = Bt(dz,qz);
+                  const double w = Bt(dz,qz);
                   BDGu_ += w * DGu[qz][qy][qx];
                }
                BDGu[dz][qy][qx] = BDGu_;
@@ -739,7 +749,7 @@ void SmemPAConvectionApply3D(const int ne,
                double BBDGu_ = 0.0;
                for (int qy = 0; qy < Q1D; ++qy)
                {
-                  const double w  = Bt(dy,qy);
+                  const double w = Bt(dy,qy);
                   BBDGu_ += w * BDGu[dz][qy][qx];
                }
                BBDGu[dz][dy][qx] = BBDGu_;
@@ -756,7 +766,7 @@ void SmemPAConvectionApply3D(const int ne,
                double BBBDGu = 0.0;
                for (int qx = 0; qx < Q1D; ++qx)
                {
-                  const double w  = Bt(dx,qx);
+                  const double w = Bt(dx,qx);
                   BBBDGu += w * BBDGu[dz][dy][qx];
                }
                y(dx,dy,dz,e) = BBBDGu;
@@ -766,6 +776,117 @@ void SmemPAConvectionApply3D(const int ne,
    });
 }
 
+template<int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0, int T_MAX = 0>
+static void QEvalVGF2D(const int NE,
+                       const double *b_,
+                       const double *x_,
+                       double *y_,
+                       const int vdim = 1,
+                       const int d1d = 0,
+                       const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto X = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto C = Reshape(y_, VDIM, Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
+   {
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      mfem::kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+
+      MFEM_SHARED double DD[NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[NBZ][MQ1*MQ1];
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         mfem::kernels::LoadX<MD1,NBZ>(e,D1D,c,X,DD);
+         mfem::kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,DD,DQ);
+         mfem::kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ,QQ);
+
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               double G;
+               mfem::kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,G);
+               C(c,qx,qy,e) = G;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0, int T_MAX = 0>
+static void QEvalVGF3D(const int NE,
+                       const double *b_,
+                       const double *x_,
+                       double *y_,
+                       const int vdim = 1,
+                       const int d1d = 0,
+                       const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto X = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto C = Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      mfem::kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+
+      MFEM_SHARED double DDD[MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         mfem::kernels::LoadX<MD1>(e,D1D,c,X,DDD);
+         mfem::kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD,DDQ);
+         mfem::kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ,DQQ);
+         mfem::kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ,QQQ);
+
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qz,z,Q1D)
+               {
+                  double G;
+                  mfem::kernels::PullEval<MQ1>(qx,qy,qz,QQQ,G);
+                  C(c,qx,qy,qz,e) = G;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
 void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    // Assumes tensor-product elements
@@ -778,16 +899,90 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    const int nq = ir->GetNPoints();
    dim = mesh->Dimension();
    ne = fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const DofToQuad::Mode mode = DofToQuad::TENSOR;
+#ifdef MFEM_USE_UMPIRE
+   const MemoryType temp_type = Device::GetDeviceMemoryType() == MemoryType::DEVICE_UMPIRE
+      ? MemoryType::DEVICE_UMPIRE_2 : Device::GetDeviceMemoryType();
+#else
+   const MemoryType temp_type = Device::GetDeviceMemoryType();
+#endif
+
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode, temp_type);
+   maps = &el.GetDofToQuad(*ir, mode);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
-   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
+   pa_data.SetSize(symmDims * nq * ne, temp_type);
    Vector vel;
-   if (VectorConstantCoefficient *cQ = dynamic_cast<VectorConstantCoefficient*>(Q))
+   if (VectorConstantCoefficient *cQ =
+          dynamic_cast<VectorConstantCoefficient*>(Q))
    {
       vel = cQ->GetVec();
    }
+   else if (VectorGridFunctionCoefficient *vgfQ =
+               dynamic_cast<VectorGridFunctionCoefficient*>(Q))
+   {
+      Vector xe;
+      vel.SetSize(dim * nq * ne, temp_type);
+
+      const GridFunction *gf = vgfQ->GetGridFunction();
+      const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+      const FiniteElementSpace &gf_fes = *gf->FESpace();
+
+      const int vdim = gf_fes.GetVDim();
+      const Operator *R = gf_fes.GetElementRestriction(ordering);
+      const FiniteElement &el_gf = *gf_fes.GetFE(0);
+      const DofToQuad *maps_gf = &el_gf.GetDofToQuad(*ir, mode);
+      const int D1D = maps_gf->ndof;
+      const int Q1D = maps_gf->nqpt;
+
+      MFEM_VERIFY(R,"");
+      MFEM_VERIFY(vdim == dim, "");
+      MFEM_VERIFY(dim==2 || dim==3,"");
+
+      xe.SetSize(R->Height(), Device::GetMemoryType());
+      xe.UseDevice(true);
+      R->Mult(*gf, xe);
+
+      const auto B = maps_gf->B.Read();
+      const auto x = xe.Read();
+      auto y = vel.Write();
+
+      const int id = (D1D << 4 ) | Q1D;
+      if (dim == 2)
+      {
+         switch (id)
+         {
+            case 0x22: QEvalVGF2D<2,2,2>(ne,B,x,y); break;
+            case 0x33: QEvalVGF2D<2,3,3>(ne,B,x,y); break;
+            case 0x34: QEvalVGF2D<2,3,4>(ne,B,x,y); break;
+            default:
+            {
+               constexpr int MAX_DQ = 8;
+               MFEM_VERIFY(D1D <= MAX_DQ, "");
+               MFEM_VERIFY(Q1D <= MAX_DQ, "");
+               QEvalVGF2D<0,0,0,MAX_DQ>(ne,B,x,y,vdim,D1D,Q1D);
+            }
+         }
+      }
+      if (dim == 3)
+      {
+         switch (id)
+         {
+            case 0x23: QEvalVGF3D<3,2,3>(ne,B,x,y); break;
+            case 0x34: QEvalVGF3D<3,3,4>(ne,B,x,y); break;
+            case 0x35: QEvalVGF3D<3,3,5>(ne,B,x,y); break;
+            case 0x46: QEvalVGF3D<3,4,6>(ne,B,x,y); break;
+            case 0x48: QEvalVGF3D<3,4,8>(ne,B,x,y); break;
+            default:
+            {
+               constexpr int MAX_DQ = 6;
+               MFEM_VERIFY(D1D <= MAX_DQ, "");
+               MFEM_VERIFY(Q1D <= MAX_DQ, "");
+               QEvalVGF3D<0,0,0,MAX_DQ>(ne,B,x,y,vdim,D1D,Q1D);
+            }
+         }
+      }
+   }
    else if (VectorQuadratureFunctionCoefficient* cQ =
                dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
    {
@@ -841,9 +1036,12 @@ static void PAConvectionApply(const int dim,
       switch ((D1D << 4 ) | Q1D)
       {
          case 0x22: return SmemPAConvectionApply2D<2,2,8>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x33: return SmemPAConvectionApply2D<3,3,3>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x44: return SmemPAConvectionApply2D<4,4,2>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x33: return SmemPAConvectionApply2D<3,3,4>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x34: return SmemPAConvectionApply2D<3,4,4>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x44: return SmemPAConvectionApply2D<4,4,4>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x46: return SmemPAConvectionApply2D<4,6,4>(NE,B,G,Bt,Gt,op,x,y);
          case 0x55: return SmemPAConvectionApply2D<5,5,2>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x58: return SmemPAConvectionApply2D<5,8,2>(NE,B,G,Bt,Gt,op,x,y);
          case 0x66: return SmemPAConvectionApply2D<6,6,1>(NE,B,G,Bt,Gt,op,x,y);
          case 0x77: return SmemPAConvectionApply2D<7,7,1>(NE,B,G,Bt,Gt,op,x,y);
          case 0x88: return SmemPAConvectionApply2D<8,8,1>(NE,B,G,Bt,Gt,op,x,y);
@@ -856,8 +1054,12 @@ static void PAConvectionApply(const int dim,
       switch ((D1D << 4 ) | Q1D)
       {
          case 0x23: return SmemPAConvectionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x24: return SmemPAConvectionApply3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x26: return SmemPAConvectionApply3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
          case 0x34: return SmemPAConvectionApply3D<3,4>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x35: return SmemPAConvectionApply3D<3,5>(NE,B,G,Bt,Gt,op,x,y);
          case 0x45: return SmemPAConvectionApply3D<4,5>(NE,B,G,Bt,Gt,op,x,y);
+         case 0x48: return SmemPAConvectionApply3D<4,8>(NE,B,G,Bt,Gt,op,x,y);
          case 0x56: return SmemPAConvectionApply3D<5,6>(NE,B,G,Bt,Gt,op,x,y);
          case 0x67: return SmemPAConvectionApply3D<6,7>(NE,B,G,Bt,Gt,op,x,y);
          case 0x78: return SmemPAConvectionApply3D<7,8>(NE,B,G,Bt,Gt,op,x,y);

fem/bilininteg_diffusion_pa.cpp

@@ -170,47 +170,53 @@ static void PADiffusionSetup3D(const int Q1D,
                                const Vector &c,
                                Vector &d)
 {
-   const int NQ = Q1D*Q1D*Q1D;
    const bool const_c = c.Size() == 1;
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
-   auto D = Reshape(d.Write(), NQ, 6, NE);
-   MFEM_FORALL(e, NE,
+   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
+   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
+                  Reshape(c.Read(), Q1D,Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, 6, NE);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
    {
-      for (int q = 0; q < NQ; ++q)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double detJ = J11 * (J22 * J33 - J32 * J23) -
-         /* */               J21 * (J12 * J33 - J32 * J13) +
-         /* */               J31 * (J12 * J23 - J22 * J13);
-         const double coeff = const_c ? C(0,0) : C(q,e);
-         const double c_detJ = W[q] * coeff / detJ;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-         // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-         D(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-         D(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-         D(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-         D(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-         D(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-         D(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               const double J11 = J(qx,qy,qz,0,0,e);
+               const double J21 = J(qx,qy,qz,1,0,e);
+               const double J31 = J(qx,qy,qz,2,0,e);
+               const double J12 = J(qx,qy,qz,0,1,e);
+               const double J22 = J(qx,qy,qz,1,1,e);
+               const double J32 = J(qx,qy,qz,2,1,e);
+               const double J13 = J(qx,qy,qz,0,2,e);
+               const double J23 = J(qx,qy,qz,1,2,e);
+               const double J33 = J(qx,qy,qz,2,2,e);
+               const double detJ = J11 * (J22 * J33 - J32 * J23) -
+               /* */               J21 * (J12 * J33 - J32 * J13) +
+               /* */               J31 * (J12 * J23 - J22 * J13);
+               const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
+               const double c_detJ = W(qx,qy,qz) * coeff / detJ;
+               // adj(J)
+               const double A11 = (J22 * J33) - (J23 * J32);
+               const double A12 = (J32 * J13) - (J12 * J33);
+               const double A13 = (J12 * J23) - (J22 * J13);
+               const double A21 = (J31 * J23) - (J21 * J33);
+               const double A22 = (J11 * J33) - (J13 * J31);
+               const double A23 = (J21 * J13) - (J11 * J23);
+               const double A31 = (J21 * J32) - (J31 * J22);
+               const double A32 = (J31 * J12) - (J11 * J32);
+               const double A33 = (J11 * J22) - (J12 * J21);
+               // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+               D(qx,qy,qz,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+               D(qx,qy,qz,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+               D(qx,qy,qz,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+               D(qx,qy,qz,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+               D(qx,qy,qz,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+               D(qx,qy,qz,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
+            }
+         }
       }
    });
 }
@@ -276,9 +282,10 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
    const int nq = ir->GetNPoints();
    dim = mesh->Dimension();
    ne = fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   const DofToQuad::Mode mode = DofToQuad::TENSOR;
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
    const int sdim = mesh->SpaceDimension();
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   maps = &el.GetDofToQuad(*ir, mode);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * ne, Device::GetDeviceMemoryType());
@@ -733,6 +740,7 @@ static void PADiffusionAssembleDiagonal(const int dim,
          case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,B,G,D,Y);
          case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,B,G,D,Y);
          case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,B,G,D,Y);
+         case 0x46: return SmemPADiffusionDiagonal3D<4,6>(NE,B,G,D,Y);
          case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,B,G,D,Y);
          case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,B,G,D,Y);
          case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,B,G,D,Y);
@@ -1693,6 +1701,7 @@ static void PADiffusionApply(const int dim,
       switch (ID)
       {
          case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
+         case 0x24: return SmemPADiffusionApply3D<2,4>(NE,B,G,D,X,Y);
          case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
          case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,D,X,Y);
          case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,D,X,Y);

fem/bilininteg_mass_pa.cpp

@@ -25,6 +25,7 @@ namespace mfem
 
 void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
 {
+
    // Assuming the same element type
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
@@ -45,22 +46,38 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
    nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
-                                    GeometricFactors::JACOBIANS);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const DofToQuad::Mode mode = DofToQuad::TENSOR;
+   const int flags = GeometricFactors::JACOBIANS |
+                     GeometricFactors::COORDINATES;
+#ifdef MFEM_USE_UMPIRE
+   const MemoryType temp_type = Device::GetDeviceMemoryType() == MemoryType::DEVICE_UMPIRE
+      ? MemoryType::DEVICE_UMPIRE_2 : Device::GetDeviceMemoryType();
+#else
+   const MemoryType temp_type = Device::GetDeviceMemoryType();
+#endif
+   geom = mesh->GetGeometricFactors(*ir, flags, mode, temp_type);
+   maps = &el.GetDofToQuad(*ir, mode);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
-   Vector coeff;
+   Vector *coeff{nullptr};
+   bool own_coeff{true};
    if (Q == nullptr)
    {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
+      coeff = new Vector;
+      coeff->SetSize(1);
+      (*coeff)(0) = 1.0;
    }
    else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
    {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
+      coeff = new Vector;
+      coeff->SetSize(1);
+      (*coeff)(0) = cQ->constant;
+   }
+   else if (QuadratureCoefficient* cQ = dynamic_cast<QuadratureCoefficient*>(Q))
+   {
+      coeff = cQ->Data();
+      own_coeff = false;
    }
    else if (QuadratureFunctionCoefficient* cQ =
                dynamic_cast<QuadratureFunctionCoefficient*>(Q))
@@ -73,12 +90,13 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
                   "IntegrationRule used within integrator and in"
                   " QuadratureFunction appear to be different");
       qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+      coeff->MakeRef(const_cast<QuadratureFunction &>(qFun),0);
    }
    else
    {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      coeff = new Vector;
+      coeff->SetSize(nq * ne);
+      auto C = Reshape(coeff->HostWrite(), nq, ne);
       for (int e = 0; e < ne; ++e)
       {
          ElementTransformation& T = *fes.GetElementTransformation(e);
@@ -93,11 +111,11 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    {
       const int NE = ne;
       const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
+      const bool const_c = coeff->Size() == 1;
       auto w = ir->GetWeights().Read();
       auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
       auto C =
-         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
+         const_c ? Reshape(coeff->Read(), 1,1) : Reshape(coeff->Read(), NQ,NE);
       auto v = Reshape(pa_data.Write(), NQ, NE);
       MFEM_FORALL(e, NE,
       {
@@ -116,28 +134,43 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    if (dim==3)
    {
       const int NE = ne;
-      const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
-      auto W = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
-      auto C =
-         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
-      auto v = Reshape(pa_data.Write(), NQ,NE);
-      MFEM_FORALL(e, NE,
+      const int Q1D = quad1D;
+      const bool const_c = coeff->Size() == 1;
+      const auto W = Reshape(ir->GetWeights().Read(),Q1D,Q1D,Q1D);
+      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
+      const auto C = const_c ?
+                     Reshape(coeff->Read(), 1,1,1,1) :
+                     Reshape(coeff->Read(), Q1D,Q1D,Q1D,NE);
+      auto V = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
+      MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
       {
-         for (int q = 0; q < NQ; ++q)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
          {
-            const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e);
-            const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e);
-            const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e);
-            const double detJ = J11 * (J22 * J33 - J32 * J23) -
-            /* */               J21 * (J12 * J33 - J32 * J13) +
-            /* */               J31 * (J12 * J23 - J22 * J13);
-            const double coeff = const_c ? C(0,0) : C(q,e);
-            v(q,e) = W[q] * coeff * detJ;
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qz,z,Q1D)
+               {
+                  const double J11 = J(qx,qy,qz,0,0,e);
+                  const double J12 = J(qx,qy,qz,0,1,e);
+                  const double J13 = J(qx,qy,qz,0,2,e);
+                  const double J21 = J(qx,qy,qz,1,0,e);
+                  const double J22 = J(qx,qy,qz,1,1,e);
+                  const double J23 = J(qx,qy,qz,1,2,e);
+                  const double J31 = J(qx,qy,qz,2,0,e);
+                  const double J32 = J(qx,qy,qz,2,1,e);
+                  const double J33 = J(qx,qy,qz,2,2,e);
+                  const double detJ = J11 * (J22 * J33 - J32 * J23) -
+                  /* */               J21 * (J12 * J33 - J32 * J13) +
+                  /* */               J31 * (J12 * J23 - J22 * J13);
+                  const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
+                  V(qx,qy,qz,e) = W(qx,qy,qz) * coeff * detJ;
+               }
+            }
          }
       });
    }
+
+   if (own_coeff) { delete coeff; }
 }
 
 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
@@ -439,8 +472,12 @@ static void PAMassAssembleDiagonal(const int dim, const int D1D,
       switch ((D1D << 4 ) | Q1D)
       {
          case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
+         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
+         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
          case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
+         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
          case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
+         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
          case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
          case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
          case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
@@ -1174,10 +1211,13 @@ static void PAMassApply(const int dim,
          case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
          case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
          case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
+         case 0x35: return SmemPAMassApply2D<3,5,16>(NE,B,Bt,D,X,Y);
          case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
          case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
+         case 0x46: return SmemPAMassApply2D<4,6,8>(NE,B,Bt,D,X,Y);
          case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
          case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
+         case 0x57: return SmemPAMassApply2D<5,7,8>(NE,B,Bt,D,X,Y);
          case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
          case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
          case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
@@ -1185,6 +1225,7 @@ static void PAMassApply(const int dim,
          case 0x99: return SmemPAMassApply2D<9,9,2>(NE,B,Bt,D,X,Y);
          default:   return PAMassApply2D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
+      mfem::out << "Unknown 2D kernel 0x" << std::hex << id << std::endl;
    }
    else if (dim == 3)
    {
@@ -1193,7 +1234,9 @@ static void PAMassApply(const int dim,
          case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
          case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
          case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
+         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
          case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
+         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
          case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
          case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
          case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
@@ -1205,8 +1248,8 @@ static void PAMassApply(const int dim,
          case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
          default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
+      mfem::out << "Unknown 3D kernel 0x" << std::hex << id << std::endl;
    }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
    MFEM_ABORT("Unknown kernel.");
 }
 

fem/coefficient.cpp

@@ -12,6 +12,7 @@
 // Implementation of Coefficient class
 
 #include "fem.hpp"
+#include "../linalg/dtensor.hpp"
 
 #include <cmath>
 #include <limits>
@@ -21,6 +22,13 @@ namespace mfem
 
 using namespace std;
 
+double QuadratureCoefficient::Eval(ElementTransformation & T,
+                                   const IntegrationPoint & ip)
+{
+   auto coeff = mfem::Reshape(qData->HostRead(), nip, NE);
+   return coeff(ip.index, T.ElementNo);
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {

fem/coefficient.hpp

@@ -87,6 +87,33 @@ public:
    { return (constant); }
 };
 
+
+/// class for quadrature coefficient
+class QuadratureCoefficient : public Coefficient
+{
+
+private:
+   const int nip;
+   const int NE;
+public:
+   Vector *qData{nullptr};
+
+   //Set external data
+   QuadratureCoefficient(Vector *Data, int in_nip, int in_NE)
+      : qData(Data), nip(in_nip), NE(in_NE)
+   { }
+
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+
+   Vector *Data()
+   {
+      return qData;
+   }
+
+};
+
+/// class for piecewise constant coefficient
 /** @brief A piecewise constant coefficient with the constants keyed
     off the element attribute numbers. */
 class PWConstCoefficient : public Coefficient

fem/fespace.cpp

@@ -944,7 +944,7 @@ const Operator *FiniteElementSpace::GetFaceRestriction(
 }
 
 const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
-   const IntegrationRule &ir) const
+   const IntegrationRule &ir, const DofToQuad::Mode mode) const
 {
    for (int i = 0; i < E2Q_array.Size(); i++)
    {
@@ -952,13 +952,13 @@ const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
       if (qi->IntRule == &ir) { return qi; }
    }
 
-   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, ir);
+   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, ir, mode);
    E2Q_array.Append(qi);
    return qi;
 }
 
 const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
-   const QuadratureSpace &qs) const
+   const QuadratureSpace &qs, const DofToQuad::Mode mode) const
 {
    for (int i = 0; i < E2Q_array.Size(); i++)
    {
@@ -966,7 +966,7 @@ const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
       if (qi->qspace == &qs) { return qi; }
    }
 
-   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, qs);
+   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, qs, mode);
    E2Q_array.Append(qi);
    return qi;
 }
@@ -983,8 +983,8 @@ const FaceQuadratureInterpolator
          if (qi->IntRule == &ir) { return qi; }
       }
 
-      FaceQuadratureInterpolator *qi = new FaceQuadratureInterpolator(*this, ir,
-                                                                      type);
+      FaceQuadratureInterpolator *qi =
+         new FaceQuadratureInterpolator(*this, ir, type);
       E2IFQ_array.Append(qi);
       return qi;
    }

fem/fespace.hpp

@@ -367,7 +367,7 @@ public:
        All elements will use the same IntegrationRule, @a ir as the target
        quadrature points. */
    const QuadratureInterpolator *GetQuadratureInterpolator(
-      const IntegrationRule &ir) const;
+      const IntegrationRule &ir, const DofToQuad::Mode = DofToQuad::FULL) const;
 
    /** @brief Return a QuadratureInterpolator that interpolates E-vectors to
        quadrature point values and/or derivatives (Q-vectors). */
@@ -378,7 +378,7 @@ public:
        The target quadrature points in the elements are described by the given
        QuadratureSpace, @a qs. */
    const QuadratureInterpolator *GetQuadratureInterpolator(
-      const QuadratureSpace &qs) const;
+      const QuadratureSpace &qs, const DofToQuad::Mode = DofToQuad::FULL) const;
 
    /** @brief Return a FaceQuadratureInterpolator that interpolates E-vectors to
        quadrature point values and/or derivatives (Q-vectors). */

fem/nonlinearform.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "fem.hpp"
+#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -27,7 +28,7 @@ void NonlinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
          // This is the default behavior.
          break;
       case AssemblyLevel::PARTIAL:
-         ext = new PANonlinearFormExtension(this);
+         ext = new PANonlinearForm(this);
          break;
       default:
          mfem_error("Unknown assembly level for this form.");
@@ -80,6 +81,13 @@ void NonlinearForm::SetEssentialVDofs(const Array<int> &ess_vdofs_list)
 
 double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
 {
+   if (ext)
+   {
+      MFEM_VERIFY(!fnfi.Size(), "Interior faces terms not yet implemented!");
+      MFEM_VERIFY(!bfnfi.Size(), "Boundary face terms not yet implemented!");
+      return ext->GetGridFunctionEnergy(x);
+   }
+
    Array<int> vdofs;
    Vector el_x;
    const FiniteElement *fe;
@@ -138,6 +146,14 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
    if (ext)
    {
       ext->Mult(px, py);
+      if (Serial())
+      {
+         if (cP) { cP->MultTranspose(py, y); }
+         const int N = ess_tdof_list.Size();
+         const auto tdof = ess_tdof_list.Read();
+         auto Y = y.ReadWrite();
+         MFEM_FORALL(i, N, Y[tdof[i]] = 0.0; );
+      }
       return;
    }
 
@@ -264,7 +280,16 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const
 {
    if (ext)
    {
-      MFEM_ABORT("Not yet implemented!");
+      Operator &grad = ext->GetGradient(Prolongate(x));
+      hGrad.Reset(&grad, false);
+      if (Serial())
+      {
+         Operator *Gop;
+         if (cP) { hGrad.Reset(new RAPOperator(*cP, grad, *cP)); }
+         hGrad.Ptr()->Operator::FormSystemOperator(ess_tdof_list, Gop);
+         hGrad.Reset(Gop);
+      }
+      return *hGrad.Ptr();
    }
 
    const int skip_zeros = 0;
@@ -426,7 +451,31 @@ void NonlinearForm::Update()
 
 void NonlinearForm::Setup()
 {
-   if (ext) { return ext->AssemblePA(); }
+   if (ext) { return ext->Setup(); }
+}
+
+void NonlinearForm::AssembleGradientDiagonal(Vector &diag) const
+{
+   if (ext)
+   {
+      MFEM_ASSERT(diag.Size() == fes->GetTrueVSize(),
+                  "Vector for holding diagonal has wrong size!");
+      const Operator *P = fes->GetProlongationMatrix();
+      if (!IsIdentityProlongation(P))
+      {
+         Vector local_diag(P->Height());
+         ext->AssembleGradientDiagonal(local_diag);
+         P->MultTranspose(local_diag, diag);
+      }
+      else
+      {
+         ext->AssembleGradientDiagonal(diag);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Not implemented. Can be obtained through GetGradient().");
+   }
 }
 
 NonlinearForm::~NonlinearForm()

fem/nonlinearform.hpp

@@ -45,6 +45,7 @@ protected:
    Array<Array<int>*>              bfnfi_marker; // not owned
 
    mutable SparseMatrix *Grad, *cGrad; // owned
+   mutable OperatorHandle hGrad;
 
    /// A list of all essential true dofs
    Array<int> ess_tdof_list;
@@ -165,6 +166,15 @@ public:
    /// Setup the NonlinearForm
    virtual void Setup();
 
+   /** @brief Assemble the diagonal of the gradient into diag
+
+       For adaptively refined meshes, this returns P^T d_e, where d_e is the
+       locally assembled diagonal on each element and P^T is the transpose of
+       the conforming prolongation. In general this is not the correct diagonal
+       for an AMR mesh. */
+   void AssembleGradientDiagonal(Vector &diag) const;
+
+
    /// Get the finite element space prolongation matrix
    virtual const Operator *GetProlongation() const { return P; }
    /// Get the finite element space restriction matrix

fem/nonlinearform_ext.cpp

@@ -13,62 +13,101 @@
 // PABilinearFormExtension and MFBilinearFormExtension.
 
 #include "nonlinearform.hpp"
+#include "../general/forall.hpp"
 
 namespace mfem
 {
 
-NonlinearFormExtension::NonlinearFormExtension(NonlinearForm *form)
-   : Operator(form->FESpace()->GetTrueVSize()), n(form)
+NonlinearFormExtension::NonlinearFormExtension(const NonlinearForm *nlf)
+   : Operator(nlf->FESpace()->GetTrueVSize()), nlf(nlf) { }
+
+PANonlinearForm::PANonlinearForm(NonlinearForm *nlf):
+   NonlinearFormExtension(nlf),
+   x_grad(NULL),
+   fes(*nlf->FESpace()),
+   dnfi(*nlf->GetDNFI()),
+   R(fes.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC))
 {
-   // empty
+   MFEM_VERIFY(R, "Not yet implemented!");
+   xe.SetSize(R->Height(), Device::GetMemoryType());
+   ye.SetSize(R->Height(), Device::GetMemoryType());
+   ye.UseDevice(true);
 }
 
-PANonlinearFormExtension::PANonlinearFormExtension(NonlinearForm *form):
-   NonlinearFormExtension(form), fes(*form->FESpace())
+double PANonlinearForm::GetGridFunctionEnergy(const Vector &x) const
 {
-   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   elem_restrict_lex = fes.GetElementRestriction(ordering);
-   if (elem_restrict_lex)
+   double energy = 0.0;
+
+   R->Mult(x, xe);
+   for (int i = 0; i < dnfi.Size(); i++)
    {
-      localX.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-      localY.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-      localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
+      energy += dnfi[i]->GetGridFunctionEnergyPA(xe);
    }
+   return energy;
 }
 
-void PANonlinearFormExtension::AssemblePA()
+void PANonlinearForm::Setup()
 {
-   Array<NonlinearFormIntegrator*> &integrators = *n->GetDNFI();
-   const int Ni = integrators.Size();
-   for (int i = 0; i < Ni; ++i)
-   {
-      integrators[i]->AssemblePA(*n->FESpace());
-   }
+   for (int i = 0; i < dnfi.Size(); ++i) { dnfi[i]->AssemblePA(fes); }
 }
 
-void PANonlinearFormExtension::Mult(const Vector &x, Vector &y) const
+void PANonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   Array<NonlinearFormIntegrator*> &integrators = *n->GetDNFI();
-   const int iSz = integrators.Size();
-   if (elem_restrict_lex)
-   {
-      elem_restrict_lex->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(localX, localY);
-      }
-      elem_restrict_lex->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(x, y);
-      }
-   }
+   ye = 0.0;
+   R->Mult(x, xe);
+   for (int i = 0; i < dnfi.Size(); ++i) { dnfi[i]->AddMultPA(xe, ye); }
+   R->MultTranspose(ye, y);
 }
 
+void PANonlinearForm::AssembleGradientDiagonal(Vector &diag) const
+{
+   MFEM_VERIFY(x_grad, "GetGradient() has not been called");
+   R->Mult(*x_grad, xe);
+
+   ye = 0.0;
+   for (int i = 0; i < dnfi.Size(); ++i)
+   {
+      dnfi[i]->AssembleGradientDiagonalPA(xe, ye);
+   }
+   R->MultTranspose(ye, diag);
 }
+
+Operator &PANonlinearForm::GetGradient(const Vector &x) const
+{
+   // Store the last x that was used to compute the gradient.
+   x_grad = &x;
+
+   Grad.Reset(new PANonlinearForm::Gradient(x, *this));
+   return *Grad.Ptr();
+}
+
+PANonlinearForm::Gradient::Gradient(const Vector &x, const PANonlinearForm &e):
+   Operator(e.fes.GetVSize()), R(e.R), dnfi(e.dnfi)
+{
+   ge.UseDevice(true);
+   ge.SetSize(R->Height(), Device::GetMemoryType());
+   R->Mult(x, ge);
+
+   xe.UseDevice(true);
+   xe.SetSize(R->Height(), Device::GetMemoryType());
+
+   ye.UseDevice(true);
+   ye.SetSize(R->Height(), Device::GetMemoryType());
+
+   ze.UseDevice(true);
+   ze.SetSize(R->Height(), Device::GetMemoryType());
+
+   // Do we still need to do this?
+   for (int i = 0; i < dnfi.Size(); ++i) { dnfi[i]->AssemblePA(e.fes); }
+}
+
+void PANonlinearForm::Gradient::Mult(const Vector &x, Vector &y) const
+{
+   ze = x;
+   ye = 0.0;
+   R->Mult(ze, xe);
+   for (int i = 0; i < dnfi.Size(); ++i) { dnfi[i]->AddMultGradPA(ge, xe, ye); }
+   R->MultTranspose(ye, y);
+}
+
+} // namespace mfem

fem/nonlinearform_ext.hpp

@@ -17,28 +17,60 @@
 
 namespace mfem
 {
-class NonlinearForm;
 
+class NonlinearForm;
+class NonlinearFormIntegrator;
+
+/** @brief Class extending the NonlinearForm class to support the different
+    AssemblyLevel%s. */
 class NonlinearFormExtension : public Operator
 {
 protected:
-   NonlinearForm *n; ///< Not owned
+   const NonlinearForm *nlf;
 public:
-   NonlinearFormExtension(NonlinearForm *form);
-   virtual void AssemblePA() = 0;
+   NonlinearFormExtension(const NonlinearForm*);
+   virtual void Setup() = 0;
+   virtual Operator &GetGradient(const Vector&) const = 0;
+   virtual double GetGridFunctionEnergy(const Vector &x) const = 0;
+   virtual void AssembleGradientDiagonal(Vector &diag) const
+   {
+      MFEM_ABORT("Not implemented for this assembly level!");
+   }
 };
 
+class PANonlinearForm;
+
+
 /// Data and methods for partially-assembled nonlinear forms
-class PANonlinearFormExtension : public NonlinearFormExtension
+class PANonlinearForm : public NonlinearFormExtension
 {
+private:
+   class Gradient : public Operator
+   {
+   protected:
+      const Operator *R;
+      mutable Vector ge, xe, ye, ze;
+      const Array<NonlinearFormIntegrator*> &dnfi;
+   public:
+      Gradient(const Vector &x, const PANonlinearForm &ext);
+      virtual void Mult(const Vector &x, Vector &y) const;
+   };
+
 protected:
-   const FiniteElementSpace &fes; // Not owned
-   mutable Vector localX, localY;
-   const Operator *elem_restrict_lex; // Not owned
+   mutable Vector xe, ye;
+   mutable const Vector *x_grad;
+   mutable OperatorHandle Grad;
+   const FiniteElementSpace &fes;
+   const Array<NonlinearFormIntegrator*> &dnfi;
+   const Operator *R;
+
 public:
-   PANonlinearFormExtension(NonlinearForm*);
-   void AssemblePA();
+   PANonlinearForm(NonlinearForm *nlf);
+   void Setup();
    void Mult(const Vector &x, Vector &y) const;
+   Operator &GetGradient(const Vector &x) const;
+   double GetGridFunctionEnergy(const Vector &x) const;
+   void AssembleGradientDiagonal(Vector &diag) const;
 };
 }
 #endif // NONLINEARFORM_EXT_HPP

fem/nonlininteg.cpp

@@ -15,6 +15,13 @@
 namespace mfem
 {
 
+double NonlinearFormIntegrator::GetGridFunctionEnergyPA(const Vector &x) const
+{
+   mfem_error ("NonlinearFormIntegrator::GetGridFunctionEnergyPA(...)\n"
+               "   is not implemented for this class.");
+   return 0.0;
+}
+
 void NonlinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
    mfem_error ("NonlinearFormIntegrator::AssemblePA(...)\n"
@@ -34,6 +41,20 @@ void NonlinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
                "   is not implemented for this class.");
 }
 
+void NonlinearFormIntegrator::AddMultGradPA(const Vector&,
+                                            const Vector&, Vector&) const
+{
+   mfem_error ("NonlinearFormIntegrator::AddMultGradPA(...)\n"
+               "   is not implemented for this class.");
+}
+
+void NonlinearFormIntegrator::AssembleGradientDiagonalPA(const mfem::Vector &x,
+                                                         mfem::Vector &diag) const
+{
+   mfem_error ("NonlinearFormIntegrator::AssembleDiagonalPA(...)\n"
+               "   is not implemented for this class.");
+}
+
 void NonlinearFormIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr,
    const Vector &elfun, Vector &elvect)

fem/nonlininteg.hpp

@@ -68,6 +68,9 @@ public:
                                    ElementTransformation &Tr,
                                    const Vector &elfun);
 
+   /// Compute the local energy with partial assembly.
+   virtual double GetGridFunctionEnergyPA(const Vector &x) const;
+
    /// Method defining partial assembly.
    /** The result of the partial assembly is stored internally so that it can be
        used later in the methods AddMultPA(). */
@@ -88,6 +91,12 @@ public:
        called. */
    virtual void AddMultPA(const Vector &x, Vector &y) const;
 
+   /// Method for partially assembled gradient action.
+   virtual void AddMultGradPA(const Vector &g,
+                              const Vector &x, Vector &y) const;
+
+   virtual void AssembleGradientDiagonalPA(const Vector &x, Vector &diag) const;
+
    virtual ~NonlinearFormIntegrator() { }
 };
 

fem/pfespace.cpp

@@ -3147,7 +3147,7 @@ static void SetSubVector(const int N,
                          const Array<int> &indices,
                          const Vector &in, Vector &out)
 {
-   auto y = out.Write();
+   auto y = out.ReadWrite();
    const auto x = in.Read();
    const auto I = indices.Read();
    MFEM_FORALL(i, N, y[I[i]] = x[i];);
@@ -3234,7 +3234,7 @@ static void AddSubVector(const int num_unique_dst_indices,
                          const Vector &src,
                          Vector &dst)
 {
-   auto y = dst.Write();
+   auto y = dst.ReadWrite();
    const auto x = src.Read();
    const auto DST_I = unique_dst_indices.Read();
    const auto SRC_O = unique_to_src_offsets.Read();

fem/pgridfunc.cpp

@@ -711,9 +711,9 @@ void ParGridFunction::SaveAsOne(std::ostream &out)
    int *nfdofs = new int[NRanks];
    int *nrdofs = new int[NRanks];
 
-   double * h_data = const_cast<double *>(this->HostRead());
+   HostReadWrite();
+   values[0] = data;
 
-   values[0] = h_data;
    nv[0]     = pfes -> GetVSize();
    nvdofs[0] = pfes -> GetNVDofs();
    nedofs[0] = pfes -> GetNEDofs();
@@ -814,7 +814,7 @@ void ParGridFunction::SaveAsOne(std::ostream &out)
       MPI_Send(&nvdofs[0], 1, MPI_INT, 0, 456, MyComm);
       MPI_Send(&nedofs[0], 1, MPI_INT, 0, 457, MyComm);
       MPI_Send(&nfdofs[0], 1, MPI_INT, 0, 458, MyComm);
-      MPI_Send(h_data, nv[0], MPI_DOUBLE, 0, 460, MyComm);
+      MPI_Send(data, nv[0], MPI_DOUBLE, 0, 460, MyComm);
    }
 
    delete [] values;

fem/pnonlinearform.cpp

@@ -14,6 +14,7 @@
 #ifdef MFEM_USE_MPI
 
 #include "fem.hpp"
+#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -49,6 +50,7 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 
    if (fnfi.Size())
    {
+      MFEM_VERIFY(!NonlinearForm::ext,"");
       // Terms over shared interior faces in parallel.
       ParFiniteElementSpace *pfes = ParFESpace();
       ParMesh *pmesh = pfes->GetParMesh();
@@ -86,15 +88,16 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 
    P->MultTranspose(aux2, y);
 
-   y.HostReadWrite();
-   for (int i = 0; i < ess_tdof_list.Size(); i++)
-   {
-      y(ess_tdof_list[i]) = 0.0;
-   }
+   const int N = ess_tdof_list.Size();
+   const auto idx = ess_tdof_list.Read();
+   auto Y = y.ReadWrite();
+   MFEM_FORALL(i, N, Y[idx[i]] = 0.0; );
 }
 
 const SparseMatrix &ParNonlinearForm::GetLocalGradient(const Vector &x) const
 {
+   if (NonlinearForm::ext) { MFEM_ABORT("Not yet implemented!"); }
+
    NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c.
 
    return *Grad;
@@ -104,16 +107,20 @@ Operator &ParNonlinearForm::GetGradient(const Vector &x) const
 {
    ParFiniteElementSpace *pfes = ParFESpace();
 
-   pGrad.Clear();
+   Operator &grad = NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c.
 
-   NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c.
+   pGrad.Clear();
 
    OperatorHandle dA(pGrad.Type()), Ph(pGrad.Type());
 
    if (fnfi.Size() == 0)
    {
-      dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
-                             pfes->GetDofOffsets(), Grad);
+      if (NonlinearForm::ext) { dA.Reset(&grad, false); }
+      else
+      {
+         dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
+                                pfes->GetDofOffsets(), Grad);
+      }
    }
    else
    {

fem/quadinterpolator.hpp

@@ -41,10 +41,11 @@ protected:
    const FiniteElementSpace *fespace;  ///< Not owned
    const QuadratureSpace *qspace;      ///< Not owned
    const IntegrationRule *IntRule;     ///< Not owned
+
    mutable QVectorLayout q_layout;     ///< Output Q-vector layout
+   mutable bool use_tensor_products;   ///< Tensor product evaluation mmode
 
-   mutable bool use_tensor_products;
-
+public:
    static const int MAX_NQ2D = 100;
    static const int MAX_ND2D = 100;
    static const int MAX_VDIM2D = 3;
@@ -53,7 +54,6 @@ protected:
    static const int MAX_ND3D = 1000;
    static const int MAX_VDIM3D = 3;
 
-public:
    enum EvalFlags
    {
       VALUES       = 1 << 0,  ///< Evaluate the values at quadrature points
@@ -61,21 +61,28 @@ public:
       /** @brief Assuming the derivative at quadrature points form a matrix,
           this flag can be used to compute and store their determinants. This
           flag can only be used in Mult(). */
-      DETERMINANTS = 1 << 2
+      DETERMINANTS = 1 << 2,
+      PHYSICAL_DERIVATIVES = 1 << 3 ///< Evaluate the physical derivatives
    };
 
    QuadratureInterpolator(const FiniteElementSpace &fes,
-                          const IntegrationRule &ir);
+                          const IntegrationRule &ir,
+                          const bool use_tensor_products = false);
 
    QuadratureInterpolator(const FiniteElementSpace &fes,
-                          const QuadratureSpace &qs);
+                          const QuadratureSpace &qs,
+                          const bool use_tensor_products = false);
 
    /** @brief Disable the use of tensor product evaluations, for tensor-product
        elements, e.g. quads and hexes. */
-   /** Currently, tensor product evaluations are not implemented and this method
-       has no effect. */
-   void DisableTensorProducts(bool disable = true) const
-   { use_tensor_products = !disable; }
+   void DisableTensorProducts() const { use_tensor_products = false; }
+
+   /** @brief Enable the use of tensor product evaluations, for tensor-product
+       elements, e.g. quads and hexes. */
+   void EnableTensorProducts() const { use_tensor_products = true; }
+
+   /** @brief Query the current evaluation mode. */
+   bool UseTensorProducts() const { return use_tensor_products; }
 
    /** @brief Query the current output Q-vector layout. The default value is
        QVectorLayout::byNODES. */
@@ -83,8 +90,7 @@ public:
 
    /** @brief Set the desired output Q-vector layout. The default value is
        QVectorLayout::byNODES. */
-   void SetOutputLayout(QVectorLayout out_layout) const
-   { q_layout = out_layout; }
+   void SetOutputLayout(QVectorLayout layout) const { q_layout = layout; }
 
    /// Interpolate the E-vector @a e_vec to quadrature points.
    /** The @a eval_flags are a bitwise mask of constants from the EvalFlags
@@ -99,26 +105,36 @@ public:
              Vector &q_val, Vector &q_der, Vector &q_det) const;
 
    /// Interpolate the values of the E-vector @a e_vec at quadrature points.
+   template <QVectorLayout>
+   void Values(const Vector &e_vec, Vector &q_val) const;
    void Values(const Vector &e_vec, Vector &q_val) const;
 
    /** @brief Interpolate the derivatives of the E-vector @a e_vec at quadrature
        points. */
+   template <QVectorLayout>
+   void Derivatives(const Vector &e_vec, Vector &q_der) const;
    void Derivatives(const Vector &e_vec, Vector &q_der) const;
 
    /** @brief Interpolate the derivatives in physical space of the E-vector
        @a e_vec at quadrature points. */
+   template <QVectorLayout>
    void PhysDerivatives(const Vector &e_vec, Vector &q_der) const;
+   void PhysDerivatives(const Vector &e_vec, Vector &q_der) const;
+
+   /// Compute the determinant of the E-vector @a e_vec at quadrature points.
+   void Determinants(const Vector &e_vec, Vector &q_det) const;
 
    /// Perform the transpose operation of Mult(). (TODO)
    void MultTranspose(unsigned eval_flags, const Vector &q_val,
                       const Vector &q_der, Vector &e_vec) const;
 
    // Compute kernels follow (cannot be private or protected with nvcc)
-
    /// Template compute kernel for 2D.
    template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
-   static void Eval2D(const int NE,
+   static void Mult2D(const int NE,
                       const int vdim,
+                      const QVectorLayout q_layout,
+                      const GeometricFactors *geom,
                       const DofToQuad &maps,
                       const Vector &e_vec,
                       Vector &q_val,
@@ -128,8 +144,10 @@ public:
 
    /// Template compute kernel for 3D.
    template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
-   static void Eval3D(const int NE,
+   static void Mult3D(const int NE,
                       const int vdim,
+                      const QVectorLayout q_layout,
+                      const GeometricFactors *geom,
                       const DofToQuad &maps,
                       const Vector &e_vec,
                       Vector &q_val,

fem/quadinterpolator_det.cpp

@@ -0,0 +1,208 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop_pa.hpp"
+#include "quadinterpolator.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+#include "../fem/kernels.hpp"
+#include "../linalg/kernels.hpp"
+
+using namespace mfem;
+
+namespace mfem
+{
+
+template<int T_D1D = 0, int T_Q1D = 0, int MAX_D1D = 0, int MAX_Q1D = 0>
+static void Det2D(const int NE,
+                  const double *b,
+                  const double *g,
+                  const double *x,
+                  double *y,
+                  const int vdim = 1,
+                  const int d1d = 0,
+                  const int q1d = 0)
+{
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x,  D1D, D1D, DIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double J[4];
+            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,J);
+            Y(qx,qy,e) = kernels::Det<2>(J);
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int MAX_D1D = 0, int MAX_Q1D = 0>
+static void Det3D(const int NE,
+                  const double *b,
+                  const double *g,
+                  const double *x,
+                  double *y,
+                  const int vdim = 1,
+                  const int d1d = 0,
+                  const int q1d = 0)
+{
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x, D1D, D1D, D1D, DIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double sm0[9][MDQ*MDQ*MDQ];
+      MFEM_SHARED double sm1[9][MDQ*MDQ*MDQ];
+
+      double (*DDD)[MD1*MD1*MD1] = (double (*)[MD1*MD1*MD1]) (sm0);
+      double (*DDQ)[MD1*MD1*MQ1] = (double (*)[MD1*MD1*MQ1]) (sm1);
+      double (*DQQ)[MD1*MQ1*MQ1] = (double (*)[MD1*MQ1*MQ1]) (sm0);
+      double (*QQQ)[MQ1*MQ1*MQ1] = (double (*)[MQ1*MQ1*MQ1]) (sm1);
+
+      kernels::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double J[9];
+               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, J);
+               Y(qx,qy,qz,e) = kernels::Det<3>(J);
+            }
+         }
+      }
+   });
+}
+
+void QuadratureInterpolator::Determinants(const Vector &e_vec,
+                                          Vector &q_det) const
+{
+   if (use_tensor_products)
+   {
+      const int NE = fespace->GetNE();
+      if (NE == 0) { return; }
+      const int vdim = fespace->GetVDim();
+      const int dim = fespace->GetMesh()->Dimension();
+      const FiniteElement *fe = fespace->GetFE(0);
+      const IntegrationRule *ir =
+         IntRule ? IntRule : &qspace->GetElementIntRule(0);
+      const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
+      const int D1D = maps.ndof;
+      const int Q1D = maps.nqpt;
+      const double *B = maps.B.Read();
+      const double *G = maps.G.Read();
+      const double *X = e_vec.Read();
+      double *Y = q_det.Write();
+
+      const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
+
+      switch (id)
+      {
+         case 0x2222: return Det2D<2,2>(NE,B,G,X,Y);
+         case 0x2223: return Det2D<2,3>(NE,B,G,X,Y);
+         case 0x2224: return Det2D<2,4>(NE,B,G,X,Y);
+         case 0x2226: return Det2D<2,6>(NE,B,G,X,Y);
+         case 0x2234: return Det2D<3,4>(NE,B,G,X,Y);
+         case 0x2236: return Det2D<3,6>(NE,B,G,X,Y);
+         case 0x2244: return Det2D<4,4>(NE,B,G,X,Y);
+         case 0x2246: return Det2D<4,6>(NE,B,G,X,Y);
+         case 0x2256: return Det2D<5,6>(NE,B,G,X,Y);
+
+         case 0x3324: return Det3D<2,4>(NE,B,G,X,Y);
+         case 0x3333: return Det3D<3,3>(NE,B,G,X,Y);
+         case 0x3335: return Det3D<3,5>(NE,B,G,X,Y);
+         case 0x3336: return Det3D<3,6>(NE,B,G,X,Y);
+         //case 0x3348: return Det3D<4,8>(NE,B,G,X,Y);
+
+         default:
+         {
+            if (dim == 2)
+            {
+               constexpr int MD1 = 8;
+               constexpr int MQ1 = 8;
+               MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
+                           << " are not supported!");
+               MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
+                           << MQ1 << " 1D points are not supported!");
+               return Det2D<0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D);
+            }
+            if (dim == 3)
+            {
+               constexpr int MD1 = 6;
+               constexpr int MQ1 = 6;
+               MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
+                           << " are not supported!");
+               MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
+                           << MQ1 << " 1D points are not supported!");
+               return Det3D<0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D);
+            }
+         }
+
+      }
+      MFEM_ABORT("Kernel " << std::hex << id << std::dec << " not supported yet");
+   }
+   else
+   {
+      Vector empty;
+      Mult(e_vec, DETERMINANTS, empty, empty, q_det);
+   }
+}
+
+} // namespace mfem

fem/quadinterpolator_eval.hpp

@@ -0,0 +1,233 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1, int MAX_D1D = 0, int MAX_Q1D = 0>
+static void Eval2D(const int NE,
+                   const double *b_,
+                   const double *x_,
+                   double *y_,
+                   const int vdim = 0,
+                   const int d1d = 0,
+                   const int q1d = 0)
+{
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED double s_B[MQ1*MD1];
+      DeviceTensor<2,double> B(s_B, Q1D, D1D);
+
+      MFEM_SHARED double s_DD[NBZ][MD1*MD1];
+      DeviceTensor<2,double> DD((double*)(s_DD+tidz), MD1, MD1);
+
+      MFEM_SHARED double s_DQ[NBZ][MD1*MQ1];
+      DeviceTensor<2,double> DQ((double*)(s_DQ+tidz), MD1, MQ1);
+
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B(q,d) = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               DD(dx,dy) = x(dx,dy,c,e);
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  u += B(qx,dx) *  DD(dx,dy);
+               }
+               DQ(dy,qx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  u += DQ(dy,qx) * B(qy,dy);
+               }
+               if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,e) = u; }
+               if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,c,e) = u; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int MAX_D1D = 0, int MAX_Q1D = 0>
+static void Eval3D(const int NE,
+                   const double *b_,
+                   const double *x_,
+                   double *y_,
+                   const int vdim = 0,
+                   const int d1d = 0,
+                   const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED double s_B[MQ1*MD1];
+      DeviceTensor<2,double> B(s_B, Q1D, D1D);
+
+      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
+      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
+      DeviceTensor<3,double> DDD(sm0, MD1, MD1, MD1);
+      DeviceTensor<3,double> DDQ(sm1, MD1, MD1, MQ1);
+      DeviceTensor<3,double> DQQ(sm0, MD1, MQ1, MQ1);
+
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B(q,d) = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  DDD(dx,dy,dz) = x(dx,dy,dz,c,e);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     u += B(qx,dx) * DDD(dx,dy,dz);
+                  }
+                  DDQ(dz,dy,qx) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  for (int dy = 0; dy < D1D; ++dy)
+                  {
+                     u += DDQ(dz,dy,qx) * B(qy,dy);
+                  }
+                  DQQ(dz,qy,qx) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  for (int dz = 0; dz < D1D; ++dz)
+                  {
+                     u +=  DQQ(dz,qy,qx) * B(qz,dz);
+                  }
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,qz,e) = u; }
+                  if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,qz,c,e) = u; }
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+} // namespace mfem

fem/quadinterpolator_eval_by_nodes.cpp

@@ -0,0 +1,110 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "quadinterpolator_eval.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+template<>
+void QuadratureInterpolator::Values<QVectorLayout::byNODES>(
+   const Vector &e_vec, Vector &q_val) const
+{
+   const int NE = fespace->GetNE();
+   if (NE == 0) { return; }
+   const int vdim = fespace->GetVDim();
+   const int dim = fespace->GetMesh()->Dimension();
+   const FiniteElement *fe = fespace->GetFE(0);
+   const IntegrationRule *ir =
+      IntRule ? IntRule : &qspace->GetElementIntRule(0);
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const double *B = maps.B.Read();
+   const double *X = e_vec.Read();
+   double *Y = q_val.Write();
+
+   constexpr QVectorLayout L = QVectorLayout::byNODES;
+
+   const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
+
+   switch (id)
+   {
+      case 0x2133: return Eval2D<L,1,3,3>(NE,B,X,Y);
+      case 0x2124: return Eval2D<L,1,2,4>(NE,B,X,Y);
+      case 0x2132: return Eval2D<L,1,3,2>(NE,B,X,Y);
+      case 0x2134: return Eval2D<L,1,3,4>(NE,B,X,Y);
+      case 0x2143: return Eval2D<L,1,4,3>(NE,B,X,Y);
+      case 0x2144: return Eval2D<L,1,4,4>(NE,B,X,Y);
+
+      case 0x2222: return Eval2D<L,2,2,2>(NE,B,X,Y);
+      case 0x2223: return Eval2D<L,2,2,3>(NE,B,X,Y);
+      case 0x2224: return Eval2D<L,2,2,4>(NE,B,X,Y);
+      case 0x2225: return Eval2D<L,2,2,5>(NE,B,X,Y);
+      case 0x2226: return Eval2D<L,2,2,6>(NE,B,X,Y);
+      case 0x2233: return Eval2D<L,2,3,3>(NE,B,X,Y);
+      case 0x2234: return Eval2D<L,2,3,4>(NE,B,X,Y);
+      case 0x2236: return Eval2D<L,2,3,6>(NE,B,X,Y);
+      case 0x2243: return Eval2D<L,2,4,3>(NE,B,X,Y);
+      case 0x2244: return Eval2D<L,2,4,4>(NE,B,X,Y);
+      case 0x2245: return Eval2D<L,2,4,5>(NE,B,X,Y);
+      case 0x2246: return Eval2D<L,2,4,6>(NE,B,X,Y);
+      case 0x2247: return Eval2D<L,2,4,7>(NE,B,X,Y);
+      case 0x2256: return Eval2D<L,2,5,6>(NE,B,X,Y);
+
+      case 0x3124: return Eval3D<L,1,2,4>(NE,B,X,Y);
+      case 0x3133: return Eval3D<L,1,3,3>(NE,B,X,Y);
+      case 0x3134: return Eval3D<L,1,3,4>(NE,B,X,Y);
+      case 0x3136: return Eval3D<L,1,3,6>(NE,B,X,Y);
+      case 0x3143: return Eval3D<L,1,4,3>(NE,B,X,Y);
+      case 0x3144: return Eval3D<L,1,4,4>(NE,B,X,Y);
+      case 0x3148: return Eval3D<L,1,4,8>(NE,B,X,Y);
+
+      case 0x3222: return Eval3D<L,2,2,2>(NE,B,X,Y);
+      case 0x3223: return Eval3D<L,2,2,3>(NE,B,X,Y);
+      case 0x3234: return Eval3D<L,2,3,4>(NE,B,X,Y);
+
+      case 0x3323: return Eval3D<L,3,2,3>(NE,B,X,Y);
+      case 0x3324: return Eval3D<L,3,2,4>(NE,B,X,Y);
+      case 0x3325: return Eval3D<L,3,2,5>(NE,B,X,Y);
+      case 0x3326: return Eval3D<L,3,2,6>(NE,B,X,Y);
+      case 0x3333: return Eval3D<L,3,3,3>(NE,B,X,Y);
+      case 0x3334: return Eval3D<L,3,3,4>(NE,B,X,Y);
+      case 0x3335: return Eval3D<L,3,3,5>(NE,B,X,Y);
+      case 0x3336: return Eval3D<L,3,3,6>(NE,B,X,Y);
+      case 0x3343: return Eval3D<L,3,4,3>(NE,B,X,Y);
+      case 0x3344: return Eval3D<L,3,4,4>(NE,B,X,Y);
+      case 0x3346: return Eval3D<L,3,4,6>(NE,B,X,Y);
+      case 0x3347: return Eval3D<L,3,4,7>(NE,B,X,Y);
+      case 0x3348: return Eval3D<L,3,4,8>(NE,B,X,Y);
+
+      default:
+      {
+         constexpr int MD1 = 8;
+         constexpr int MQ1 = 8;
+         MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
+                     << " are not supported!");
+         MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
+                     << MQ1 << " 1D points are not supported!");
+         if (dim == 2) { Eval2D<L,0,0,0,0,MD1,MQ1>(NE,B,X,Y,vdim,D1D,Q1D); }
+         if (dim == 3) { Eval3D<L,0,0,0,MD1,MQ1>(NE,B,X,Y,vdim,D1D,Q1D); }
+         return;
+      }
+   }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Kernel not supported yet");
+}
+
+} // namespace mfem

fem/quadinterpolator_eval_by_vdim.cpp

@@ -0,0 +1,79 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "quadinterpolator_eval.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+template<>
+void QuadratureInterpolator::Values<QVectorLayout::byVDIM>(
+   const Vector &e_vec, Vector &q_val) const
+{
+   const int NE = fespace->GetNE();
+   if (NE == 0) { return; }
+   const int vdim = fespace->GetVDim();
+   const int dim = fespace->GetMesh()->Dimension();
+   const FiniteElement *fe = fespace->GetFE(0);
+   const IntegrationRule *ir =
+      IntRule ? IntRule : &qspace->GetElementIntRule(0);
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const double *B = maps.B.Read();
+   const double *X = e_vec.Read();
+   double *Y = q_val.Write();
+
+   constexpr QVectorLayout L = QVectorLayout::byVDIM;
+
+   const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
+
+   switch (id)
+   {
+      case 0x2124: return Eval2D<L,1,2,4,8>(NE,B,X,Y);
+      case 0x2136: return Eval2D<L,1,3,6,4>(NE,B,X,Y);
+      case 0x2148: return Eval2D<L,1,4,8,2>(NE,B,X,Y);
+
+      case 0x2224: return Eval2D<L,2,2,4,8>(NE,B,X,Y);
+      case 0x2234: return Eval2D<L,2,3,4,8>(NE,B,X,Y);
+      case 0x2236: return Eval2D<L,2,3,6,4>(NE,B,X,Y);
+      case 0x2248: return Eval2D<L,2,4,8,2>(NE,B,X,Y);
+
+      case 0x3124: return Eval3D<L,1,2,4>(NE,B,X,Y);
+      case 0x3136: return Eval3D<L,1,3,6>(NE,B,X,Y);
+      case 0x3148: return Eval3D<L,1,4,8>(NE,B,X,Y);
+
+      case 0x3324: return Eval3D<L,3,2,4>(NE,B,X,Y);
+      case 0x3336: return Eval3D<L,3,3,6>(NE,B,X,Y);
+      case 0x3348: return Eval3D<L,3,4,8>(NE,B,X,Y);
+
+      default:
+      {
+         constexpr int MD1 = 8;
+         constexpr int MQ1 = 8;
+         MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
+                     << " are not supported!");
+         MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
+                     << MQ1 << " 1D points are not supported!");
+         if (dim == 2) { Eval2D<L,0,0,0,0,MD1,MQ1>(NE,B,X,Y,vdim,D1D,Q1D); }
+         if (dim == 3) { Eval3D<L,0,0,0,MD1,MQ1>(NE,B,X,Y,vdim,D1D,Q1D); }
+         return;
+      }
+   }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Kernel not supported yet");
+}
+
+} // namespace mfem

fem/quadinterpolator_face.cpp

@@ -495,8 +495,9 @@ void FaceQuadratureInterpolator::Mult(
    }
 }
 
-void FaceQuadratureInterpolator::Values(
-   const Vector &e_vec, Vector &q_val) const
+
+void FaceQuadratureInterpolator::Values(const Vector &e_vec,
+                                        Vector &q_val) const
 {
    Vector q_der, q_det, q_nor;
    Mult(e_vec, VALUES, q_val, q_der, q_det, q_nor);

fem/quadinterpolator_grad.hpp

@@ -0,0 +1,282 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1, int MAX_D1D = 0, int MAX_Q1D = 0>
+static void Grad2D(const int NE,
+                   const double *b_,
+                   const double *g_,
+                   const double *x_,
+                   double *y_,
+                   const int vdim = 0,
+                   const int d1d = 0,
+                   const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto g = Reshape(g_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT ==QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, VDIM, 2, NE):
+            Reshape(y_, VDIM, 2, Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      const int tidz = MFEM_THREAD_ID(z);
+      MFEM_SHARED double s_B[MQ1*MD1];
+      MFEM_SHARED double s_G[MQ1*MD1];
+      DeviceTensor<2,double> B(s_B, Q1D, D1D);
+      DeviceTensor<2,double> G(s_G, Q1D, D1D);
+
+      MFEM_SHARED double s_X[NBZ][MD1*MD1];
+      DeviceTensor<2,double> X((double*)(s_X+tidz), MD1, MD1);
+
+      MFEM_SHARED double s_DQ[2][NBZ][MD1*MQ1];
+      DeviceTensor<2,double> DQ0((double*)(s_DQ[0]+tidz), MD1, MQ1);
+      DeviceTensor<2,double> DQ1((double*)(s_DQ[1]+tidz), MD1, MQ1);
+
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B(q,d) = b(q,d);
+               G(q,d) = g(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int c = 0; c < VDIM; ++c)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               X(dx,dy) = x(dx,dy,c,e);
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const double input = X(dx,dy);
+                  u += input * B(qx,dx);
+                  v += input * G(qx,dx);
+               }
+               DQ0(dy,qx) = u;
+               DQ1(dy,qx) = v;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  u += DQ1(dy,qx) * B(qy,dy);
+                  v += DQ0(dy,qx) * G(qy,dy);
+               }
+               if (Q_LAYOUT == QVectorLayout::byNODES)
+               {
+                  y(qx,qy,c,0,e) = u;
+                  y(qx,qy,c,1,e) = v;
+               }
+               if (Q_LAYOUT == QVectorLayout::byVDIM)
+               {
+                  y(c,0,qx,qy,e) = u;
+                  y(c,1,qx,qy,e) = v;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int MAX_D1D = 0, int MAX_Q1D = 0>
+static void Grad3D(const int NE,
+                   const double *b_,
+                   const double *g_,
+                   const double *x_,
+                   double *y_,
+                   const int vdim = 0,
+                   const int d1d = 0,
+                   const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto g = Reshape(g_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT ==QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, Q1D, VDIM, 3, NE):
+            Reshape(y_, VDIM, 3, Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+
+      const int tidz = MFEM_THREAD_ID(z);
+      MFEM_SHARED double s_B[MQ1*MD1];
+      MFEM_SHARED double s_G[MQ1*MD1];
+      DeviceTensor<2,double> B(s_B, Q1D, D1D);
+      DeviceTensor<2,double> G(s_G, Q1D, D1D);
+
+      MFEM_SHARED double sm0[3][MQ1*MQ1*MQ1];
+      MFEM_SHARED double sm1[3][MQ1*MQ1*MQ1];
+      DeviceTensor<3,double> X((double*)(sm0+2), MD1, MD1, MD1);
+      DeviceTensor<3,double> DDQ0((double*)(sm0+0), MD1, MD1, MQ1);
+      DeviceTensor<3,double> DDQ1((double*)(sm0+1), MD1, MD1, MQ1);
+      DeviceTensor<3,double> DQQ0((double*)(sm1+0), MD1, MQ1, MQ1);
+      DeviceTensor<3,double> DQQ1((double*)(sm1+1), MD1, MQ1, MQ1);
+      DeviceTensor<3,double> DQQ2((double*)(sm1+2), MD1, MQ1, MQ1);
+
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B(q,d) = b(q,d);
+               G(q,d) = g(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int c = 0; c < VDIM; ++c)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dz,z,D1D)
+               {
+                  X(dx,dy,dz) = x(dx,dy,dz,c,e);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  double v = 0.0;
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     const double input = X(dx,dy,dz);
+                     u += input * B(qx,dx);
+                     v += input * G(qx,dx);
+                  }
+                  DDQ0(dz,dy,qx) = u;
+                  DDQ1(dz,dy,qx) = v;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  double v = 0.0;
+                  double w = 0.0;
+                  for (int dy = 0; dy < D1D; ++dy)
+                  {
+                     u += DDQ1(dz,dy,qx) * B(qy,dy);
+                     v += DDQ0(dz,dy,qx) * G(qy,dy);
+                     w += DDQ0(dz,dy,qx) * B(qy,dy);
+                  }
+                  DQQ0(dz,qy,qx) = u;
+                  DQQ1(dz,qy,qx) = v;
+                  DQQ2(dz,qy,qx) = w;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  double v = 0.0;
+                  double w = 0.0;
+                  for (int dz = 0; dz < D1D; ++dz)
+                  {
+                     u += DQQ0(dz,qy,qx) * B(qz,dz);
+                     v += DQQ1(dz,qy,qx) * B(qz,dz);
+                     w += DQQ2(dz,qy,qx) * G(qz,dz);
+                  }
+                  if (Q_LAYOUT == QVectorLayout::byNODES)
+                  {
+                     y(qx,qy,qz,c,0,e) = u;
+                     y(qx,qy,qz,c,1,e) = v;
+                     y(qx,qy,qz,c,2,e) = w;
+                  }
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     y(c,0,qx,qy,qz,e) = u;
+                     y(c,1,qx,qy,qz,e) = v;
+                     y(c,2,qx,qy,qz,e) = w;
+                  }
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+} // namespace mfem

fem/quadinterpolator_grad_by_nodes.cpp

@@ -0,0 +1,109 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "quadinterpolator_grad.hpp"
+
+namespace mfem
+{
+
+template<>
+void QuadratureInterpolator::Derivatives<QVectorLayout::byNODES>(
+   const Vector &e_vec, Vector &q_der) const
+{
+   const int NE = fespace->GetNE();
+   if (NE == 0) { return; }
+
+   const int vdim = fespace->GetVDim();
+   const int dim = fespace->GetMesh()->Dimension();
+   const FiniteElement *fe = fespace->GetFE(0);
+   const IntegrationRule *ir =
+      IntRule ? IntRule : &qspace->GetElementIntRule(0);
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const double *B = maps.B.Read();
+   const double *G = maps.G.Read();
+   const double *X = e_vec.Read();
+   double *Y = q_der.Write();
+
+   constexpr QVectorLayout L = QVectorLayout::byNODES;
+
+   const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
+
+   switch (id)
+   {
+      case 0x2133: return Grad2D<L,1,3,3,16>(NE,B,G,X,Y);
+      case 0x2134: return Grad2D<L,1,3,4,16>(NE,B,G,X,Y);
+      case 0x2143: return Grad2D<L,1,4,3,16>(NE,B,G,X,Y);
+      case 0x2144: return Grad2D<L,1,4,4,16>(NE,B,G,X,Y);
+
+      case 0x2222: return Grad2D<L,2,2,2,16>(NE,B,G,X,Y);
+      case 0x2223: return Grad2D<L,2,2,3,8>(NE,B,G,X,Y);
+      case 0x2224: return Grad2D<L,2,2,4,4>(NE,B,G,X,Y);
+      case 0x2225: return Grad2D<L,2,2,5,4>(NE,B,G,X,Y);
+      case 0x2226: return Grad2D<L,2,2,6,2>(NE,B,G,X,Y);
+
+      case 0x2233: return Grad2D<L,2,3,3,2>(NE,B,G,X,Y);
+      case 0x2234: return Grad2D<L,2,3,4,4>(NE,B,G,X,Y);
+      case 0x2243: return Grad2D<L,2,4,3,4>(NE,B,G,X,Y);
+      case 0x2236: return Grad2D<L,2,3,6,2>(NE,B,G,X,Y);
+
+      case 0x2244: return Grad2D<L,2,4,4,2>(NE,B,G,X,Y);
+      case 0x2245: return Grad2D<L,2,4,5,2>(NE,B,G,X,Y);
+      case 0x2246: return Grad2D<L,2,4,6,2>(NE,B,G,X,Y);
+      case 0x2247: return Grad2D<L,2,4,7,2>(NE,B,G,X,Y);
+
+      case 0x2256: return Grad2D<L,2,5,6,2>(NE,B,G,X,Y);
+
+      case 0x3124: return Grad3D<L,1,2,4>(NE,B,G,X,Y);
+      case 0x3133: return Grad3D<L,1,3,3>(NE,B,G,X,Y);
+      case 0x3134: return Grad3D<L,1,3,4>(NE,B,G,X,Y);
+      case 0x3136: return Grad3D<L,1,3,6>(NE,B,G,X,Y);
+      case 0x3144: return Grad3D<L,1,4,4>(NE,B,G,X,Y);
+      case 0x3148: return Grad3D<L,1,4,8>(NE,B,G,X,Y);
+
+      case 0x3323: return Grad3D<L,3,2,3>(NE,B,G,X,Y);
+      case 0x3324: return Grad3D<L,3,2,4>(NE,B,G,X,Y);
+      case 0x3325: return Grad3D<L,3,2,5>(NE,B,G,X,Y);
+      case 0x3326: return Grad3D<L,3,2,6>(NE,B,G,X,Y);
+
+      case 0x3333: return Grad3D<L,3,3,3>(NE,B,G,X,Y);
+      case 0x3334: return Grad3D<L,3,3,4>(NE,B,G,X,Y);
+      case 0x3335: return Grad3D<L,3,3,5>(NE,B,G,X,Y);
+      case 0x3336: return Grad3D<L,3,3,6>(NE,B,G,X,Y);
+      case 0x3344: return Grad3D<L,3,4,4>(NE,B,G,X,Y);
+      case 0x3346: return Grad3D<L,3,4,6>(NE,B,G,X,Y);
+      case 0x3347: return Grad3D<L,3,4,7>(NE,B,G,X,Y);
+      case 0x3348: return Grad3D<L,3,4,8>(NE,B,G,X,Y);
+      default:
+      {
+         constexpr int MD1 = 8;
+         constexpr int MQ1 = 8;
+         MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
+                     << " are not supported!");
+         MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
+                     << MQ1 << " 1D points are not supported!");
+         if (dim == 2)
+         {
+            return Grad2D<L,0,0,0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D);
+         }
+         if (dim == 3)
+         {
+            return Grad3D<L,0,0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D);
+         }
+      }
+   }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Kernel not supported yet");
+}
+
+} // namespace mfem

fem/quadinterpolator_grad_by_vdim.cpp

@@ -0,0 +1,76 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "quadinterpolator_grad.hpp"
+
+namespace mfem
+{
+
+template<>
+void QuadratureInterpolator::Derivatives<QVectorLayout::byVDIM>(
+   const Vector &e_vec, Vector &q_der) const
+{
+   const int NE = fespace->GetNE();
+   if (NE == 0) { return; }
+
+   const int vdim = fespace->GetVDim();
+   const int dim = fespace->GetMesh()->Dimension();
+   const FiniteElement *fe = fespace->GetFE(0);
+   const IntegrationRule *ir =
+      IntRule ? IntRule : &qspace->GetElementIntRule(0);
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const double *B = maps.B.Read();
+   const double *G = maps.G.Read();
+   const double *X = e_vec.Read();
+   double *Y = q_der.Write();
+
+   constexpr QVectorLayout L = QVectorLayout::byVDIM;
+
+   const int id = (dim<<12) | (vdim<<8) | (D1D<<4) | Q1D;
+
+   switch (id)
+   {
+      case 0x2134: return Grad2D<L,1,3,4,8>(NE,B,G,X,Y);
+      case 0x2146: return Grad2D<L,1,4,6,4>(NE,B,G,X,Y);
+      case 0x2158: return Grad2D<L,1,5,8,2>(NE,B,G,X,Y);
+
+      case 0x2234: return Grad2D<L,2,3,4,8>(NE,B,G,X,Y);
+      case 0x2246: return Grad2D<L,2,4,6,4>(NE,B,G,X,Y);
+      case 0x2258: return Grad2D<L,2,5,8,2>(NE,B,G,X,Y);
+
+      case 0x3134: return Grad3D<L,1,3,4>(NE,B,G,X,Y);
+      case 0x3146: return Grad3D<L,1,4,6>(NE,B,G,X,Y);
+      case 0x3158: return Grad3D<L,1,5,8>(NE,B,G,X,Y);
+
+      case 0x3334: return Grad3D<L,3,3,4>(NE,B,G,X,Y);
+      case 0x3346: return Grad3D<L,3,4,6>(NE,B,G,X,Y);
+      case 0x3358: return Grad3D<L,3,5,8>(NE,B,G,X,Y);
+      default:
+      {
+         constexpr int MD1 = 8;
+         constexpr int MQ1 = 8;
+         MFEM_VERIFY(D1D <= MD1, "Orders higher than " << MD1-1
+                     << " are not supported!");
+         MFEM_VERIFY(Q1D <= MQ1, "Quadrature rules with more than "
+                     << MQ1 << " 1D points are not supported!");
+         if (dim == 2) { Grad2D<L,0,0,0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D); }
+         if (dim == 3) { Grad3D<L,0,0,0,MD1,MQ1>(NE,B,G,X,Y,vdim,D1D,Q1D); }
+         return;
+      }
+   }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Kernel not supported yet");
+}
+
+} // namespace mfem

fem/quadinterpolator_grad_phys.hpp

@@ -0,0 +1,303 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1, int MAX_D1D = 0, int MAX_Q1D = 0>
+static void PhysGrad2D(const int NE,
+                       const double *b_,
+                       const double *g_,
+                       const double *j_,
+                       const double *x_,
+                       double *y_,
+                       const int vdim = 0,
+                       const int d1d = 0,
+                       const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto g = Reshape(g_, Q1D, D1D);
+   const auto j = Reshape(j_, Q1D, Q1D, 2, 2, NE);
+   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT ==QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, VDIM, 2, NE):
+            Reshape(y_, VDIM, 2, Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      const int tidz = MFEM_THREAD_ID(z);
+      MFEM_SHARED double s_B[MQ1][MD1];
+      MFEM_SHARED double s_G[MQ1][MD1];
+      DeviceTensor<2,double> B((double*)(s_B), Q1D, D1D);
+      DeviceTensor<2,double> G((double*)(s_G), Q1D, D1D);
+
+      MFEM_SHARED double s_X[NBZ][MD1*MD1];
+      DeviceTensor<2,double> X((double*)(s_X+tidz), MD1, MD1);
+
+      MFEM_SHARED double sm[2][NBZ][MD1*MQ1];
+      DeviceTensor<2,double> DQ0((double*)(sm[0]+tidz), MD1, MQ1);
+      DeviceTensor<2,double> DQ1((double*)(sm[1]+tidz), MD1, MQ1);
+
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B(q,d) = b(q,d);
+               G(q,d) = g(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int c = 0; c < VDIM; ++c)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               X(dx,dy) = x(dx,dy,c,e);
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const double input = X(dx,dy);
+                  u += input * B(qx,dx);
+                  v += input * G(qx,dx);
+               }
+               DQ0(dy,qx) = u;
+               DQ1(dy,qx) = v;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  u += DQ1(dy,qx) * B(qy,dy);
+                  v += DQ0(dy,qx) * G(qy,dy);
+               }
+               double Jloc[4], Jinv[4];
+               Jloc[0] = j(qx,qy,0,0,e);
+               Jloc[1] = j(qx,qy,1,0,e);
+               Jloc[2] = j(qx,qy,0,1,e);
+               Jloc[3] = j(qx,qy,1,1,e);
+               kernels::CalcInverse<2>(Jloc, Jinv);
+               if (Q_LAYOUT == QVectorLayout::byVDIM)
+               {
+                  y(c,0,qx,qy,e) = Jinv[0]*u + Jinv[1]*v;
+                  y(c,1,qx,qy,e) = Jinv[2]*u + Jinv[3]*v;
+               }
+               if (Q_LAYOUT == QVectorLayout::byNODES)
+               {
+                  y(qx,qy,c,0,e) = Jinv[0]*u + Jinv[1]*v;
+                  y(qx,qy,c,1,e) = Jinv[2]*u + Jinv[3]*v;
+
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int MAX_D = 0, int MAX_Q = 0>
+static  void PhysGrad3D(const int NE,
+                        const double *b_,
+                        const double *g_,
+                        const double *j_,
+                        const double *x_,
+                        double *y_,
+                        const int vdim = 1,
+                        const int d1d = 0,
+                        const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto g = Reshape(g_, Q1D, D1D);
+   const auto j = Reshape(j_, Q1D, Q1D, Q1D, 3, 3, NE);
+   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT ==QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, Q1D, VDIM, 3, NE):
+            Reshape(y_, VDIM, 3, Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D;
+      const int tidz = MFEM_THREAD_ID(z);
+      MFEM_SHARED double s_B[MQ1][MD1];
+      MFEM_SHARED double s_G[MQ1][MD1];
+      DeviceTensor<2,double> B((double*)(s_B), Q1D, D1D);
+      DeviceTensor<2,double> G((double*)(s_G), Q1D, D1D);
+
+      MFEM_SHARED double sm0[3][MQ1*MQ1*MQ1];
+      MFEM_SHARED double sm1[3][MQ1*MQ1*MQ1];
+      DeviceTensor<3,double> X((double*)(sm0+2), MD1, MD1, MD1);
+      DeviceTensor<3,double> DDQ0((double*)(sm0+0), MD1, MD1, MQ1);
+      DeviceTensor<3,double> DDQ1((double*)(sm0+1), MD1, MD1, MQ1);
+      DeviceTensor<3,double> DQQ0((double*)(sm1+0), MD1, MQ1, MQ1);
+      DeviceTensor<3,double> DQQ1((double*)(sm1+1), MD1, MQ1, MQ1);
+      DeviceTensor<3,double> DQQ2((double*)(sm1+2), MD1, MQ1, MQ1);
+
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B(q,d) = b(q,d);
+               G(q,d) = g(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int c = 0; c < VDIM; ++c)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dz,z,D1D)
+               {
+                  X(dx,dy,dz) = x(dx,dy,dz,c,e);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  double v = 0.0;
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     const double coords = X(dx,dy,dz);
+                     u += coords * B(qx,dx);
+                     v += coords * G(qx,dx);
+                  }
+                  DDQ0(dz,dy,qx) = u;
+                  DDQ1(dz,dy,qx) = v;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  double v = 0.0;
+                  double w = 0.0;
+                  for (int dy = 0; dy < D1D; ++dy)
+                  {
+                     u += DDQ1(dz,dy,qx) * B(qy,dy);
+                     v += DDQ0(dz,dy,qx) * G(qy,dy);
+                     w += DDQ0(dz,dy,qx) * B(qy,dy);
+                  }
+                  DQQ0(dz,qy,qx) = u;
+                  DQQ1(dz,qy,qx) = v;
+                  DQQ2(dz,qy,qx) = w;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  double u = 0.0;
+                  double v = 0.0;
+                  double w = 0.0;
+                  for (int dz = 0; dz < D1D; ++dz)
+                  {
+                     u += DQQ0(dz,qy,qx) * B(qz,dz);
+                     v += DQQ1(dz,qy,qx) * B(qz,dz);
+                     w += DQQ2(dz,qy,qx) * G(qz,dz);
+                  }
+                  double Jloc[9], Jinv[9];
+                  for (int col = 0; col < 3; col++)
+                  {
+                     for (int row = 0; row < 3; row++)
+                     {
+                        Jloc[row+3*col] = j(qx,qy,qz,row,col,e);
+                     }
+                  }
+                  kernels::CalcInverse<3>(Jloc, Jinv);
+                  if (Q_LAYOUT == QVectorLayout::byNODES)
+                  {
+                     y(qx,qy,qz,c,0,e) = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
+                     y(qx,qy,qz,c,1,e) = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
+                     y(qx,qy,qz,c,2,e) = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
+                  }
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     y(c,0,qx,qy,qz,e) = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
+                     y(c,1,qx,qy,qz,e) = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
+                     y(c,2,qx,qy,qz,e) = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
+                  }
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+} // namespace mfem

fem/quadinterpolator_grad_phys_by_nodes.cpp

@@ -0,0 +1,110 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "quadinterpolator_grad_phys.hpp"
+
+namespace mfem
+{
+
+template<>
+void QuadratureInterpolator::PhysDerivatives<QVectorLayout::byNODES>(
+   const Vector &e_vec, Vector &q_der) const
+{
+   const int NE = fespace->GetNE();
+   if (NE == 0) { return; }
+   Mesh *mesh = fespace->GetMesh();
+   const int vdim = fespace->GetVDim();
+   const int dim = fespace->GetMesh()->Dimension();
+   const FiniteElement *fe = fespace->GetFE(0);
+   const IntegrationRule *ir =
+      IntRule ? IntRule : &qspace->GetElementIntRule(0);
+   constexpr DofToQuad::Mode mode = DofToQuad::TENSOR;
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
+   const GeometricFactors *geom =
+      mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
+
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+
+   const double *B = maps.B.Read();
+   const double *G = maps.G.Read();
+   const double *J = geom->J.Read();
+   const double *X = e_vec.Read();
+   double *Y = q_der.Write();
+
+   constexpr QVectorLayout L = QVectorLayout::byNODES;
+
+   const int id = (vdim<<8) | (D1D<<4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x133: return PhysGrad2D<L,1,3,3,8>(NE,B,G,J,X,Y);
+         case 0x134: return PhysGrad2D<L,1,3,4,8>(NE,B,G,J,X,Y);
+         case 0x143: return PhysGrad2D<L,1,4,3,4>(NE,B,G,J,X,Y);
+         case 0x144: return PhysGrad2D<L,1,4,4,4>(NE,B,G,J,X,Y);
+         case 0x146: return PhysGrad2D<L,1,4,6,4>(NE,B,G,J,X,Y);
+         case 0x158: return PhysGrad2D<L,1,5,8,2>(NE,B,G,J,X,Y);
+
+         case 0x233: return PhysGrad2D<L,2,3,3,8>(NE,B,G,J,X,Y);
+         case 0x234: return PhysGrad2D<L,2,3,4,8>(NE,B,G,J,X,Y);
+         case 0x243: return PhysGrad2D<L,2,4,3,4>(NE,B,G,J,X,Y);
+         case 0x244: return PhysGrad2D<L,2,4,4,4>(NE,B,G,J,X,Y);
+         case 0x246: return PhysGrad2D<L,2,4,6,4>(NE,B,G,J,X,Y);
+         case 0x258: return PhysGrad2D<L,2,5,8,2>(NE,B,G,J,X,Y);
+         default:
+         {
+            constexpr int MD = MAX_D1D;
+            constexpr int MQ = MAX_Q1D;
+            MFEM_VERIFY(D1D <= MD, "Orders higher than " << MD-1
+                        << " are not supported!");
+            MFEM_VERIFY(Q1D <= MQ, "Quadrature rules with more than " << MQ
+                        << " 1D points are not supported!");
+            PhysGrad2D<L,0,0,0,0,MD,MQ>(NE, B, G, J, X, Y, vdim, D1D, Q1D);
+            return;
+         }
+      }
+   }
+   if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x133: return PhysGrad3D<L,1,3,3>(NE,B,G,J,X,Y);
+         case 0x134: return PhysGrad3D<L,1,3,4>(NE,B,G,J,X,Y);
+         case 0x144: return PhysGrad3D<L,1,4,4>(NE,B,G,J,X,Y);
+         case 0x146: return PhysGrad3D<L,1,4,6>(NE,B,G,J,X,Y);
+         case 0x158: return PhysGrad3D<L,1,5,8>(NE,B,G,J,X,Y);
+
+         case 0x333: return PhysGrad3D<L,3,3,3>(NE,B,G,J,X,Y);
+         case 0x334: return PhysGrad3D<L,3,3,4>(NE,B,G,J,X,Y);
+         case 0x344: return PhysGrad3D<L,3,4,4>(NE,B,G,J,X,Y);
+         case 0x346: return PhysGrad3D<L,3,4,6>(NE,B,G,J,X,Y);
+         case 0x358: return PhysGrad3D<L,3,5,8>(NE,B,G,J,X,Y);
+         default:
+         {
+            constexpr int MD = 8;
+            constexpr int MQ = 8;
+            MFEM_VERIFY(D1D <= MD, "Orders higher than " << MD-1
+                        << " are not supported!");
+            MFEM_VERIFY(Q1D <= MQ, "Quadrature rules with more than " << MQ
+                        << " 1D points are not supported!");
+            PhysGrad3D<L,0,0,0,MD,MQ>(NE, B, G, J, X, Y, vdim, D1D, Q1D);
+            return;
+         }
+      }
+   }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Unknown kernel");
+}
+
+} // namespace mfem

fem/quadinterpolator_grad_phys_by_vdim.cpp

@@ -0,0 +1,101 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "quadinterpolator.hpp"
+#include "quadinterpolator_grad_phys.hpp"
+
+namespace mfem
+{
+
+template<>
+void QuadratureInterpolator::PhysDerivatives<QVectorLayout::byVDIM>(
+   const Vector &e_vec, Vector &q_der) const
+{
+   const int NE = fespace->GetNE();
+   if (NE == 0) { return; }
+   Mesh *mesh = fespace->GetMesh();
+   const int vdim = fespace->GetVDim();
+   const int dim = fespace->GetMesh()->Dimension();
+   const FiniteElement *fe = fespace->GetFE(0);
+   const IntegrationRule *ir =
+      IntRule ? IntRule : &qspace->GetElementIntRule(0);
+   constexpr DofToQuad::Mode mode = DofToQuad::TENSOR;
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
+   const GeometricFactors *geom =
+      mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
+
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+
+   const double *B = maps.B.Read();
+   const double *G = maps.G.Read();
+   const double *J = geom->J.Read();
+   const double *X = e_vec.Read();
+   double *Y = q_der.Write();
+
+   constexpr QVectorLayout L = QVectorLayout::byVDIM;
+
+   const int id = (vdim<<8) | (D1D<<4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x134: return PhysGrad2D<L,1,3,4,8>(NE, B, G, J, X, Y);
+         case 0x146: return PhysGrad2D<L,1,4,6,4>(NE, B, G, J, X, Y);
+         case 0x158: return PhysGrad2D<L,1,5,8,2>(NE, B, G, J, X, Y);
+
+         case 0x233: return PhysGrad2D<L,2,3,3,8>(NE, B, G, J, X, Y);
+         case 0x234: return PhysGrad2D<L,2,3,4,8>(NE, B, G, J, X, Y);
+         case 0x246: return PhysGrad2D<L,2,4,6,4>(NE, B, G, J, X, Y);
+         case 0x258: return PhysGrad2D<L,2,5,8,2>(NE, B, G, J, X, Y);
+         default:
+         {
+            constexpr int MD = MAX_D1D;
+            constexpr int MQ = MAX_Q1D;
+            MFEM_VERIFY(D1D <= MD, "Orders higher than " << MD-1
+                        << " are not supported!");
+            MFEM_VERIFY(Q1D <= MQ, "Quadrature rules with more than " << MQ
+                        << " 1D points are not supported!");
+            PhysGrad2D<L,0,0,0,0,MD,MQ>(NE, B, G, J, X, Y, vdim, D1D, Q1D);
+            return;
+         }
+      }
+   }
+   if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x134: return PhysGrad3D<L,1,3,4>(NE, B, G, J, X, Y);
+         case 0x146: return PhysGrad3D<L,1,4,6>(NE, B, G, J, X, Y);
+         case 0x158: return PhysGrad3D<L,1,5,8>(NE, B, G, J, X, Y);
+
+         case 0x334: return PhysGrad3D<L,3,3,4>(NE, B, G, J, X, Y);
+         case 0x346: return PhysGrad3D<L,3,4,6>(NE, B, G, J, X, Y);
+         case 0x358: return PhysGrad3D<L,3,5,8>(NE, B, G, J, X, Y);
+         default:
+         {
+            constexpr int MD = 8;
+            constexpr int MQ = 8;
+            MFEM_VERIFY(D1D <= MD, "Orders higher than " << MD-1
+                        << " are not supported!");
+            MFEM_VERIFY(Q1D <= MQ, "Quadrature rules with more than " << MQ
+                        << " 1D points are not supported!");
+            PhysGrad3D<L,0,0,0,MD,MQ>(NE, B, G, J, X, Y, vdim, D1D, Q1D);
+            return;
+         }
+      }
+   }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Unknown kernel");
+}
+
+} // namespace mfem

fem/tmop.cpp

@@ -13,6 +13,7 @@
 #include "linearform.hpp"
 #include "pgridfunc.hpp"
 #include "tmop_tools.hpp"
+#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -441,8 +442,8 @@ void TMOP_Metric_058::AssembleH(const DenseMatrix &Jpt,
 double TMOP_Metric_077::EvalW(const DenseMatrix &Jpt) const
 {
    ie.SetJacobian(Jpt.GetData());
-   const double I2 = ie.Get_I2b();
-   return  0.5*(I2*I2 + 1./(I2*I2) - 2.);
+   const double I2b = ie.Get_I2b();
+   return  0.5*(I2b*I2b + 1./(I2b*I2b) - 2.);
 }
 
 void TMOP_Metric_077::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
@@ -1038,11 +1039,10 @@ void DiscreteAdaptTC::SetTspecAtIndex(int idx, const ParGridFunction &tspec_)
 {
    const int vdim     = tspec_.FESpace()->GetVDim(),
              dof_cnt  = tspec_.Size()/vdim;
-   for (int i = 0; i < dof_cnt*vdim; i++)
-   {
-      tspec(i+idx*dof_cnt) = tspec_(i);
-   }
-
+   const auto tspec__d = tspec_.Read();
+   auto tspec_d = tspec.ReadWrite();
+   const int offset = idx*dof_cnt;
+   MFEM_FORALL(i, dof_cnt*vdim, tspec_d[i+offset] = tspec__d[i];);
    FinalizeParDiscreteTargetSpec(tspec_);
 }
 
@@ -1102,34 +1102,33 @@ void DiscreteAdaptTC::SetDiscreteTargetBase(const GridFunction &tspec_)
    // make a copy of tspec->tspec_temp, increase its size, and
    // copy data from tspec_temp -> tspec, then add new entries
    Vector tspec_temp = tspec;
+   tspec.UseDevice(true);
+   tspec_sav.UseDevice(true);
    tspec.SetSize(ncomp*dof_cnt);
 
-   for (int i = 0; i < tspec_temp.Size(); i++)
-   {
-      tspec(i) = tspec_temp(i);
-   }
+   const auto tspec_temp_d = tspec_temp.Read();
+   auto tspec_d = tspec.ReadWrite();
+   MFEM_FORALL(i, tspec_temp.Size(), tspec_d[i] = tspec_temp_d[i];);
 
-   for (int i = 0; i < dof_cnt*vdim; i++)
-   {
-      tspec(i+(ncomp-vdim)*dof_cnt) = tspec_(i);
-   }
+   const auto tspec__d = tspec_.Read();
+   const int offset = (ncomp-vdim)*dof_cnt;
+   MFEM_FORALL(i, dof_cnt*vdim, tspec_d[i+offset] = tspec__d[i];);
 }
 
 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const GridFunction &tspec_)
 {
    const int vdim     = tspec_.FESpace()->GetVDim(),
              dof_cnt  = tspec_.Size()/vdim;
-   for (int i = 0; i < dof_cnt*vdim; i++)
-   {
-      tspec(i+idx*dof_cnt) = tspec_(i);
-   }
 
+   const auto tspec__d = tspec_.Read();
+   auto tspec_d = tspec.ReadWrite();
+   const int offset = idx*dof_cnt;
+   MFEM_FORALL(i, dof_cnt*vdim, tspec_d[i+offset] = tspec__d[i];);
    FinalizeSerialDiscreteTargetSpec();
 }
 
 void DiscreteAdaptTC::SetSerialDiscreteTargetSize(const GridFunction &tspec_)
 {
-
    if (sizeidx > -1) { SetTspecAtIndex(sizeidx, tspec_); return; }
    sizeidx = ncomp;
    SetDiscreteTargetBase(tspec_);
@@ -1257,16 +1256,17 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
 
          Vector shape(ndofs), tspec_vals(ntspec_dofs), par_vals,
                 par_vals_c1, par_vals_c2, par_vals_c3;
-
          Array<int> dofs;
          DenseMatrix D_rho(dim), Q_phi(dim), R_theta(dim);
          tspec_fesv->GetElementVDofs(e_id, dofs);
+         tspec.UseDevice(true);
          tspec.GetSubVector(dofs, tspec_vals);
 
          for (int q = 0; q < nqp; q++)
          {
             const IntegrationPoint &ip = ir.IntPoint(q);
             tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+
             Jtr(q) = Wideal; // Initialize to identity
             for (int d = 0; d < 4; d++)
             {
@@ -1872,6 +1872,17 @@ void AdaptivityEvaluator::SetParMetaInfo(const ParMesh &m,
 }
 #endif
 
+void AdaptivityEvaluator::ClearGeometricFactors()
+{
+#ifdef MFEM_USE_MPI
+   if (pmesh) pmesh->DeleteGeometricFactors();
+   if (pfes) pfes->GetParMesh()->DeleteGeometricFactors();
+#else
+   if (mesh) mesh->DeleteGeometricFactors();
+   if (fes) fes->GetMesh()->DeleteGeometricFactors();
+#endif
+}
+
 AdaptivityEvaluator::~AdaptivityEvaluator()
 {
    delete fes;
@@ -1899,6 +1910,7 @@ void TMOP_Integrator::EnableLimiting(const GridFunction &n0,
 {
    EnableLimiting(n0, w0, lfunc);
    lim_dist = &dist;
+   if (PA.enabled) { EnableLimitingPA(n0); }
 }
 void TMOP_Integrator::EnableLimiting(const GridFunction &n0, Coefficient &w0,
                                      TMOP_LimiterFunction *lfunc)
@@ -2044,7 +2056,8 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
          PMatI.MultTranspose(shape, p);
          pos0.MultTranspose(shape, p0);
          val += lim_normal *
-                lim_func->Eval(p, p0, d_vals(i)) * coeff0->Eval(*Tpr, ip);
+                lim_func->Eval(p, p0, d_vals(i)) *
+                coeff0->Eval(*Tpr, ip);
       }
 
       if (adaptive_limiting)
@@ -2724,6 +2737,8 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
 
 void TMOP_Integrator::UpdateAfterMeshChange(const Vector &new_x)
 {
+   PA.setup_Jtr = false;
+   PA.setup_Grad = false;
    // Update zeta if adaptive limiting is enabled.
    if (zeta) { adapt_eval->ComputeAtNewPosition(new_x, *zeta); }
 }
@@ -2879,6 +2894,49 @@ void TMOPComboIntegrator::ParEnableNormalization(const ParGridFunction &x)
 }
 #endif
 
+void TMOPComboIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      tmopi[i]->AssemblePA(fes);
+   }
+}
+
+void TMOPComboIntegrator::AssembleGradientDiagonalPA(const Vector &xe,
+                                                     Vector &de) const
+{
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      tmopi[i]->AssembleGradientDiagonalPA(xe, de);
+   }
+}
+
+void TMOPComboIntegrator::AddMultPA(const Vector &xe, Vector &ye) const
+{
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      tmopi[i]->AddMultPA(xe, ye);
+   }
+}
+
+void TMOPComboIntegrator::AddMultGradPA(const Vector &xe, const Vector &re,
+                                        Vector &ce) const
+{
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      tmopi[i]->AddMultGradPA(xe, re, ce);
+   }
+}
+
+double TMOPComboIntegrator::GetGridFunctionEnergyPA(const Vector &xe) const
+{
+   double energy = 0.0;
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      energy += tmopi[i]->GetGridFunctionEnergyPA(xe);
+   }
+   return energy;
+}
 
 void InterpolateTMOP_QualityMetric(TMOP_QualityMetric &metric,
                                    const TargetConstructor &tc,

fem/tmop.hpp

@@ -68,6 +68,10 @@ public:
    */
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const = 0;
+
+   /** @brief Return the metric ID.
+    */
+   virtual int Id() const { return 0; }
 };
 
 
@@ -85,6 +89,8 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 1; }
 };
 
 /// Skew metric, 2D.
@@ -176,6 +182,8 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 2; }
 };
 
 /// Shape & area, ideal barrier metric, 2D
@@ -192,6 +200,8 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 7; }
 };
 
 /// Shape & area metric, 2D
@@ -278,7 +288,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
 };
 
 /// Shape, ideal barrier metric, 2D
@@ -296,7 +305,6 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
-
 };
 
 /// Area, ideal barrier metric, 2D
@@ -314,6 +322,7 @@ public:
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
 
+   virtual int Id() const { return 77; }
 };
 
 /// Shape & orientation metric, 2D.
@@ -400,6 +409,8 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 302; }
 };
 
 /// Shape, ideal barrier metric, 3D
@@ -416,6 +427,8 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 303; }
 };
 
 /// Volume metric, 3D
@@ -432,6 +445,8 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 315; }
 };
 
 /// Volume, ideal barrier metric, 3D
@@ -466,6 +481,8 @@ public:
 
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 321; }
 };
 
 /// Shifted barrier form of 3D metric 16 (volume, ideal barrier metric), 3D
@@ -589,6 +606,8 @@ public:
 
    virtual void ComputeAtNewPosition(const Vector &new_nodes,
                                      Vector &new_field) = 0;
+
+   void ClearGeometricFactors();
 };
 
 /** @brief Base class representing target-matrix construction algorithms for
@@ -664,9 +683,14 @@ public:
        nodes are used by all target types except IDEAL_SHAPE_UNIT_SIZE. */
    void SetNodes(const GridFunction &n) { nodes = &n; avg_volume = 0.0; }
 
+   /** @brief Get the nodes to be used in the target-matrix construction. */
+   const GridFunction *GetNodes() const { return nodes; }
+
    /// Used by target type IDEAL_SHAPE_EQUAL_SIZE. The default volume scale is 1.
    void SetVolumeScale(double vol_scale) { volume_scale = vol_scale; }
 
+   const TargetType &Type() const { return target_type; }
+
    /// Checks if the target matrices contain non-trivial size specification.
    virtual bool ContainsVolumeInfo() const;
 
@@ -678,6 +702,16 @@ public:
                                       const Vector &elfun,
                                       DenseTensor &Jtr) const;
 
+   template<int DIM>
+   bool ComputeElementTargetsPA(const FiniteElementSpace *fes,
+                                const IntegrationRule *ir,
+                                DenseTensor &Jtr,
+                                const Vector &xe = Vector()) const;
+   virtual bool ComputeElementTargetsPA(const FiniteElementSpace *fes,
+                                        const IntegrationRule *ir,
+                                        DenseTensor &Jtr,
+                                        const Vector &xe = Vector()) const;
+
    virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
                                               const Vector &elfun,
                                               IsoparametricTransformation &Tpr,
@@ -723,6 +757,11 @@ public:
                                       const Vector &elfun,
                                       DenseTensor &Jtr) const;
 
+   virtual bool ComputeElementTargetsPA(const FiniteElementSpace *fes,
+                                        const IntegrationRule *ir,
+                                        DenseTensor &Jtr,
+                                        const Vector &xe = Vector()) const;
+
    virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
                                               const Vector &elfun,
                                               IsoparametricTransformation &Tpr,
@@ -765,14 +804,21 @@ protected:
    // Owned.
    AdaptivityEvaluator *adapt_eval;
 
-   void SetDiscreteTargetBase(const GridFunction &tspec_);
-   void SetTspecAtIndex(int idx, const GridFunction &tspec_);
+   // PA extension
+   struct { mutable Vector tspec_e; } PA;
+
    void FinalizeSerialDiscreteTargetSpec();
 #ifdef MFEM_USE_MPI
-   void SetTspecAtIndex(int idx, const ParGridFunction &tspec_);
    void FinalizeParDiscreteTargetSpec(const ParGridFunction &tspec_);
 #endif
 
+public: // MFEM_FORALL nvcc restriction that it must be public
+   void SetDiscreteTargetBase(const GridFunction &tspec_);
+   void SetTspecAtIndex(int idx, const GridFunction &tspec_);
+#ifdef MFEM_USE_MPI
+   void SetTspecAtIndex(int idx, const ParGridFunction &tspec_);
+#endif
+
 public:
    DiscreteAdaptTC(TargetType ttype)
       : TargetConstructor(ttype),
@@ -855,6 +901,7 @@ public:
    const Vector &GetTspecPert1H()   { return tspec_pert1h; }
    const Vector &GetTspecPert2H()   { return tspec_pert2h; }
    const Vector &GetTspecPertMixH() { return tspec_pertmix; }
+   const FiniteElementSpace *GetTspecFesv() const { return tspec_fesv; }
 
    /** @brief Given an element and quadrature rule, computes ref->target
        transformation Jacobians for each quadrature point in the element.
@@ -866,6 +913,11 @@ public:
                                       const Vector &elfun,
                                       DenseTensor &Jtr) const;
 
+   virtual bool ComputeElementTargetsPA(const FiniteElementSpace *fes,
+                                        const IntegrationRule *ir,
+                                        DenseTensor &Jtr,
+                                        const Vector &xe = Vector()) const;
+
    virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
                                               const Vector &elfun,
                                               IsoparametricTransformation &Tpr,
@@ -941,10 +993,25 @@ protected:
    //        output - the result of AssembleElementVector() (dof x dim).
    DenseMatrix DSh, DS, Jrt, Jpr, Jpt, P, PMatI, PMatO;
 
+   // PA extension
+   struct
+   {
+      bool enabled;
+      int dim, ne, nq;
+      mutable DenseTensor Jtr;
+      mutable bool setup_Grad, setup_Jtr;
+      mutable Vector E, O, W, X0, H, C0, LD, H0;
+      const DofToQuad *maps;
+      const DofToQuad *maps_lim = nullptr;
+      const GeometricFactors *geom;
+      const FiniteElementSpace *fes;
+      const Operator *R;
+      const IntegrationRule *ir;
+   } PA;
+
    void ComputeNormalizationEnergies(const GridFunction &x,
                                      double &metric_energy, double &lim_energy);
 
-
    void AssembleElementVectorExact(const FiniteElement &el,
                                    ElementTransformation &T,
                                    const Vector &elfun, Vector &elvect);
@@ -1015,7 +1082,7 @@ public:
         zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
         discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
-   { }
+   { PA.enabled = false; }
 
    ~TMOP_Integrator();
 
@@ -1083,6 +1150,45 @@ public:
    virtual void AssembleElementGrad(const FiniteElement &el,
                                     ElementTransformation &T,
                                     const Vector &elfun, DenseMatrix &elmat);
+   /// PA extension
+   void SetupGradPA(const Vector &xe) const;
+   void EnableLimitingPA(const GridFunction &n0);
+   void ComputeElementTargetsPA(const Vector &xe = Vector()) const;
+
+   using NonlinearFormIntegrator::GetGridFunctionEnergyPA;
+   double GetGridFunctionEnergyPA_2D(const Vector&) const;
+   double GetGridFunctionEnergyPA_C0_2D(const Vector&) const;
+   double GetGridFunctionEnergyPA_3D(const Vector&) const;
+   double GetGridFunctionEnergyPA_C0_3D(const Vector&) const;
+   virtual double GetGridFunctionEnergyPA(const Vector&) const;
+
+   using NonlinearFormIntegrator::AssemblePA;
+   virtual void AssemblePA(const FiniteElementSpace&);
+
+   virtual void AssembleGradientDiagonalPA(const Vector&, Vector&) const;
+   void AssembleDiagonalPA_2D(Vector&) const;
+   void AssembleDiagonalPA_3D(Vector&) const;
+   void AssembleDiagonalPA_C0_2D(Vector&) const;
+   void AssembleDiagonalPA_C0_3D(Vector&) const;
+
+   using NonlinearFormIntegrator::AddMultPA;
+   void AddMultPA_2D(const Vector&, Vector&) const;
+   void AddMultPA_3D(const Vector&, Vector&) const;
+   void AddMultPA_C0_2D(const Vector&, Vector&) const;
+   void AddMultPA_C0_3D(const Vector&, Vector&) const;
+   virtual void AddMultPA(const Vector&, Vector&) const;
+
+   using NonlinearFormIntegrator::AddMultGradPA;
+   void AddMultGradPA_2D(const Vector&, Vector&) const;
+   void AddMultGradPA_3D(const Vector&, const Vector&, Vector&) const;
+   void AddMultGradPA_C0_2D(const Vector&, const Vector&, Vector&) const;
+   void AddMultGradPA_C0_3D(const Vector&, const Vector&, Vector&) const;
+   virtual void AddMultGradPA(const Vector&, const Vector&, Vector&) const;
+
+   void AssembleGradPA_2D(const Vector&) const;
+   void AssembleGradPA_3D(const Vector&) const;
+   void AssembleGradPA_C0_2D(const Vector&) const;
+   void AssembleGradPA_C0_3D(const Vector&) const;
 
    DiscreteAdaptTC *GetDiscreteAdaptTC() const { return discr_tc; }
 
@@ -1105,6 +1211,17 @@ public:
 
    /** @brief Flag to control if exact action of Integration is effected. */
    void SetExactActionFlag(bool flag_) { exact_action = flag_; }
+
+   void ReleaseTemporaryMemory()
+   {
+      if (PA.enabled)
+      {
+         PA.H.GetMemory().DeleteDevice();
+         PA.H0.GetMemory().DeleteDevice();
+         //PA.Jtr.GetMemory().DeleteDevice();
+         //PA.setup_Jtr = false;
+      }
+   }
 };
 
 class TMOPComboIntegrator : public NonlinearFormIntegrator
@@ -1153,6 +1270,17 @@ public:
 #ifdef MFEM_USE_MPI
    void ParEnableNormalization(const ParGridFunction &x);
 #endif
+
+   /// PA extension
+   using NonlinearFormIntegrator::AssemblePA;
+   virtual void AssemblePA(const FiniteElementSpace&);
+   virtual void AssembleGradientDiagonalPA(const Vector&, Vector&) const;
+   using NonlinearFormIntegrator::AddMultPA;
+   virtual void AddMultPA(const Vector&, Vector&) const;
+   using NonlinearFormIntegrator::AddMultGradPA;
+   virtual void AddMultGradPA(const Vector&, const Vector&, Vector&) const;
+   using NonlinearFormIntegrator::GetGridFunctionEnergyPA;
+   virtual double GetGridFunctionEnergyPA(const Vector&) const;
 };
 
 /// Interpolates the @a metric's values at the nodes of @a metric_gf.

fem/tmop_pa.cpp

@@ -0,0 +1,363 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "linearform.hpp"
+#include "pgridfunc.hpp"
+#include "tmop_tools.hpp"
+#include "quadinterpolator.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+
+void TMOP_Integrator::SetupGradPA(const Vector &xe) const
+{
+   MFEM_VERIFY(PA.R, "PA extension setup has not been done!");
+   PA.setup_Grad = true;
+
+   if (PA.dim == 2)
+   {
+      AssembleGradPA_2D(xe);
+      if (coeff0) { AssembleGradPA_C0_2D(xe); }
+   }
+
+   if (PA.dim == 3)
+   {
+      AssembleGradPA_3D(xe);
+      if (coeff0) { AssembleGradPA_C0_3D(xe); }
+   }
+}
+
+// We might come here w/o knowing that PA will be used.
+// It is the case when EnableLimiting is called before the Setup => AssemblePA.
+void TMOP_Integrator::EnableLimitingPA(const GridFunction &n0)
+{
+   MFEM_VERIFY(PA.enabled, "EnableLimitingPA but PA is not enabled!");
+   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+
+   // Nodes0
+   const FiniteElementSpace *n0_fes = n0.FESpace();
+   const Operator *n0_R = n0_fes->GetElementRestriction(ordering);
+   PA.X0.SetSize(n0_R->Height(), Device::GetMemoryType());
+   PA.X0.UseDevice(true);
+   n0_R->Mult(n0, PA.X0);
+
+   // Get the 1D maps for the distance FE space.
+   const IntegrationRule &ir = *EnergyIntegrationRule(*n0.FESpace()->GetFE(0));
+   PA.maps_lim =
+      &lim_dist->FESpace()->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
+
+   // lim_dist & lim_func checks
+   MFEM_VERIFY(lim_dist, "No lim_dist!")
+   const FiniteElementSpace *ld_fes = lim_dist->FESpace();
+   const Operator *ld_R = ld_fes->GetElementRestriction(ordering);
+   MFEM_VERIFY(ld_R, "No lim_dist restriction operator found!");
+   PA.LD.SetSize(ld_R->Height(), Device::GetMemoryType());
+   PA.LD.UseDevice(true);
+   ld_R->Mult(*lim_dist, PA.LD);
+
+   // Only TMOP_QuadraticLimiter is supported
+   MFEM_VERIFY(lim_func, "No lim_func!")
+   MFEM_VERIFY(dynamic_cast<TMOP_QuadraticLimiter*>(lim_func),
+               "Only TMOP_QuadraticLimiter is supported");
+}
+
+bool TargetConstructor::ComputeElementTargetsPA(const FiniteElementSpace *fes,
+                                                const IntegrationRule *ir,
+                                                DenseTensor &Jtr,
+                                                const Vector &xe) const
+{
+   MFEM_VERIFY(Jtr.SizeI() == Jtr.SizeJ() && Jtr.SizeI() > 1, "");
+   const int dim = Jtr.SizeI();
+   if (dim == 2) { return ComputeElementTargetsPA<2>(fes, ir, Jtr, xe); }
+   if (dim == 3) { return ComputeElementTargetsPA<3>(fes, ir, Jtr, xe); }
+   return false;
+}
+
+bool AnalyticAdaptTC::ComputeElementTargetsPA(const FiniteElementSpace *fes,
+                                              const IntegrationRule *ir,
+                                              DenseTensor &Jtr,
+                                              const Vector &xe) const
+{
+   return false;
+}
+
+
+// Code paths leading to ComputeElementTargets:
+// - GetElementEnergy(elfun) which is done through GetGridFunctionEnergyPA(x)
+// - AssembleElementVectorExact(elfun)
+// - AssembleElementGradExact(elfun)
+// - EnableNormalization(x) -> ComputeNormalizationEnergies(x)
+// - (AssembleElementVectorFD(elfun))
+// - (AssembleElementGradFD(elfun))
+// ============================================================================
+//      - TargetConstructor():
+//          - IDEAL_SHAPE_UNIT_SIZE: Wideal
+//          - IDEAL_SHAPE_EQUAL_SIZE: α * Wideal
+//          - IDEAL_SHAPE_GIVEN_SIZE: β * Wideal
+//          - GIVEN_SHAPE_AND_SIZE:   β * Wideal
+//      - AnalyticAdaptTC(elfun):
+//          - GIVEN_FULL: matrix_tspec->Eval(Jtr(elfun))
+//      - DiscreteAdaptTC():
+//          - IDEAL_SHAPE_GIVEN_SIZE: size^{1.0/dim} * Jtr(i)  (size)
+//          - GIVEN_SHAPE_AND_SIZE:   Jtr(i) *= D_rho          (ratio)
+//                                    Jtr(i) *= Q_phi          (skew)
+//                                    Jtr(i) *= R_theta        (orientation)
+void TMOP_Integrator::ComputeElementTargetsPA(const Vector &xe) const
+{
+   PA.setup_Jtr = false;
+   const FiniteElementSpace *fes = PA.fes;
+   const IntegrationRule *ir = EnergyIntegrationRule(*fes->GetFE(0));
+   const TargetConstructor::TargetType &target_type = targetC->Type();
+   const DiscreteAdaptTC *discr_tc = GetDiscreteAdaptTC();
+
+   // Skip when TargetConstructor needs the nodes but have not been set
+   const bool use_nodes =
+      target_type == TargetConstructor::IDEAL_SHAPE_EQUAL_SIZE ||
+      target_type == TargetConstructor::IDEAL_SHAPE_GIVEN_SIZE ||
+      target_type == TargetConstructor::GIVEN_SHAPE_AND_SIZE;
+   if (targetC && !discr_tc && use_nodes && !targetC->GetNodes()) { return; }
+
+   // Try to use the TargetConstructor ComputeElementTargetsPA
+   PA.setup_Jtr = targetC->ComputeElementTargetsPA(fes, ir, PA.Jtr);
+   if (PA.setup_Jtr) { return; }
+
+   // Defaulting to host version
+   PA.Jtr.HostWrite();
+
+   const int NE = PA.ne;
+   const int NQ = PA.nq;
+   const int dim = PA.dim;
+   DenseTensor &Jtr = PA.Jtr;
+
+   Vector x;
+   const bool useable_input_vector = xe.Size() > 0;
+   const bool use_input_vector = target_type == TargetConstructor::GIVEN_FULL;
+
+   if (use_input_vector && !useable_input_vector) { return; }
+
+   if (discr_tc && !discr_tc->GetTspecFesv()) { return; }
+
+   if (use_input_vector)
+   {
+      x.SetSize(PA.R->Width(), Device::GetMemoryType());
+      x.UseDevice(true);
+      PA.R->MultTranspose(xe, x);
+      // Scale by weights
+      const int N = PA.W.Size();
+      const auto W = Reshape(PA.W.Read(), N);
+      auto X = Reshape(x.ReadWrite(), N);
+      MFEM_FORALL(i, N, X(i) /= W(i););
+   }
+
+   // Use TargetConstructor::ComputeElementTargets to fill the PA.Jtr
+   Vector elfun;
+   Array<int> vdofs;
+   DenseTensor J;
+   for (int e = 0; e < NE; e++)
+   {
+      const FiniteElement &fe = *fes->GetFE(e);
+      if (use_input_vector)
+      {
+         fes->GetElementVDofs(e, vdofs);
+         x.GetSubVector(vdofs, elfun);
+      }
+      J.UseExternalData(Jtr(e*NQ).Data(), dim, dim, NQ);
+      targetC->ComputeElementTargets(e, fe, *ir, elfun, J);
+   }
+   PA.setup_Jtr = true;
+}
+
+void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   PA.enabled = true;
+   MFEM_ASSERT(fes.GetMesh()->GetNE() > 0, "");
+   PA.ir = EnergyIntegrationRule(*fes.GetFE(0));
+   const IntegrationRule *ir = PA.ir;
+   MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
+               "PA Only supports Ordering::byNODES!");
+
+   PA.fes = &fes;
+   Mesh *mesh = fes.GetMesh();
+   const int nq = PA.nq = ir->GetNPoints();
+   const int ne = PA.ne = fes.GetMesh()->GetNE();
+   const int dim = PA.dim = mesh->Dimension();
+   MFEM_VERIFY(PA.dim == 2 || PA.dim == 3, "Not yet implemented!");
+
+   const DofToQuad::Mode mode = DofToQuad::TENSOR;
+   PA.maps = &fes.GetFE(0)->GetDofToQuad(*ir, mode);
+   PA.geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mode);
+
+   // Energy vector
+   PA.E.UseDevice(true);
+   PA.E.SetSize(ne*nq, Device::GetDeviceMemoryType());
+
+   // Setup initialization
+   PA.setup_Jtr = false;
+   PA.setup_Grad = false;
+
+
+#ifdef MFEM_USE_UMPIRE
+   const MemoryType temp_type = Device::GetDeviceMemoryType() == MemoryType::DEVICE_UMPIRE
+      ? MemoryType::DEVICE_UMPIRE_2 : Device::GetDeviceMemoryType();
+#else
+   const MemoryType temp_type = Device::GetDeviceMemoryType();
+#endif
+
+   // H for Grad
+   PA.H.SetSize(dim*dim * dim*dim * nq*ne, temp_type);
+   // H0 for coeff0
+   PA.H0.SetSize(dim * dim * nq*ne, temp_type);
+
+   // Restriction setup
+   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   PA.R = fes.GetElementRestriction(ordering);
+   MFEM_VERIFY(PA.R, "Not yet implemented!");
+
+   // Weight of the R^t
+   PA.W.SetSize(PA.R->Width(), Device::GetDeviceMemoryType());
+   PA.W.UseDevice(true);
+   PA.O.SetSize(dim*ne*nq, Device::GetDeviceMemoryType());
+   PA.O.UseDevice(true);
+   PA.O = 1.0;
+   PA.R->MultTranspose(PA.O, PA.W);
+
+   // Scalar vector of '1'
+   PA.O.SetSize(ne*nq, Device::GetDeviceMemoryType());
+   PA.O = 1.0;
+
+   // TargetConstructor TargetType setup
+   PA.Jtr.SetSize(dim, dim, PA.ne*PA.nq);//, temp_type);
+   ComputeElementTargetsPA();
+
+   // Coeff0 PA.C0
+   PA.C0.UseDevice(true);
+   if (coeff0 == nullptr)
+   {
+      PA.C0.SetSize(1, Device::GetMemoryType());
+      PA.C0.HostWrite();
+      PA.C0(0) = 0.0;
+   }
+   else if (ConstantCoefficient* cQ =
+               dynamic_cast<ConstantCoefficient*>(coeff0))
+   {
+      PA.C0.SetSize(1, Device::GetMemoryType());
+      PA.C0.HostWrite();
+      PA.C0(0) = cQ->constant;
+   }
+   else
+   {
+      PA.C0.SetSize(PA.nq * PA.ne, Device::GetMemoryType());
+      auto C0 = Reshape(PA.C0.HostWrite(), PA.nq, PA.ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C0(q,e) = coeff0->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
+
+   if (coeff0)
+   {
+      MFEM_VERIFY(nodes0, "nodes0 has not been set!");
+      EnableLimitingPA(*nodes0);
+   }
+}
+
+void TMOP_Integrator::AssembleGradientDiagonalPA(const Vector &xe,
+                                                 Vector &de) const
+{
+   MFEM_VERIFY(PA.R, "PA extension setup has not been done!");
+
+   if (!PA.setup_Jtr) { ComputeElementTargetsPA(xe); }
+
+   if (!PA.setup_Grad) { SetupGradPA(xe); }
+
+   if (PA.dim == 2)
+   {
+      AssembleDiagonalPA_2D(de);
+      if (coeff0) { AssembleDiagonalPA_C0_2D(de); }
+   }
+   else if (PA.dim == 3)
+   {
+      AssembleDiagonalPA_3D(de);
+      if (coeff0) { AssembleDiagonalPA_C0_3D(de); }
+   }
+   else
+   {
+      MFEM_ABORT("3D diagonal computation is WIP.");
+   }
+}
+
+void TMOP_Integrator::AddMultPA(const Vector &xe, Vector &ye) const
+{
+   if (!PA.setup_Jtr) { ComputeElementTargetsPA(); }
+
+   if (PA.dim == 2)
+   {
+      AddMultPA_2D(xe,ye);
+      if (coeff0) { AddMultPA_C0_2D(xe,ye); }
+   }
+
+   if (PA.dim == 3)
+   {
+      AddMultPA_3D(xe,ye);
+      if (coeff0) { AddMultPA_C0_3D(xe,ye); }
+   }
+}
+
+void TMOP_Integrator::AddMultGradPA(const Vector &xe,
+                                    const Vector &re, Vector &ce) const
+{
+   if (!PA.setup_Jtr) { ComputeElementTargetsPA(xe); }
+
+   if (!PA.setup_Grad) { SetupGradPA(xe); }
+
+   if (PA.dim == 2)
+   {
+      AddMultGradPA_2D(re,ce);
+      if (coeff0) { AddMultGradPA_C0_2D(xe,re,ce); }
+   }
+
+   if (PA.dim == 3)
+   {
+      AddMultGradPA_3D(xe,re,ce);
+      if (coeff0) { AddMultGradPA_C0_3D(xe,re,ce); }
+   }
+}
+
+double TMOP_Integrator::GetGridFunctionEnergyPA(const Vector &xe) const
+{
+   double energy = 0.0;
+
+   ComputeElementTargetsPA(xe);
+
+   if (PA.dim == 2)
+   {
+      energy = GetGridFunctionEnergyPA_2D(xe);
+      if (coeff0) { energy += GetGridFunctionEnergyPA_C0_2D(xe); }
+   }
+
+   if (PA.dim == 3)
+   {
+      energy = GetGridFunctionEnergyPA_3D(xe);
+      if (coeff0) { energy += GetGridFunctionEnergyPA_C0_3D(xe); }
+   }
+
+   return energy;
+}
+
+} // namespace mfem

fem/tmop_pa.hpp

@@ -0,0 +1,143 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_TMOP_PA_HPP
+#define MFEM_TMOP_PA_HPP
+
+#include "../config/config.hpp"
+#include "../linalg/dtensor.hpp"
+
+#include "../fem/kernels.hpp"
+
+#include <unordered_map>
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+/// Generic emplace
+template<typename K, const int N,
+         typename Key_t = typename K::Key_t,
+         typename Kernel_t = typename K::Kernel_t>
+void emplace(std::unordered_map<Key_t, Kernel_t> &map)
+{
+   constexpr Key_t key = K::template GetKey<N>();
+   constexpr Kernel_t value = K::template GetValue<key>();
+   map.emplace(key, value);
+}
+
+/// Instances
+template<class K, typename T, T... idx>
+struct instances
+{
+   static void Fill(std::unordered_map<typename K::Key_t,
+                    typename K::Kernel_t> &map)
+   {
+      using unused = int[];
+      (void) unused {0, (emplace<K,idx>(map), 0)... };
+   }
+};
+
+/// Cat instances
+template<class K, typename Offset, typename Lhs, typename Rhs> struct cat;
+template<class K, typename T, T Offset, T... Lhs, T... Rhs>
+struct cat<K, std::integral_constant<T, Offset>,
+          instances<K, T, Lhs...>,
+          instances<K, T, Rhs...> >
+{ using type = instances<K, T, Lhs..., (Offset + Rhs)...>; };
+
+/// Sequence, empty and one element terminal cases
+template<class K, typename T, typename N>
+struct sequence
+{
+   using Lhs = std::integral_constant<T, N::value/2>;
+   using Rhs = std::integral_constant<T, N::value-Lhs::value>;
+   using type = typename cat<K, Lhs,
+         typename sequence<K, T, Lhs>::type,
+         typename sequence<K, T, Rhs>::type>::type;
+};
+
+template<class K, typename T>
+struct sequence<K, T, std::integral_constant<T,0> >
+{ using type = instances<K,T>; };
+
+template<class K, typename T>
+struct sequence<K, T, std::integral_constant<T,1> >
+{ using type = instances<K,T,0>; };
+
+/// Make_sequence
+template<class Instance, typename T = typename Instance::Key_t>
+using make_sequence =
+   typename sequence<Instance, T, std::integral_constant<T,Instance::N> >::type;
+
+/// Instantiator class
+template<class Instance,
+         typename Key_t = typename Instance::Key_t,
+         typename Return_t = typename Instance::Return_t,
+         typename Kernel_t = typename Instance::Kernel_t>
+class Instantiator
+{
+private:
+   using map_t = std::unordered_map<Key_t, Kernel_t>;
+   map_t map;
+
+public:
+   Instantiator() { make_sequence<Instance>().Fill(map); }
+
+   bool Find(const Key_t id)
+   {
+      return (map.find(id) != map.end()) ? true : false;
+   }
+
+   Kernel_t At(const Key_t id) { return map.at(id); }
+};
+
+/// MFEM_REGISTER_TMOP_KERNELS macro:
+/// - forward declaration of the kernel
+/// - kernel pointer declaration
+/// - struct K##name##_T definition
+/// - Instantiator definition
+/// - re-use kernel return type and name before its body
+#define MFEM_REGISTER_TMOP_KERNELS(return_t, kernel, ...) \
+template<int T_D1D = 0, int T_Q1D = 0, int T_MAX = 0> \
+    return_t kernel(__VA_ARGS__);\
+typedef return_t (*kernel##_p)(__VA_ARGS__);\
+struct K##kernel##_T {\
+   static const int N = 14;\
+   using Key_t = std::size_t;\
+   using Kernel_t = kernel##_p;\
+   using Return_t = return_t;\
+   template<Key_t I> static constexpr Key_t GetKey() noexcept { return \
+     I==0 ? 0x22 : I==1 ? 0x23 : I==2 ? 0x24 : I==3 ? 0x25 : I==4 ? 0x26 :\
+     I==5 ? 0x33 : I==6 ? 0x34 : I==7 ? 0x35 : I==8 ? 0x36  :\
+     I==9 ? 0x44 : I==10 ? 0x45 : I==11 ? 0x46 :\
+     I==12 ? 0x55 : I==13 ? 0x56 : 0; }\
+   template<Key_t ID> static constexpr Kernel_t GetValue() noexcept\
+   { return &kernel<(ID>>4)&0xF, ID&0xF>; }\
+};\
+static kernels::Instantiator<K##kernel##_T> K##kernel;\
+template<int T_D1D, int T_Q1D, int T_MAX> return_t kernel(__VA_ARGS__)
+
+/// MFEM_LAUNCH_TMOP_KERNEL macro
+#define MFEM_LAUNCH_TMOP_KERNEL(kernel, id, ...)\
+if (K##kernel.Find(id)) { return K##kernel.At(id)(__VA_ARGS__,0,0); }\
+else {\
+   constexpr int T_MAX = 4;\
+   MFEM_VERIFY(D1D <= MAX_D1D && Q1D <= MAX_Q1D, "Max size error!");\
+   return kernel<0,0,T_MAX>(__VA_ARGS__,D1D,Q1D); }
+
+} // namespace kernels
+
+} // namespace mfem
+
+#endif // MFEM_TMOP_PA_HPP

fem/tmop_pa_h2d.cpp

@@ -0,0 +1,161 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+/* // Original i-j assembly (old invariants code).
+   for (int e = 0; e < NE; e++)
+   {
+      for (int q = 0; q < nqp; q++)
+      {
+         el.CalcDShape(ip, DSh);
+         Mult(DSh, Jrt, DS);
+         for (int i = 0; i < dof; i++)
+         {
+            for (int j = 0; j < dof; j++)
+            {
+               for (int r = 0; r < dim; r++)
+               {
+                  for (int c = 0; c < dim; c++)
+                  {
+                     for (int rr = 0; rr < dim; rr++)
+                     {
+                        for (int cc = 0; cc < dim; cc++)
+                        {
+                           const double H = h(r, c, rr, cc);
+                           A(e, i + r*dof, j + rr*dof) +=
+                                 weight_q * DS(i, c) * DS(j, cc) * H;
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   }*/
+
+MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_2D,
+                           const int NE,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const DenseTensor &j,
+                           const Vector &h,
+                           Vector &diagonal,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto G = Reshape(g.Read(), Q1D, D1D);
+   const auto J = Reshape(j.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto H = Reshape(h.Read(), DIM, DIM, DIM, DIM, Q1D, Q1D, NE);
+
+   auto D = Reshape(diagonal.ReadWrite(), D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
+   {
+      constexpr int DIM = 2;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      MFEM_SHARED double qd[DIM*DIM*MQ1*MD1];
+      DeviceTensor<4,double> QD(qd, DIM, DIM, MQ1, MD1);
+
+      for (int v = 0; v < DIM; v++)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QD(0,0,qx,dy) = 0.0;
+               QD(0,1,qx,dy) = 0.0;
+               QD(1,0,qx,dy) = 0.0;
+               QD(1,1,qx,dy) = 0.0;
+
+               MFEM_UNROLL(MQ1);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double *Jtr = &J(0,0,qx,qy,e);
+
+                  // Jrt = Jtr^{-1}
+                  double j[4];
+                  ConstDeviceMatrix Jrt(j,2,2);
+                  kernels::CalcInverse<2>(Jtr, j);
+
+                  const double gg = G(qy,dy) * G(qy,dy);
+                  const double gb = G(qy,dy) * B(qy,dy);
+                  const double bb = B(qy,dy) * B(qy,dy);
+                  const double bgb[4] = { bb, gb, gb, gg };
+                  ConstDeviceMatrix BG(bgb,2,2);
+
+                  for (int i = 0; i < DIM; i++)
+                  {
+                     for (int j = 0; j < DIM; j++)
+                     {
+                        const double Jij = Jrt(i,i) * Jrt(j,j);
+                        const double alpha = Jij * BG(i,j);
+                        QD(i,j,qx,dy) += alpha * H(v,i,v,j,qx,qy,e);
+                     }
+                  }
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double d = 0.0;
+               MFEM_UNROLL(MQ1);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double gg = G(qx,dx) * G(qx,dx);
+                  const double gb = G(qx,dx) * B(qx,dx);
+                  const double bb = B(qx,dx) * B(qx,dx);
+                  d += gg * QD(0,0,qx,dy);
+                  d += gb * QD(0,1,qx,dy);
+                  d += gb * QD(1,0,qx,dy);
+                  d += bb * QD(1,1,qx,dy);
+               }
+               D(dx,dy,v,e) += d;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+void TMOP_Integrator::AssembleDiagonalPA_2D(Vector &D) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   const Vector &H = PA.H;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AssembleDiagonalPA_Kernel_2D,id,N,B,G,J,H,D);
+}
+
+} // namespace mfem

fem/tmop_pa_h2d_c0.cpp

@@ -0,0 +1,96 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_C0_2D,
+                           const int NE,
+                           const Array<double> &b,
+                           const Vector &h0,
+                           Vector &diagonal,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto H0 = Reshape(h0.Read(), DIM, DIM, Q1D, Q1D, NE);
+
+   auto D = Reshape(diagonal.ReadWrite(), D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
+   {
+      constexpr int DIM = 2;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      MFEM_SHARED double qd[MQ1*MD1];
+      DeviceTensor<2,double> QD(qd, MQ1, MD1);
+
+      for (int v = 0; v < DIM; v++)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QD(qx,dy) = 0.0;
+               MFEM_UNROLL(MQ1);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double bb = B(qy,dy) * B(qy,dy);
+                  QD(qx,dy) += bb * H0(v,v,qx,qy,e);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double d = 0.0;
+               MFEM_UNROLL(MQ1);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double bb = B(qx,dx) * B(qx,dx);
+                  d += bb * QD(qx,dy);
+               }
+               D(dx,dy,v,e) += d;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+void TMOP_Integrator::AssembleDiagonalPA_C0_2D(Vector &D) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const Array<double> &B = PA.maps->B;
+   const Vector &H0 = PA.H0;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AssembleDiagonalPA_Kernel_C0_2D,id,N,B,H0,D);
+}
+
+} // namespace mfem

fem/tmop_pa_h2m.cpp

@@ -0,0 +1,128 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_2D,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const DenseTensor &j_,
+                           const Vector &h_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
+   const auto H = Reshape(h_.Read(), DIM, DIM, DIM, DIM, Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const double *Jtr = &J(0,0,qx,qy,e);
+
+            // Jrt = Jtr^{-1}
+            double Jrt[4];
+            kernels::CalcInverse<2>(Jtr, Jrt);
+
+            // Jpr = X^T.DSh
+            double Jpr[4];
+            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jpr);
+
+            // Jpt = Jpr . Jrt
+            double Jpt[4];
+            kernels::Mult(2,2,2, Jpr, Jrt, Jpt);
+
+            // B = Jpt : H
+            double B[4];
+            DeviceMatrix M(B,2,2);
+            ConstDeviceMatrix J(Jpt,2,2);
+            for (int i = 0; i < DIM; i++)
+            {
+               for (int j = 0; j < DIM; j++)
+               {
+                  M(i,j) = 0.0;
+                  for (int r = 0; r < DIM; r++)
+                  {
+                     for (int c = 0; c < DIM; c++)
+                     {
+                        M(i,j) += H(r,c,i,j,qx,qy,e) * J(r,c);
+                     }
+                  }
+               }
+            }
+            // C = Jrt . B
+            double C[4];
+            kernels::MultABt(2,2,2, Jrt, B, C);
+
+            // Overwrite QQ = Jrt . (Jpt : H)^t
+            kernels::PushGrad<MQ1,NBZ>(qx,qy, C, QQ);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::LoadBGt<MD1,MQ1>(D1D,Q1D,b,g,BG);
+      kernels::GradYt<MD1,MQ1,NBZ>(D1D,Q1D,BG,QQ,DQ);
+      kernels::GradXt<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,Y,e);
+   });
+}
+
+void TMOP_Integrator::AddMultGradPA_2D(const Vector &R, Vector &C) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   const Vector &H = PA.H;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultGradPA_Kernel_2D,id,N,B,G,J,H,R,C);
+}
+
+} // namespace mfem

fem/tmop_pa_h2m_c0.cpp

@@ -0,0 +1,107 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_2D,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Vector &h0_,
+                           const Vector &r_,
+                           Vector &c_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto H0 = Reshape(h0_.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto R = Reshape(r_.Read(), D1D, D1D, DIM, NE);
+
+   auto Y = Reshape(c_.ReadWrite(), D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      constexpr int DIM = 2;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+
+      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[2][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[2][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,R,XY);
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+
+      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY,DQ);
+      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Xh = X^T . Sh
+            double Xh[2];
+            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,Xh);
+
+            double B[4];
+            DeviceMatrix H(B,2,2);
+            for (int i = 0; i < DIM; i++)
+            {
+               for (int j = 0; j < DIM; j++)
+               {
+                  H(i,j) = H0(i,j,qx,qy,e);
+               }
+            }
+
+            // p2 = B . Xh
+            double p2[2];
+            kernels::Mult(2,2,B,Xh,p2);
+            kernels::PushEval<MQ1,NBZ>(qx,qy,p2,QQ);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::LoadBt<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::EvalXt<MD1,MQ1,NBZ>(D1D,Q1D,B,QQ,DQ);
+      kernels::EvalYt<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ,Y,e);
+   });
+}
+
+void TMOP_Integrator::AddMultGradPA_C0_2D(const Vector &X, const Vector &R,
+                                          Vector &C) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const Array<double> &B = PA.maps->B;
+   const Vector &H0 = PA.H0;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultGradPA_Kernel_C0_2D,id,N,B,H0,R,C);
+}
+
+} // namespace mfem

fem/tmop_pa_h2s.cpp

@@ -0,0 +1,247 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+#include "../linalg/dinvariants.hpp"
+
+namespace mfem
+{
+
+using Args = kernels::InvariantsEvaluator2D::Buffers;
+
+// weight * ddI1
+static MFEM_HOST_DEVICE inline
+void EvalH_001(const int e, const int qx, const int qy,
+               const double weight, const double *Jpt,
+               DeviceTensor<7,double> H)
+{
+   constexpr int DIM = 2;
+   double ddI1[4];
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).ddI1(ddI1));
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1(ie.Get_ddI1(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               const double h = ddi1(r,c);
+               H(r,c,i,j,qx,qy,e) = weight * h;
+            }
+         }
+      }
+   }
+}
+
+// 0.5 * weight * dI1b
+static MFEM_HOST_DEVICE inline
+void EvalH_002(const int e, const int qx, const int qy,
+               const double weight, const double *Jpt,
+               DeviceTensor<7,double> H)
+{
+   constexpr int DIM = 2;
+   double ddI1[4], ddI1b[4], dI2b[4];
+   kernels::InvariantsEvaluator2D ie(Args()
+                                     .J(Jpt)
+                                     .ddI1(ddI1)
+                                     .ddI1b(ddI1b)
+                                     .dI2b(dI2b));
+   const double w = 0.5 * weight;
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               const double h = ddi1b(r,c);
+               H(r,c,i,j,qx,qy,e) = w * h;
+            }
+         }
+      }
+   }
+}
+
+static MFEM_HOST_DEVICE inline
+void EvalH_007(const int e, const int qx, const int qy,
+               const double weight, const double *Jpt,
+               DeviceTensor<7,double> H)
+{
+   constexpr int DIM = 2;
+   double ddI1[4], ddI2[4], dI1[4], dI2[4], dI2b[4];
+   kernels::InvariantsEvaluator2D ie(Args()
+                                     .J(Jpt)
+                                     .ddI1(ddI1)
+                                     .ddI2(ddI2)
+                                     .dI1(dI1)
+                                     .dI2(dI2)
+                                     .dI2b(dI2b));
+   const double c1 = 1./ie.Get_I2();
+   const double c2 = weight*c1*c1;
+   const double c3 = ie.Get_I1()*c2;
+   ConstDeviceMatrix di1(ie.Get_dI1(),DIM,DIM);
+   ConstDeviceMatrix di2(ie.Get_dI2(),DIM,DIM);
+
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1(ie.Get_ddI1(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi2(ie.Get_ddI2(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               H(r,c,i,j,qx,qy,e) =
+                  weight * (1.0 + c1) * ddi1(r,c)
+                  - c3 * ddi2(r,c)
+                  - c2 * ( di1(i,j) * di2(r,c) + di2(i,j) * di1(r,c) )
+                  + 2.0 * c1 * c3 * di2(r,c) * di2(i,j);
+            }
+         }
+      }
+   }
+}
+
+static MFEM_HOST_DEVICE inline
+void EvalH_077(const int e, const int qx, const int qy,
+               const double weight, const double *Jpt,
+               DeviceTensor<7,double> H)
+{
+   constexpr int DIM = 2;
+   double dI2[4], dI2b[4], ddI2[4];
+   kernels::InvariantsEvaluator2D ie(Args()
+                                     .J(Jpt)
+                                     .dI2(dI2)
+                                     .dI2b(dI2b)
+                                     .ddI2(ddI2));
+   const double I2 = ie.Get_I2(), I2inv_sq = 1.0 / (I2 * I2);
+   ConstDeviceMatrix di2(ie.Get_dI2(),DIM,DIM);
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi2(ie.Get_ddI2(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               H(r,c,i,j,qx,qy,e) =
+                  weight * 0.5 * (1.0 - I2inv_sq) * ddi2(r,c)
+                  + weight * (I2inv_sq / I2) * di2(r,c) * di2(i,j);
+            }
+         }
+      }
+   }
+}
+
+MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
+                           const Vector &x_,
+                           const double metric_normal,
+                           const int mid,
+                           const int NE,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const DenseTensor &j_,
+                           Vector &h_,
+                           const int d1d,
+                           const int q1d)
+{
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+               "Metric not yet implemented!");
+
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto W = Reshape(w_.Read(), Q1D, Q1D);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
+   auto H = Reshape(h_.Write(), DIM, DIM, DIM, DIM, Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double s_BG[2][MQ1*MD1];
+      MFEM_SHARED double s_X[2][NBZ][MD1*MD1];
+      MFEM_SHARED double s_DQ[4][NBZ][MD1*MQ1];
+      MFEM_SHARED double s_QQ[4][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,X,s_X);
+      kernels::LoadBG<MD1,MQ1>(D1D, Q1D, b, g, s_BG);
+
+      kernels::GradX<MD1,MQ1,NBZ>(D1D, Q1D, s_BG, s_X, s_DQ);
+      kernels::GradY<MD1,MQ1,NBZ>(D1D, Q1D, s_BG, s_DQ, s_QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const double *Jtr = &J(0,0,qx,qy,e);
+            const double detJtr = kernels::Det<2>(Jtr);
+            const double weight = metric_normal * W(qx,qy) * detJtr;
+
+            // Jrt = Jtr^{-1}
+            double Jrt[4];
+            kernels::CalcInverse<2>(Jtr, Jrt);
+
+            // Jpr = X^t.DSh
+            double Jpr[4];
+            kernels::PullGrad<MQ1,NBZ>(qx,qy,s_QQ,Jpr);
+
+            // Jpt = Jpr.Jrt
+            double Jpt[4];
+            kernels::Mult(2,2,2, Jpr, Jrt, Jpt);
+
+            // metric->AssembleH
+            if (mid ==  1) { EvalH_001(e,qx,qy,weight,Jpt,H); }
+            if (mid ==  2) { EvalH_002(e,qx,qy,weight,Jpt,H); }
+            if (mid ==  7) { EvalH_007(e,qx,qy,weight,Jpt,H); }
+            if (mid == 77) { EvalH_077(e,qx,qy,weight,Jpt,H); }
+         } // qx
+      } // qy
+   });
+}
+
+void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
+{
+   const int N = PA.ne;
+   const int M = metric->Id();
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double mn = metric_normal;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   Vector &H = PA.H;
+
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,M,N,W,B,G,J,H);
+}
+
+} // namespace mfem

fem/tmop_pa_h2s_c0.cpp

@@ -0,0 +1,125 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_C0_2D,
+                           const double lim_normal,
+                           const Vector &lim_dist,
+                           const Vector &c0_,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &bld_,
+                           Vector &h0_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const bool const_c0 = c0_.Size() == 1;
+   const auto C0 = const_c0 ?
+                   Reshape(c0_.Read(), 1, 1, 1) :
+                   Reshape(c0_.Read(), Q1D, Q1D, NE);
+   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, NE);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
+
+   auto H0 = Reshape(h0_.Write(), DIM, DIM, Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      MFEM_SHARED double BLD[MQ1*MD1];
+
+      MFEM_SHARED double XY[NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,LD,XY);
+
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
+
+      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,BLD,XY,DQ);
+      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,BLD,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const double *Jtr = &J(0,0,qx,qy,e);
+            const double detJtr = kernels::Det<2>(Jtr);
+            const double weight = W(qx,qy) * detJtr;
+            const double coeff0 = const_c0 ? C0(0,0,0) : C0(qx,qy,e);
+            const double weight_m = weight * lim_normal * coeff0;
+
+            double D;
+            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,D);
+            const double dist = D; // GetValues, default comp set to 0
+
+            // lim_func->Eval_d2(p1, p0, d_vals(q), grad_grad);
+            // d2.Diag(1.0 / (dist * dist), x.Size());
+            const double c = 1.0 / (dist * dist);
+            double grad_grad[4];
+            kernels::Diag<2>(c, grad_grad);
+            ConstDeviceMatrix gg(grad_grad,DIM,DIM);
+
+            for (int i = 0; i < DIM; i++)
+            {
+               for (int j = 0; j < DIM; j++)
+               {
+                  H0(i,j,qx,qy,e) = weight_m * gg(i,j);
+               }
+            }
+         }
+      }
+   });
+}
+
+void TMOP_Integrator::AssembleGradPA_C0_2D(const Vector &X) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double ln = lim_normal;
+   const Vector &LD = PA.LD;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W   = PA.ir->GetWeights();
+   const Array<double> &B   = PA.maps->B;
+   const Array<double> &BLD = PA.maps_lim->B;
+   const Vector &C0 = PA.C0;
+   Vector &H0 = PA.H0;
+
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_C0_2D,id,ln,LD,C0,N,J,W,B,BLD,H0);
+}
+
+} // namespace mfem

fem/tmop_pa_h3d.cpp

@@ -0,0 +1,150 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_3D,
+                           const int NE,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const DenseTensor &j,
+                           const Vector &h,
+                           Vector &diagonal,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto G = Reshape(g.Read(), Q1D, D1D);
+   const auto J = Reshape(j.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto H = Reshape(h.Read(), DIM, DIM, DIM, DIM, Q1D, Q1D, Q1D, NE);
+
+   auto D = Reshape(diagonal.ReadWrite(), D1D, D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      constexpr int DIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      MFEM_SHARED double qqd[MQ1*MQ1*MD1];
+      MFEM_SHARED double qdd[MQ1*MD1*MD1];
+      DeviceTensor<3,double> QQD(qqd, MQ1, MQ1, MD1);
+      DeviceTensor<3,double> QDD(qdd, MQ1, MD1, MD1);
+
+      for (int v = 0; v < DIM; ++v)
+      {
+         for (int i = 0; i < DIM; i++)
+         {
+            for (int j = 0; j < DIM; j++)
+            {
+               // first tensor contraction, along z direction
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qy,y,Q1D)
+                  {
+                     MFEM_FOREACH_THREAD(dz,z,D1D)
+                     {
+                        QQD(qx,qy,dz) = 0.0;
+                        MFEM_UNROLL(MQ1);
+                        for (int qz = 0; qz < Q1D; ++qz)
+                        {
+                           const double *Jtr = &J(0,0,qx,qy,qz,e);
+                           double jrt[9];
+                           ConstDeviceMatrix Jrt(jrt,3,3);
+                           kernels::CalcInverse<3>(Jtr, jrt);
+                           const double Bz = B(qz,dz);
+                           const double Gz = G(qz,dz);
+                           const double L = i==2 ? Gz : Bz;
+                           const double R = j==2 ? Gz : Bz;
+                           const double Jij = Jrt(i,i) * Jrt(j,j);
+                           const double h = H(v,i,v,j,qx,qy,qz,e);
+                           QQD(qx,qy,dz) += L * Jij * h * R;
+                        }
+                     }
+                  }
+               }
+               MFEM_SYNC_THREAD;
+               // second tensor contraction, along y direction
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  MFEM_FOREACH_THREAD(dz,z,D1D)
+                  {
+                     MFEM_FOREACH_THREAD(dy,y,D1D)
+                     {
+                        QDD(qx,dy,dz) = 0.0;
+                        MFEM_UNROLL(MQ1);
+                        for (int qy = 0; qy < Q1D; ++qy)
+                        {
+                           const double By = B(qy,dy);
+                           const double Gy = G(qy,dy);
+                           const double L = i==1 ? Gy : By;
+                           const double R = j==1 ? Gy : By;
+                           QDD(qx,dy,dz) += L * QQD(qx,qy,dz) * R;
+                        }
+                     }
+                  }
+               }
+               MFEM_SYNC_THREAD;
+               // third tensor contraction, along x direction
+               MFEM_FOREACH_THREAD(dz,z,D1D)
+               {
+                  MFEM_FOREACH_THREAD(dy,y,D1D)
+                  {
+                     MFEM_FOREACH_THREAD(dx,x,D1D)
+                     {
+                        double d = 0.0;
+                        MFEM_UNROLL(MQ1);
+                        for (int qx = 0; qx < Q1D; ++qx)
+                        {
+                           const double Bx = B(qx,dx);
+                           const double Gx = G(qx,dx);
+                           const double L = i==0 ? Gx : Bx;
+                           const double R = j==0 ? Gx : Bx;
+                           d += L * QDD(qx,dy,dz) * R;
+                        }
+                        D(dx,dy,dz,v,e) += d;
+                     }
+                  }
+               }
+               MFEM_SYNC_THREAD;
+            }
+         }
+      }
+   });
+}
+
+void TMOP_Integrator::AssembleDiagonalPA_3D(Vector &D) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   const Vector &H = PA.H;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AssembleDiagonalPA_Kernel_3D,id,N,B,G,J,H,D);
+}
+
+} // namespace mfem

fem/tmop_pa_h3d_c0.cpp

@@ -0,0 +1,123 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_C0_3D,
+                           const int NE,
+                           const Array<double> &b,
+                           const Vector &h0,
+                           Vector &diagonal,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto H0 = Reshape(h0.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+
+   auto D = Reshape(diagonal.ReadWrite(), D1D, D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      constexpr int DIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+
+      MFEM_SHARED double qqd[MQ1*MQ1*MD1];
+      MFEM_SHARED double qdd[MQ1*MD1*MD1];
+      DeviceTensor<3,double> QQD(qqd, MQ1, MQ1, MD1);
+      DeviceTensor<3,double> QDD(qdd, MQ1, MD1, MD1);
+
+      for (int v = 0; v < DIM; ++v)
+      {
+         // first tensor contraction, along z direction
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(dz,z,D1D)
+               {
+                  QQD(qx,qy,dz) = 0.0;
+                  MFEM_UNROLL(MQ1);
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     const double Bz = B(qz,dz);
+                     QQD(qx,qy,dz) += Bz * H0(v,v,qx,qy,qz,e) * Bz;
+                  }
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         // second tensor contraction, along y direction
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1D)
+               {
+                  QDD(qx,dy,dz) = 0.0;
+                  MFEM_UNROLL(MQ1);
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     const double By = B(qy,dy);
+                     QDD(qx,dy,dz) += By * QQD(qx,qy,dz) * By;
+                  }
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         // third tensor contraction, along x direction
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  double d = 0.0;
+                  MFEM_UNROLL(MQ1);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double Bx = B(qx,dx);
+                     d += Bx * QDD(qx,dy,dz) * Bx;
+                  }
+                  D(dx,dy,dz, v, e) += d;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+void TMOP_Integrator::AssembleDiagonalPA_C0_3D(Vector &D) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const Array<double> &B = PA.maps->B;
+   const Vector &H0 = PA.H0;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AssembleDiagonalPA_Kernel_C0_3D,id,N,B,H0,D);
+}
+
+} // namespace mfem

fem/tmop_pa_h3m.cpp

@@ -0,0 +1,132 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_3D,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const DenseTensor &j_,
+                           const Vector &h_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
+   const auto H = Reshape(h_.Read(), DIM, DIM, DIM, DIM, Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      constexpr int DIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[9][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double *Jtr = &J(0,0,qx,qy,qz,e);
+
+               // Jrt = Jtr^{-1}
+               double Jrt[9];
+               kernels::CalcInverse<3>(Jtr, Jrt);
+
+               // Jpr = X^T.DSh
+               double Jpr[9];
+               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, Jpr);
+
+               // Jpt = X^T.DS = (X^T.DSh).Jrt = Jpr.Jrt
+               double Jpt[9];
+               kernels::Mult(3,3,3, Jpr, Jrt, Jpt);
+
+               // B = Jpt : H
+               double B[9];
+               DeviceMatrix M(B,3,3);
+               ConstDeviceMatrix J(Jpt,3,3);
+               for (int i = 0; i < DIM; i++)
+               {
+                  for (int j = 0; j < DIM; j++)
+                  {
+                     M(i,j) = 0.0;
+                     for (int r = 0; r < DIM; r++)
+                     {
+                        for (int c = 0; c < DIM; c++)
+                        {
+                           M(i,j) += H(r,c,i,j,qx,qy,qz,e) * J(r,c);
+                        }
+                     }
+                  }
+               }
+
+               // Y +=  DS . M^t += DSh . (Jrt . M^t)
+               double A[9];
+               kernels::MultABt(3,3,3, Jrt, B, A);
+               kernels::PushGrad<MQ1>(qx,qy,qz, A, QQQ);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::LoadBGt<MD1,MQ1>(D1D,Q1D,b,g,BG);
+      kernels::GradZt<MD1,MQ1>(D1D,Q1D,BG,QQQ,DQQ);
+      kernels::GradYt<MD1,MQ1>(D1D,Q1D,BG,DQQ,DDQ);
+      kernels::GradXt<MD1,MQ1>(D1D,Q1D,BG,DDQ,Y,e);
+   });
+}
+
+void TMOP_Integrator::AddMultGradPA_3D(const Vector &X, const Vector &R,
+                                       Vector &C) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   const Vector &H = PA.H;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultGradPA_Kernel_3D,id,N,B,G,J,H,R,C);
+}
+
+} // namespace mfem

fem/tmop_pa_h3m_c0.cpp

@@ -0,0 +1,111 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_3D,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Vector &h0_,
+                           const Vector &r_,
+                           Vector &c_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto H0 = Reshape(h0_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto R = Reshape(r_.Read(), D1D, D1D, D1D, DIM, NE);
+
+   auto Y = Reshape(c_.ReadWrite(), D1D, D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      constexpr int DIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+
+      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[3][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[3][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[3][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,R,DDD);
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD,DDQ);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ,DQQ);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               // Xh = X^T . Sh
+               double Xh[3];
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,Xh);
+
+               double B[9];
+               DeviceMatrix H(B,3,3);
+               for (int i = 0; i < DIM; i++)
+               {
+                  for (int j = 0; j < DIM; j++)
+                  {
+                     H(i,j) = H0(i,j,qx,qy,qz,e);
+                  }
+               }
+
+               // p2 = B . Xh
+               double p2[3];
+               kernels::Mult(3,3,B,Xh,p2);
+               kernels::PushEval<MQ1>(qx,qy,qz,p2,QQQ);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::LoadBt<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::EvalXt<MD1,MQ1>(D1D,Q1D,B,QQQ,DQQ);
+      kernels::EvalYt<MD1,MQ1>(D1D,Q1D,B,DQQ,DDQ);
+      kernels::EvalZt<MD1,MQ1>(D1D,Q1D,B,DDQ,Y,e);
+   });
+}
+
+void TMOP_Integrator::AddMultGradPA_C0_3D(const Vector &X, const Vector &R,
+                                          Vector &C) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const Array<double> &B = PA.maps->B;
+   const Vector &H0 = PA.H0;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultGradPA_Kernel_C0_3D,id,N,B,H0,R,C);
+}
+
+} // namespace mfem

fem/tmop_pa_h3s.cpp

@@ -0,0 +1,281 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+#include "../linalg/dinvariants.hpp"
+
+namespace mfem
+{
+
+using Args = kernels::InvariantsEvaluator3D::Buffers;
+
+// dP_302 = (dI2b*dI1b + dI1b*dI2b)/9 + (I1b/9)*ddI2b + (I2b/9)*ddI1b
+static MFEM_HOST_DEVICE inline
+void EvalH_302(const int e, const int qx, const int qy, const int qz,
+               const double weight, const double *J, DeviceTensor<8,double> dP)
+{
+   double B[9];
+   double         dI1b[9],          ddI1b[9];
+   double dI2[9], dI2b[9], ddI2[9], ddI2b[9];
+   double dI3b[9];
+   constexpr int DIM = 3;
+   kernels::InvariantsEvaluator3D ie(Args()
+                                     .J(J).B(B)
+                                     .dI1b(dI1b).ddI1b(ddI1b)
+                                     .dI2(dI2).dI2b(dI2b).ddI2(ddI2).ddI2b(ddI2b)
+                                     .dI3b(dI3b));
+   const double c1 = weight/9.;
+   const double I1b = ie.Get_I1b();
+   const double I2b = ie.Get_I2b();
+   ConstDeviceMatrix di1b(ie.Get_dI1b(),DIM,DIM);
+   ConstDeviceMatrix di2b(ie.Get_dI2b(),DIM,DIM);
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi2b(ie.Get_ddI2b(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               const double dp =
+                  (di2b(r,c)*di1b(i,j) + di1b(r,c)*di2b(i,j))
+                  + ddi2b(r,c)*I1b
+                  + ddi1b(r,c)*I2b;
+               dP(r,c,i,j,qx,qy,qz,e) = c1 * dp;
+            }
+         }
+      }
+   }
+}
+
+// dP_303 = ddI1b/3
+static MFEM_HOST_DEVICE inline
+void EvalH_303(const int e, const int qx, const int qy, const int qz,
+               const double weight, const double *J, DeviceTensor<8,double> dP)
+{
+   double B[9];
+   double         dI1b[9], ddI1[9], ddI1b[9];
+   double dI2[9], dI2b[9], ddI2[9], ddI2b[9];
+   double dI3b[9], ddI3b[9];
+   constexpr int DIM = 3;
+   kernels::InvariantsEvaluator3D ie(Args()
+                                     .J(J).B(B)
+                                     .dI1b(dI1b).ddI1(ddI1).ddI1b(ddI1b)
+                                     .dI2(dI2).dI2b(dI2b).ddI2(ddI2).ddI2b(ddI2b)
+                                     .dI3b(dI3b).ddI3b(ddI3b));
+   const double c1 = weight/3.;
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               const double dp = ddi1b(r,c);
+               dP(r,c,i,j,qx,qy,qz,e) = c1 * dp;
+            }
+         }
+      }
+   }
+}
+
+// dP_315 = 2*(dI3b x dI3b) + 2*(I3b - 1)*ddI3b
+static MFEM_HOST_DEVICE inline
+void EvalH_315(const int e, const int qx, const int qy, const int qz,
+               const double weight, const double *J, DeviceTensor<8,double> dP)
+{
+   double dI3b[9], ddI3b[9];
+   constexpr int DIM = 3;
+   kernels::InvariantsEvaluator3D ie(Args().
+                                     J(J).
+                                     dI3b(dI3b).ddI3b(ddI3b));
+
+   double sign_detJ;
+   const double I3b = ie.Get_I3b(sign_detJ);
+   ConstDeviceMatrix di3b(ie.Get_dI3b(sign_detJ),DIM,DIM);
+
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi3b(ie.Get_ddI3b(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               const double dp = 2.0 * weight * (I3b - 1.0) * ddi3b(r,c) +
+                                 2.0 * weight * di3b(r,c) * di3b(i,j);
+               dP(r,c,i,j,qx,qy,qz,e) = dp;
+            }
+         }
+      }
+   }
+}
+
+// dP_321 = ddI1 + (-2/I3b^3)*(dI2 x dI3b + dI3b x dI2)
+//               + (1/I3)*ddI2
+//               + (6*I2/I3b^4)*(dI3b x dI3b)
+//               + (-2*I2/I3b^3)*ddI3b
+static MFEM_HOST_DEVICE inline
+void EvalH_321(const int e, const int qx, const int qy, const int qz,
+               const double weight, const double *J, DeviceTensor<8,double> dP)
+{
+   double B[9];
+   double         dI1b[9], ddI1[9], ddI1b[9];
+   double dI2[9], dI2b[9], ddI2[9], ddI2b[9];
+   double dI3b[9], ddI3b[9];
+   constexpr int DIM = 3;
+   kernels::InvariantsEvaluator3D ie(Args()
+                                     .J(J).B(B)
+                                     .dI1b(dI1b).ddI1(ddI1).ddI1b(ddI1b)
+                                     .dI2(dI2).dI2b(dI2b).ddI2(ddI2).ddI2b(ddI2b)
+                                     .dI3b(dI3b).ddI3b(ddI3b));
+   double sign_detJ;
+   const double I2 = ie.Get_I2();
+   const double I3b = ie.Get_I3b(sign_detJ);
+
+   ConstDeviceMatrix di2(ie.Get_dI2(),DIM,DIM);
+   ConstDeviceMatrix di3b(ie.Get_dI3b(sign_detJ),DIM,DIM);
+
+   const double c0 = 1.0/I3b;
+   const double c1 = weight*c0*c0;
+   const double c2 = -2*c0*c1;
+   const double c3 = c2*I2;
+
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1(ie.Get_ddI1(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi2(ie.Get_ddI2(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi3b(ie.Get_ddI3b(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               const double dp =
+                  weight * ddi1(r,c)
+                  + c1 * ddi2(r,c)
+                  + c3 * ddi3b(r,c)
+                  + c2 * ((di2(r,c)*di3b(i,j) + di3b(r,c)*di2(i,j)))
+                  -3*c0*c3 * di3b(r,c)*di3b(i,j);
+               dP(r,c,i,j,qx,qy,qz,e) = dp;
+            }
+         }
+      }
+   }
+}
+
+MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
+                           const double metric_normal,
+                           const int mid,
+                           const Vector &x_,
+                           const int NE,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const DenseTensor &j_,
+                           Vector &h_,
+                           const int d1d,
+                           const int q1d)
+{
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
+               "3D metric not yet implemented!");
+
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
+   auto H = Reshape(h_.Write(), DIM, DIM, DIM, DIM, Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double s_BG[2][MQ1*MD1];
+      MFEM_SHARED double s_DDD[3][MD1*MD1*MD1];
+      MFEM_SHARED double s_DDQ[9][MD1*MD1*MQ1];
+      MFEM_SHARED double s_DQQ[9][MD1*MQ1*MQ1];
+      MFEM_SHARED double s_QQQ[9][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,X,s_DDD);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,s_BG);
+
+      kernels::GradX<MD1,MQ1>(D1D,Q1D,s_BG,s_DDD,s_DDQ);
+      kernels::GradY<MD1,MQ1>(D1D,Q1D,s_BG,s_DDQ,s_DQQ);
+      kernels::GradZ<MD1,MQ1>(D1D,Q1D,s_BG,s_DQQ,s_QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double *Jtr = &J(0,0,qx,qy,qz,e);
+               const double detJtr = kernels::Det<3>(Jtr);
+               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+
+               // Jrt = Jtr^{-1}
+               double Jrt[9];
+               kernels::CalcInverse<3>(Jtr, Jrt);
+
+               // Jpr = X^T.DSh
+               double Jpr[9];
+               kernels::PullGrad<MQ1>(qx,qy,qz, s_QQQ, Jpr);
+
+               // Jpt = X^T . DS = (X^T.DSh) . Jrt = Jpr . Jrt
+               double Jpt[9];
+               kernels::Mult(3,3,3, Jpr, Jrt, Jpt);
+
+               // metric->AssembleH
+               if (mid == 302) { EvalH_302(e,qx,qy,qz,weight,Jpt,H); }
+               if (mid == 303) { EvalH_303(e,qx,qy,qz,weight,Jpt,H); }
+               if (mid == 315) { EvalH_315(e,qx,qy,qz,weight,Jpt,H); }
+               if (mid == 321) { EvalH_321(e,qx,qy,qz,weight,Jpt,H); }
+            } // qx
+         } // qy
+      } // qz
+   });
+}
+
+void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int M = metric->Id();
+   const int id = (D1D << 4 ) | Q1D;
+   const double mn = metric_normal;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   Vector &H = PA.H;
+
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,M,X,N,W,B,G,J,H);
+}
+
+} // namespace mfem

fem/tmop_pa_h3s_c0.cpp

@@ -0,0 +1,129 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_Kernel_C0_3D,
+                           const double lim_normal,
+                           const Vector &lim_dist,
+                           const Vector &c0_,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &bld_,
+                           Vector &h0_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const bool const_c0 = c0_.Size() == 1;
+   const auto C0 = const_c0 ?
+                   Reshape(c0_.Read(), 1, 1, 1, 1) :
+                   Reshape(c0_.Read(), Q1D, Q1D, Q1D, NE);
+   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, D1D, NE);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
+
+   auto H0 = Reshape(h0_.Write(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      constexpr int DIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      MFEM_SHARED double BLD[MQ1*MD1];
+
+      MFEM_SHARED double DDD[MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,LD,DDD);
+
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
+
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,BLD,DDD,DDQ);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,BLD,DDQ,DQQ);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,BLD,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double *Jtr = &J(0,0,qx,qy,qz,e);
+               const double detJtr = kernels::Det<3>(Jtr);
+               const double weight = W(qx,qy,qz) * detJtr;
+               const double coeff0 = const_c0 ? C0(0,0,0,0) : C0(qx,qy,qz,e);
+               const double weight_m = weight * lim_normal * coeff0;
+
+               double D;
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,D);
+               const double dist = D; // GetValues, default comp set to 0
+
+               // lim_func->Eval_d2(p1, p0, d_vals(q), grad_grad);
+               // d2.Diag(1.0 / (dist * dist), x.Size());
+               const double c = 1.0 / (dist * dist);
+               double grad_grad[9];
+               kernels::Diag<3>(c, grad_grad);
+               ConstDeviceMatrix gg(grad_grad,DIM,DIM);
+
+               for (int i = 0; i < DIM; i++)
+               {
+                  for (int j = 0; j < DIM; j++)
+                  {
+                     H0(i,j,qx,qy,qz,e) = weight_m * gg(i,j);
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
+void TMOP_Integrator::AssembleGradPA_C0_3D(const Vector &X) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double ln = lim_normal;
+   const Vector &LD = PA.LD;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &BLD = PA.maps_lim->B;
+   const Vector &C0 = PA.C0;
+   Vector &H0 = PA.H0;
+
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_Kernel_C0_3D,id,ln,LD,C0,N,J,W,B,BLD,H0);
+}
+
+} // namespace mfem

fem/tmop_pa_jp2.cpp

@@ -0,0 +1,178 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "tmop_tools.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(int, CheckDetJpr_Kernel_2D,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Vector &x_,
+                           Vector &DetJOk,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
+
+   auto E = Reshape(DetJOk.Write(), Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double J[4];
+            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,J);
+            const double detJ = kernels::Det<2>(J);
+            E(qx,qy,e) = (detJ <= 0.0) ? 0.0 : 1.0;
+         }
+      }
+   });
+   const double N = DetJOk.Size();
+   const double D = DetJOk * DetJOk;
+   return D < N ? 0 : 1;
+}
+
+int TMOPNewtonSolver::CheckDetJpr_2D(const FiniteElementSpace *fes,
+                                     const Vector &X) const
+{
+   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *R = fes->GetElementRestriction(ordering);
+   Vector XE(R->Height(), Device::GetDeviceMemoryType());
+   XE.UseDevice(true);
+   R->Mult(X, XE);
+
+   const DofToQuad &maps = fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
+   const int NE = fes->GetMesh()->GetNE();
+   const int NQ = ir.GetNPoints();
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const Array<double> &B = maps.B;
+   const Array<double> &G = maps.G;
+
+   Vector E(NE*NQ);
+   E.UseDevice(true);
+
+   MFEM_LAUNCH_TMOP_KERNEL(CheckDetJpr_Kernel_2D,id,NE,B,G,XE,E);
+}
+
+MFEM_REGISTER_TMOP_KERNELS(double, MinDetJpr_Kernel_2D,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Vector &x_,
+                           Vector &DetJ,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
+
+   auto D = Reshape(DetJ.Write(), Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double Jpr[4];
+            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jpr);
+            D(qx,qy,e) = kernels::Det<2>(Jpr);
+         }
+      }
+   });
+   return DetJ.Min();
+}
+
+double TMOPNewtonSolver::MinDetJpr_2D(const FiniteElementSpace *fes,
+                                      const Vector &X) const
+{
+   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *R = fes->GetElementRestriction(ordering);
+   Vector XE(R->Height(), Device::GetDeviceMemoryType());
+   XE.UseDevice(true);
+   R->Mult(X, XE);
+
+   const DofToQuad &maps = fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
+   const int NE = fes->GetMesh()->GetNE();
+   const int NQ = ir.GetNPoints();
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const Array<double> &B = maps.B;
+   const Array<double> &G = maps.G;
+
+   Vector E(NE*NQ);
+   E.UseDevice(true);
+
+   MFEM_LAUNCH_TMOP_KERNEL(MinDetJpr_Kernel_2D,id,NE,B,G,XE,E);
+}
+
+} // namespace mfem

fem/tmop_pa_jp3.cpp

@@ -0,0 +1,181 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "tmop_tools.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(int, CheckDetJpr_Kernel_3D,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Vector &x_,
+                           Vector &DetJOk,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
+
+   auto E = Reshape(DetJOk.Write(), Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[6][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double J[9];
+               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, J);
+               const double detJ = kernels::Det<3>(J);
+               E(qx,qy,qz,e) = (detJ <= 0.0) ? 0.0 : 1.0;
+            }
+         }
+      }
+   });
+   const double N = DetJOk.Size();
+   const double D = DetJOk * DetJOk;
+   return D < N ? 0 : 1;
+}
+
+int TMOPNewtonSolver::CheckDetJpr_3D(const FiniteElementSpace *fes,
+                                     const Vector &X) const
+{
+   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *R = fes->GetElementRestriction(ordering);
+   Vector XE(R->Height(), Device::GetDeviceMemoryType());
+   XE.UseDevice(true);
+   R->Mult(X, XE);
+
+   const DofToQuad &maps = fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
+   const int NE = fes->GetMesh()->GetNE();
+   const int NQ = ir.GetNPoints();
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const Array<double> &B = maps.B;
+   const Array<double> &G = maps.G;
+
+   Vector E(NE*NQ);
+   E.UseDevice(true);
+
+   MFEM_LAUNCH_TMOP_KERNEL(CheckDetJpr_Kernel_3D,id,NE,B,G,XE,E);
+}
+
+MFEM_REGISTER_TMOP_KERNELS(double, MinDetJpr_Kernel_3D,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Vector &x_,
+                           Vector &DetJ,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
+
+   auto D = Reshape(DetJ.Write(), Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[6][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double Jpr[9];
+               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, Jpr);
+               D(qx,qy,qz,e) = kernels::Det<3>(Jpr);
+            }
+         }
+      }
+   });
+   return DetJ.Min();
+}
+
+double TMOPNewtonSolver::MinDetJpr_3D(const FiniteElementSpace *fes,
+                                      const Vector &X) const
+{
+   const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *R = fes->GetElementRestriction(ordering);
+   Vector XE(R->Height(), Device::GetDeviceMemoryType());
+   XE.UseDevice(true);
+   R->Mult(X, XE);
+
+   const DofToQuad &maps = fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
+   const int NE = fes->GetMesh()->GetNE();
+   const int NQ = ir.GetNPoints();
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const Array<double> &B = maps.B;
+   const Array<double> &G = maps.G;
+
+   Vector E(NE*NQ);
+   E.UseDevice(true);
+
+   MFEM_LAUNCH_TMOP_KERNEL(MinDetJpr_Kernel_3D,id,NE,B,G,XE,E);
+}
+} // namespace mfem

fem/tmop_pa_jt2_tc.cpp

@@ -0,0 +1,155 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "gridfunc.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(bool, TC_IDEAL_SHAPE_UNIT_SIZE_2D_KERNEL,
+                           const int NE,
+                           const DenseMatrix w_, // copy
+                           DenseTensor &j_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto W = Reshape(w_.Read(), DIM,DIM);
+   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            kernels::Set(DIM,DIM, 1.0, &W(0,0), &J(0,0,qx,qy,e));
+         }
+      }
+   });
+   return true;
+}
+
+MFEM_REGISTER_TMOP_KERNELS(bool, TC_IDEAL_SHAPE_GIVEN_SIZE_2D_KERNEL,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const DenseMatrix w_ideal_, // copy
+                           const Vector &x_,
+                           DenseTensor &j_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const double detW = w_ideal_.Det();
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto Wideal = Reshape(w_ideal_.Read(), DIM,DIM);
+   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
+   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double Jtr[4];
+            const double *Wid = &Wideal(0,0);
+            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jtr);
+            const double detJ = kernels::Det<2>(Jtr);
+            const double alpha = std::pow(detJ/detW,1./2);
+            kernels::Set(DIM,DIM,alpha,Wid,&J(0,0,qx,qy,e));
+         }
+      }
+   });
+   return true;
+}
+
+template<> bool
+TargetConstructor::ComputeElementTargetsPA<2>(const FiniteElementSpace *fes,
+                                              const IntegrationRule *ir,
+                                              DenseTensor &Jtr,
+                                              const Vector&) const
+{
+   MFEM_ASSERT(target_type == IDEAL_SHAPE_UNIT_SIZE || nodes != nullptr, "");
+   MFEM_VERIFY(fes->GetFE(0)->GetGeomType() == Geometry::SQUARE, "");
+   const DenseMatrix &W = Geometries.GetGeomToPerfGeomJac(Geometry::SQUARE);
+   const FiniteElement *fe = fes->GetFE(0);
+   const int NE = fes->GetMesh()->GetNE();
+   const DofToQuad::Mode mode = DofToQuad::TENSOR;
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+
+   const Array<double> &B = maps.B;
+   const Array<double> &G = maps.G;
+
+   switch (target_type)
+   {
+      case IDEAL_SHAPE_UNIT_SIZE: // Jtr(i) = Wideal;
+      {
+         MFEM_LAUNCH_TMOP_KERNEL(TC_IDEAL_SHAPE_UNIT_SIZE_2D_KERNEL,
+                                 id,NE,W,Jtr);
+      }
+      case IDEAL_SHAPE_EQUAL_SIZE: return false;
+      case IDEAL_SHAPE_GIVEN_SIZE:
+      {
+         MFEM_VERIFY(nodes, "");
+         const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+         const Operator *R = fes->GetElementRestriction(ordering);
+         Vector X(R->Height(), Device::GetDeviceMemoryType());
+         X.UseDevice(true);
+         R->Mult(*nodes, X);
+         MFEM_ASSERT(nodes->FESpace()->GetVDim() == 2, "");
+         MFEM_LAUNCH_TMOP_KERNEL(TC_IDEAL_SHAPE_GIVEN_SIZE_2D_KERNEL,
+                                 id,NE,B,G,W,X,Jtr);
+      }
+      case GIVEN_SHAPE_AND_SIZE: return false;
+      default: return false;
+   }
+   return false;
+}
+
+} // namespace mfem

fem/tmop_pa_jt3_datc.cpp

@@ -0,0 +1,162 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(bool, DatcSize,
+                           const int NE,
+                           const int ncomp,
+                           const int sizeidx,
+                           const DenseMatrix w_, // Copy
+                           const Array<double> &b_,
+                           const Vector &x_,
+                           DenseTensor &j_,
+                           const int d1d,
+                           const int q1d)
+{
+   MFEM_VERIFY(ncomp==1,"");
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= Q1D, "");
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto W = Reshape(w_.Read(), DIM,DIM);
+   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, ncomp, NE);
+   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D,Q1D, NE);
+
+   const double infinity = std::numeric_limits<double>::infinity();
+   MFEM_VERIFY(sizeidx == 0,"");
+   MFEM_VERIFY(MFEM_CUDA_BLOCKS==256,"");
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      MFEM_SHARED double DDD[MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,sizeidx,X,DDD);
+
+      double min;
+      MFEM_SHARED double min_size[MFEM_CUDA_BLOCKS];
+      DeviceTensor<3,double> M((double*)(min_size),D1D,D1D,D1D);
+      const DeviceTensor<3,const double> D((double*)(DDD+sizeidx),D1D,D1D,D1D);
+      MFEM_FOREACH_THREAD(t,x,MFEM_CUDA_BLOCKS) { min_size[t] = infinity; }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               M(dx,dy,dz) = D(dx,dy,dz);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      for (int wrk = MFEM_CUDA_BLOCKS >> 1; wrk > 0; wrk >>= 1)
+      {
+         MFEM_FOREACH_THREAD(t,x,MFEM_CUDA_BLOCKS)
+         { if (t < wrk) { min_size[t] = fmin(min_size[t], min_size[t+wrk]); } }
+         MFEM_SYNC_THREAD;
+      }
+      min = min_size[0];
+
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD,DDQ);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ,DQQ);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ,QQQ);
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               double T;
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,T);
+               const double shape_par_vals = T;
+               const double size = fmax(shape_par_vals, min);
+               const double alpha = std::pow(size, 1.0/DIM);
+               for (int i = 0; i < DIM; i++)
+               {
+                  for (int j = 0; j < DIM; j++)
+                  {
+                     J(i,j,qx,qy,qz,e) = alpha * W(i,j);
+                  }
+               }
+            }
+         }
+      }
+   });
+   return true;
+}
+
+// PA.Jtr Size = (dim, dim, PA.ne*PA.nq);
+bool DiscreteAdaptTC::ComputeElementTargetsPA(const FiniteElementSpace *pa_fes,
+                                              const IntegrationRule *ir,
+                                              DenseTensor &Jtr,
+                                              const Vector &xe) const
+{
+   MFEM_VERIFY(target_type == IDEAL_SHAPE_GIVEN_SIZE ||
+               target_type == GIVEN_SHAPE_AND_SIZE,"");
+
+   const FiniteElementSpace *fes = tspec_fesv;
+
+   if (!fes) { return false;}
+
+   const FiniteElement &fe = *fes->GetFE(0);
+   const DenseMatrix &W = Geometries.GetGeomToPerfGeomJac(fe.GetGeomType());
+   const int DIM = W.Height();
+   const int NE = fes->GetMesh()->GetNE();
+   const DofToQuad::Mode mode = DofToQuad::TENSOR;
+   const DofToQuad &maps = fe.GetDofToQuad(*ir, mode);
+   const Array<double> &B = maps.B;
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+
+   const bool SizeKernel = sizeidx != -1;
+
+   // Until it is not implemented, return on host
+   if (skewidx != -1) { return false; }
+   if (aspectratioidx != -1) { return false; }
+   if (orientationidx != -1) { return false; }
+
+   if (DIM == 3 && SizeKernel)
+   {
+      Vector tspec_e;
+      const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+      const Operator *R = fes->GetElementRestriction(ordering);
+      MFEM_VERIFY(R,"");
+      MFEM_VERIFY(R->Height() == NE*ncomp*D1D*D1D*D1D,"");
+      tspec_e.SetSize(R->Height(), Device::GetDeviceMemoryType());
+      tspec_e.UseDevice(true);
+      tspec.UseDevice(true);
+      R->Mult(tspec, tspec_e);
+      const int id = (D1D << 4 ) | Q1D;
+      MFEM_LAUNCH_TMOP_KERNEL(DatcSize,id,NE,ncomp,sizeidx,W,B,tspec_e,Jtr);
+   }
+   return false;
+}
+
+} // namespace mfem

fem/tmop_pa_jt3_tc.cpp

@@ -0,0 +1,159 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "gridfunc.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(bool, TC_IDEAL_SHAPE_UNIT_SIZE_3D_KERNEL,
+                           const int NE,
+                           const DenseMatrix w_, // copy
+                           DenseTensor &j_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto W = Reshape(w_.Read(), DIM,DIM);
+   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D,Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               kernels::Set(DIM,DIM, 1.0, &W(0,0), &J(0,0,qx,qy,qz,e));
+            }
+         }
+      }
+   });
+   return true;
+}
+
+MFEM_REGISTER_TMOP_KERNELS(bool, TC_IDEAL_SHAPE_GIVEN_SIZE_3D_KERNEL,
+                           const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const DenseMatrix w_ideal_, // copy
+                           const Vector &x_,
+                           DenseTensor &j_,
+                           const int d1d,
+                           const int q1d)
+{
+   constexpr int DIM = 3;
+
+   const double detW = w_ideal_.Det();
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto Wideal = Reshape(w_ideal_.Read(), DIM,DIM);
+   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
+   auto J = Reshape(j_.Write(), DIM,DIM, Q1D,Q1D,Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[6][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double Jtr[9];
+               const double *Wid = &Wideal(0,0);
+               kernels::PullGrad<MQ1>(qx,qy,qz,QQQ,Jtr);
+               const double detJ = kernels::Det<3>(Jtr);
+               const double alpha = std::pow(detJ/detW,1./3);
+               kernels::Set(DIM,DIM,alpha,Wid,&J(0,0,qx,qy,qz,e));
+            }
+         }
+      }
+   });
+   return true;
+}
+
+template<> bool
+TargetConstructor::ComputeElementTargetsPA<3>(const FiniteElementSpace *fes,
+                                              const IntegrationRule *ir,
+                                              DenseTensor &Jtr,
+                                              const Vector&) const
+{
+   MFEM_ASSERT(target_type == IDEAL_SHAPE_UNIT_SIZE || nodes != nullptr, "");
+   MFEM_VERIFY(fes->GetFE(0)->GetGeomType() == Geometry::CUBE, "");
+   const DenseMatrix &W = Geometries.GetGeomToPerfGeomJac(Geometry::CUBE);
+   const FiniteElement *fe = fes->GetFE(0);
+   const int NE = fes->GetMesh()->GetNE();
+   const DofToQuad::Mode mode = DofToQuad::TENSOR;
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
+   const Array<double> &B = maps.B;
+   const Array<double> &G = maps.G;
+   const int D1D = maps.ndof;
+   const int Q1D = maps.nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+
+   switch (target_type)
+   {
+      case IDEAL_SHAPE_UNIT_SIZE: // Jtr(i) = Wideal;
+      {
+         MFEM_LAUNCH_TMOP_KERNEL(TC_IDEAL_SHAPE_UNIT_SIZE_3D_KERNEL,
+                                 id,NE,W,Jtr);
+      }
+      case IDEAL_SHAPE_EQUAL_SIZE: return false;
+      case IDEAL_SHAPE_GIVEN_SIZE:
+      {
+         MFEM_VERIFY(nodes, "");
+         const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+         const Operator *R = fes->GetElementRestriction(ordering);
+         Vector X(R->Height(), Device::GetDeviceMemoryType());
+         X.UseDevice(true);
+         R->Mult(*nodes, X);
+         MFEM_ASSERT(nodes->FESpace()->GetVDim() == 3, "");
+         MFEM_LAUNCH_TMOP_KERNEL(TC_IDEAL_SHAPE_GIVEN_SIZE_3D_KERNEL,
+                                 id,NE,B,G,W,X,Jtr);
+      }
+      case GIVEN_SHAPE_AND_SIZE: return false;
+      default: return false;
+   }
+   return false;
+}
+
+} // namespace mfem

fem/tmop_pa_p2.cpp

@@ -0,0 +1,166 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+#include "../linalg/dinvariants.hpp"
+
+namespace mfem
+{
+
+using Args = kernels::InvariantsEvaluator2D::Buffers;
+
+static MFEM_HOST_DEVICE inline
+void EvalP_001(const double *Jpt, double *P)
+{
+   double dI1[4];
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).dI1(dI1));
+   kernels::Set(2,2, 1.0, ie.Get_dI1(), P);
+}
+
+static MFEM_HOST_DEVICE inline
+void EvalP_002(const double *Jpt, double *P)
+{
+   double dI1b[4], dI2b[4];
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).dI1b(dI1b).dI2b(dI2b));
+   kernels::Set(2,2, 1./2., ie.Get_dI1b(), P);
+}
+
+static MFEM_HOST_DEVICE inline
+void EvalP_007(const double *Jpt, double *P)
+{
+   double dI1[4], dI2[4], dI2b[4];
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).dI1(dI1)
+                                     .dI2(dI2).dI2b(dI2b));
+   const double I2 = ie.Get_I2();
+   kernels::Add(2,2, 1.0 + 1.0 / I2,         ie.Get_dI1(),
+                -ie.Get_I1() / (I2*I2), ie.Get_dI2(), P);
+}
+
+static MFEM_HOST_DEVICE inline
+void EvalP_077(const double *Jpt, double *P)
+{
+   double dI2[4], dI2b[4];
+   kernels::InvariantsEvaluator2D ie(Args().
+                                     J(Jpt).
+                                     dI2(dI2).dI2b(dI2b));
+   const double I2 = ie.Get_I2();
+   kernels::Set(2,2, 0.5 * (1.0 - 1.0 / (I2 * I2)), ie.Get_dI2(), P);
+}
+
+MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
+                           const double metric_normal,
+                           const int mid,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d,
+                           const int q1d)
+{
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+               "Metric not yet implemented!");
+
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const double *Jtr = &J(0,0,qx,qy,e);
+            const double detJtr = kernels::Det<2>(Jtr);
+            const double weight = metric_normal * W(qx,qy) * detJtr;
+
+            // Jrt = Jtr^{-1}
+            double Jrt[4];
+            kernels::CalcInverse<2>(Jtr, Jrt);
+
+            // Jpr = X{^T}.DSh
+            double Jpr[4];
+            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jpr);
+
+            // Jpt = X{^T}.DS = (X{^T}.DSh).Jrt = Jpr.Jrt
+            double Jpt[4];
+            kernels::Mult(2,2,2, Jpr, Jrt, Jpt);
+
+            // metric->EvalP(Jpt, P);
+            double P[4];
+            if (mid ==  1) { EvalP_001(Jpt, P); }
+            if (mid ==  2) { EvalP_002(Jpt, P); }
+            if (mid ==  7) { EvalP_007(Jpt, P); }
+            if (mid == 77) { EvalP_077(Jpt, P); }
+            for (int i = 0; i < 4; i++) { P[i] *= weight; }
+
+            // PMatO += DS . P^t += DSh . (Jrt . P^t)
+            double A[4];
+            kernels::MultABt(2,2,2, Jrt, P, A);
+            kernels::PushGrad<MQ1,NBZ>(qx,qy,A,QQ);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::LoadBGt<MD1,MQ1>(D1D,Q1D,b,g,BG);
+      kernels::GradYt<MD1,MQ1,NBZ>(D1D,Q1D,BG,QQ,DQ);
+      kernels::GradXt<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,Y,e);
+   });
+}
+
+void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
+{
+   const int N = PA.ne;
+   const int M = metric->Id();
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   const double mn = metric_normal;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,M,N,J,W,B,G,X,Y);
+}
+
+} // namespace mfem

fem/tmop_pa_p2_c0.cpp

@@ -0,0 +1,148 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_2D,
+                           const double lim_normal,
+                           const Vector &lim_dist,
+                           const Vector &c0_,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &bld_,
+                           const Vector &x0_,
+                           const Vector &x1_,
+                           Vector &y_,
+                           const int d1d,
+                           const int q1d)
+{
+   const bool const_c0 = c0_.Size() == 1;
+
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto C0 = const_c0 ?
+                   Reshape(c0_.Read(), 1, 1, 1) :
+                   Reshape(c0_.Read(), Q1D, Q1D, NE);
+   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, NE);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D);
+   const auto X0 = Reshape(x0_.Read(), D1D, D1D, DIM, NE);
+   const auto X1 = Reshape(x1_.Read(), D1D, D1D, DIM, NE);
+
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      MFEM_SHARED double BLD[MQ1*MD1];
+
+      MFEM_SHARED double XY[NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[NBZ][MQ1*MQ1];
+
+      MFEM_SHARED double XY0[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ0[2][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ0[2][NBZ][MQ1*MQ1];
+
+      MFEM_SHARED double XY1[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ1[2][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ1[2][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,LD,XY);
+      kernels::LoadX<MD1,NBZ>(e,D1D,X0,XY0);
+      kernels::LoadX<MD1,NBZ>(e,D1D,X1,XY1);
+
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
+
+      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,BLD,XY,DQ);
+      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,BLD,DQ,QQ);
+
+      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY0,DQ0);
+      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ0,QQ0);
+
+      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY1,DQ1);
+      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ1,QQ1);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const double *Jtr = &J(0,0,qx,qy,e);
+            const double detJtr = kernels::Det<2>(Jtr);
+            const double weight = W(qx,qy) * detJtr;
+
+            double ld, p0[2], p1[2];
+            const double coeff0 = const_c0 ? C0(0,0,0) : C0(qx,qy,e);
+            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,ld);
+            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ0,p0);
+            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ1,p1);
+
+            const double dist = ld; // GetValues, default comp set to 0
+
+            double d1[2];
+            // Eval_d1
+            // subtract(1.0 / (dist * dist), x, x0, d1);
+            // z = a * (x - y)
+            // grad = a * (x - x0)
+            const double a = 1.0 / (dist * dist);
+            const double w = weight * lim_normal * coeff0;
+            kernels::Subtract<2>(w*a, p1, p0, d1);
+            kernels::PushEval<MQ1,NBZ>(qx,qy,d1,QQ0);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::LoadBt<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::EvalXt<MD1,MQ1,NBZ>(D1D,Q1D,B,QQ0,DQ0);
+      kernels::EvalYt<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ0,Y,e);
+   });
+}
+
+void TMOP_Integrator::AddMultPA_C0_2D(const Vector &X, Vector &Y) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double ln = lim_normal;
+   const Vector &LD = PA.LD;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W   = PA.ir->GetWeights();
+   const Array<double> &B   = PA.maps->B;
+   const Array<double> &BLD = PA.maps_lim->B;
+   const Vector &X0 = PA.X0;
+   const Vector &C0 = PA.C0;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_C0_2D,id,ln,LD,C0,N,J,W,B,BLD,X0,X,Y);
+}
+
+} // namespace mfem

fem/tmop_pa_p3.cpp

@@ -0,0 +1,186 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+#include "../linalg/dinvariants.hpp"
+
+namespace mfem
+{
+
+using Args = kernels::InvariantsEvaluator3D::Buffers;
+
+// P_302 = (I1b/9)*dI2b + (I2b/9)*dI1b
+static MFEM_HOST_DEVICE inline
+void EvalP_302(const double *J, double *P)
+{
+   double B[9];
+   double dI1b[9], dI2[9], dI2b[9], dI3b[9];
+   kernels::InvariantsEvaluator3D ie(Args()
+                                     .J(J).B(B)
+                                     .dI1b(dI1b)
+                                     .dI2(dI2).dI2b(dI2b)
+                                     .dI3b(dI3b));
+   const double alpha = ie.Get_I1b()/9.;
+   const double beta = ie.Get_I2b()/9.;
+   kernels::Add(3,3, alpha, ie.Get_dI2b(), beta, ie.Get_dI1b(), P);
+}
+
+// P_303 = dI1b/3
+static MFEM_HOST_DEVICE inline
+void EvalP_303(const double *J, double *P)
+{
+   double B[9];
+   double dI1b[9], dI3b[9];
+   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B).dI1b(dI1b).dI3b(dI3b));
+   kernels::Set(3,3, 1./3., ie.Get_dI1b(), P);
+}
+
+// P_315 = 2*(I3b - 1)*dI3b
+static MFEM_HOST_DEVICE inline
+void EvalP_315(const double *J, double *P)
+{
+   double dI3b[9];
+   kernels::InvariantsEvaluator3D ie(Args().J(J).dI3b(dI3b));
+
+   double sign_detJ;
+   const double I3b = ie.Get_I3b(sign_detJ);
+   kernels::Set(3,3, 2.0 * (I3b - 1.0), ie.Get_dI3b(sign_detJ), P);
+}
+
+// P_321 = dI1 + (1/I3)*dI2 - (2*I2/I3b^3)*dI3b
+static MFEM_HOST_DEVICE inline
+void EvalP_321(const double *J, double *P)
+{
+   double B[9];
+   double dI1[9], dI2[9], dI3b[9];
+   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B)
+                                     .dI1(dI1).dI2(dI2).dI3b(dI3b));
+   double sign_detJ;
+   const double I3 = ie.Get_I3();
+   const double alpha = 1.0/I3;
+   const double beta = -2.*ie.Get_I2()/(I3*ie.Get_I3b(sign_detJ));
+   kernels::Add(3,3, alpha, ie.Get_dI2(), beta, ie.Get_dI3b(sign_detJ), P);
+   kernels::Add(3,3, ie.Get_dI1(), P);
+}
+
+MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
+                           const double metric_normal,
+                           const int mid,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d,
+                           const int q1d)
+{
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
+               "3D metric not yet implemented!");
+
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double s_BG[2][MQ1*MD1];
+      MFEM_SHARED double s_DDD[3][MD1*MD1*MD1];
+      MFEM_SHARED double s_DDQ[9][MD1*MD1*MQ1];
+      MFEM_SHARED double s_DQQ[9][MD1*MQ1*MQ1];
+      MFEM_SHARED double s_QQQ[9][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,X,s_DDD);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,s_BG);
+
+      kernels::GradX<MD1,MQ1>(D1D,Q1D,s_BG,s_DDD,s_DDQ);
+      kernels::GradY<MD1,MQ1>(D1D,Q1D,s_BG,s_DDQ,s_DQQ);
+      kernels::GradZ<MD1,MQ1>(D1D,Q1D,s_BG,s_DQQ,s_QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double *Jtr = &J(0,0,qx,qy,qz,e);
+               const double detJtr = kernels::Det<3>(Jtr);
+               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+
+               // Jrt = Jtr^{-1}
+               double Jrt[9];
+               kernels::CalcInverse<3>(Jtr, Jrt);
+
+               // Jpr = X^T.DSh
+               double Jpr[9];
+               kernels::PullGrad<MQ1>(qx,qy,qz, s_QQQ, Jpr);
+
+               // Jpt = X^T.DS = (X^T.DSh).Jrt = Jpr.Jrt
+               double Jpt[9];
+               kernels::Mult(3,3,3, Jpr, Jrt, Jpt);
+
+               // metric->EvalP(Jpt, P);
+               double P[9];
+               if (mid == 302) { EvalP_302(Jpt,P); }
+               if (mid == 303) { EvalP_303(Jpt,P); }
+               if (mid == 315) { EvalP_315(Jpt,P); }
+               if (mid == 321) { EvalP_321(Jpt,P); }
+               for (int i = 0; i < 9; i++) { P[i] *= weight; }
+
+               // Y += DS . P^t += DSh . (Jrt . P^t)
+               double A[9];
+               kernels::MultABt(3,3,3, Jrt, P, A);
+               kernels::PushGrad<MQ1>(qx,qy,qz, A, s_QQQ);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::LoadBGt<MD1,MQ1>(D1D, Q1D, b, g, s_BG);
+      kernels::GradZt<MD1,MQ1>(D1D,Q1D,s_BG,s_QQQ,s_DQQ);
+      kernels::GradYt<MD1,MQ1>(D1D,Q1D,s_BG,s_DQQ,s_DDQ);
+      kernels::GradXt<MD1,MQ1>(D1D,Q1D,s_BG,s_DDQ,Y,e);
+   });
+}
+
+void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
+{
+   const int N = PA.ne;
+   const int M = metric->Id();
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   const double mn = metric_normal;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,M,N,J,W,B,G,X,Y);
+}
+
+} // namespace mfem

fem/tmop_pa_p3_c0.cpp

@@ -0,0 +1,154 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_3D,
+                           const double lim_normal,
+                           const Vector &lim_dist,
+                           const Vector &c0_,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &bld_,
+                           const Vector &x0_,
+                           const Vector &x1_,
+                           Vector &y_,
+                           const int d1d,
+                           const int q1d)
+{
+   const bool const_c0 = c0_.Size() == 1;
+
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto C0 = const_c0 ?
+                   Reshape(c0_.Read(), 1, 1, 1, 1) :
+                   Reshape(c0_.Read(), Q1D, Q1D, Q1D, NE);
+   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, D1D, NE);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
+   const auto X0 = Reshape(x0_.Read(), D1D, D1D, D1D, DIM, NE);
+   const auto X1 = Reshape(x1_.Read(), D1D, D1D, D1D, DIM, NE);
+
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, DIM, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      MFEM_SHARED double BLD[MQ1*MD1];
+
+      MFEM_SHARED double DDD[MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
+
+      MFEM_SHARED double DDD0[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ0[3][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ0[3][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ0[3][MQ1*MQ1*MQ1];
+
+      MFEM_SHARED double DDD1[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ1[3][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ1[3][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ1[3][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,LD,DDD);
+      kernels::LoadX<MD1>(e,D1D,X0,DDD0);
+      kernels::LoadX<MD1>(e,D1D,X1,DDD1);
+
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
+
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,BLD,DDD,DDQ);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,BLD,DDQ,DQQ);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,BLD,DQQ,QQQ);
+
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD0,DDQ0);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ0,DQQ0);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ0,QQQ0);
+
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD1,DDQ1);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ1,DQQ1);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ1,QQQ1);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double *Jtr = &J(0,0,qx,qy,qz,e);
+               const double detJtr = kernels::Det<3>(Jtr);
+               const double weight = W(qx,qy,qz) * detJtr;
+
+               double D, p0[3], p1[3];
+               const double coeff0 = const_c0 ? C0(0,0,0,0) : C0(qx,qy,qz,e);
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,D);
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ0,p0);
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ1,p1);
+
+               double d1[3];
+               // Eval_d1
+               // subtract(1.0 / (dist * dist), x, x0, d1);
+               // z = a * (x - y)
+               // grad = a * (x - x0)
+               const double dist = D; // GetValues, default comp set to 0
+               const double a = 1.0 / (dist * dist);
+               const double w = weight * lim_normal * coeff0;
+               kernels::Subtract<3>(w*a, p1, p0, d1);
+               kernels::PushEval<MQ1>(qx,qy,qz,d1,QQQ0);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::LoadBt<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::EvalXt<MD1,MQ1>(D1D,Q1D,B,QQQ0,DQQ0);
+      kernels::EvalYt<MD1,MQ1>(D1D,Q1D,B,DQQ0,DDQ0);
+      kernels::EvalZt<MD1,MQ1>(D1D,Q1D,B,DDQ0,Y,e);
+   });
+}
+
+void TMOP_Integrator::AddMultPA_C0_3D(const Vector &X, Vector &Y) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double ln = lim_normal;
+   const Vector &LD = PA.LD;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &BLD = PA.maps_lim->B;
+   const Vector &X0 = PA.X0;
+   const Vector &C0 = PA.C0;
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_C0_3D,id,ln,LD,C0,N,J,W,B,BLD,X0,X,Y);
+}
+
+} // namespace mfem

fem/tmop_pa_w2.cpp

@@ -0,0 +1,155 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+#include "../linalg/dinvariants.hpp"
+
+namespace mfem
+{
+
+using Args = kernels::InvariantsEvaluator2D::Buffers;
+
+static MFEM_HOST_DEVICE inline
+double EvalW_001(const double *Jpt)
+{
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt));
+   return ie.Get_I1();
+}
+
+static MFEM_HOST_DEVICE inline
+double EvalW_002(const double *Jpt)
+{
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt));
+   return 0.5 * ie.Get_I1b() - 1.0;
+}
+
+static MFEM_HOST_DEVICE inline
+double EvalW_007(const double *Jpt)
+{
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt));
+   return ie.Get_I1() * (1.0 + 1.0/ie.Get_I2()) - 4.0;
+}
+
+static MFEM_HOST_DEVICE inline
+double EvalW_077(const double *Jpt)
+{
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt));
+   const double I2 = ie.Get_I2b(); // Get_I2b or Get_I2 ?
+   return 0.5*(I2*I2 + 1./(I2*I2) - 2.);
+}
+
+MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
+                           const double metric_normal,
+                           const int mid,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Vector &x_,
+                           const Vector &ones,
+                           Vector &energy,
+                           const int d1d,
+                           const int q1d)
+{
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+               "2D metric not yet implemented!");
+
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D);
+   const auto X = Reshape(x_.Read(), D1D, D1D, DIM, NE);
+
+   auto E = Reshape(energy.Write(), Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double XY[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[4][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[4][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const double *Jtr = &J(0,0,qx,qy,e);
+            const double detJtr = kernels::Det<2>(Jtr);
+            const double weight = metric_normal * W(qx,qy) * detJtr;
+
+            // Jrt = Jtr^{-1}
+            double Jrt[4];
+            kernels::CalcInverse<2>(Jtr, Jrt);
+
+            // Jpr = X^t.DSh
+            double Jpr[4];
+            kernels::PullGrad<MQ1,NBZ>(qx,qy,QQ,Jpr);
+
+            // Jpt = X^T.DS = (X^T.DSh).Jrt = Jpr.Jrt
+            double Jpt[4];
+            kernels::Mult(2,2,2,Jpr,Jrt,Jpt);
+
+            // metric->EvalW(Jpt);
+            const double EvalW =
+            mid ==  1 ? EvalW_001(Jpt) :
+            mid ==  2 ? EvalW_002(Jpt) :
+            mid ==  7 ? EvalW_007(Jpt) :
+            mid == 77 ? EvalW_077(Jpt) : 0.0;
+
+            E(qx,qy,e) = weight * EvalW;
+         }
+      }
+   });
+   return energy * ones;
+}
+
+double TMOP_Integrator::GetGridFunctionEnergyPA_2D(const Vector &X) const
+{
+   const int N = PA.ne;
+   const int M = metric->Id();
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double m = metric_normal;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   const Vector &O = PA.O;
+   Vector &E = PA.E;
+
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,m,M,N,J,W,B,G,X,O,E);
+}
+
+} // namespace mfem

fem/tmop_pa_w2_c0.cpp

@@ -0,0 +1,139 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_2D,
+                           const double lim_normal,
+                           const Vector &lim_dist,
+                           const Vector &c0_,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &bld_,
+                           const Vector &x0_,
+                           const Vector &x1_,
+                           const Vector &ones,
+                           Vector &energy,
+                           const int d1d,
+                           const int q1d)
+{
+   const bool const_c0 = c0_.Size() == 1;
+
+   constexpr int DIM = 2;
+   constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto C0 = const_c0 ?
+                   Reshape(c0_.Read(), 1, 1, 1) :
+                   Reshape(c0_.Read(), Q1D, Q1D, NE);
+   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, NE);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D);
+   const auto X0 = Reshape(x0_.Read(), D1D, D1D, DIM, NE);
+   const auto X1 = Reshape(x1_.Read(), D1D, D1D, DIM, NE);
+
+   auto E = Reshape(energy.Write(), Q1D, Q1D, NE);
+
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      MFEM_SHARED double BLD[MQ1*MD1];
+
+      MFEM_SHARED double XY[NBZ][MD1*MD1];
+      MFEM_SHARED double DQ[NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ[NBZ][MQ1*MQ1];
+
+      MFEM_SHARED double XY0[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ0[2][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ0[2][NBZ][MQ1*MQ1];
+
+      MFEM_SHARED double XY1[2][NBZ][MD1*MD1];
+      MFEM_SHARED double DQ1[2][NBZ][MD1*MQ1];
+      MFEM_SHARED double QQ1[2][NBZ][MQ1*MQ1];
+
+      kernels::LoadX<MD1,NBZ>(e,D1D,LD,XY);
+      kernels::LoadX<MD1,NBZ>(e,D1D,X0,XY0);
+      kernels::LoadX<MD1,NBZ>(e,D1D,X1,XY1);
+
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
+
+      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,BLD,XY,DQ);
+      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,BLD,DQ,QQ);
+
+      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY0,DQ0);
+      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ0,QQ0);
+
+      kernels::EvalX<MD1,MQ1,NBZ>(D1D,Q1D,B,XY1,DQ1);
+      kernels::EvalY<MD1,MQ1,NBZ>(D1D,Q1D,B,DQ1,QQ1);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double ld, p0[2], p1[2];
+            const double *Jtr = &J(0,0,qx,qy,e);
+            const double detJtr = kernels::Det<2>(Jtr);
+            const double weight = W(qx,qy) * detJtr;
+            const double coeff0 = const_c0 ? C0(0,0,0) : C0(qx,qy,e);
+            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ,ld);
+            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ0,p0);
+            kernels::PullEval<MQ1,NBZ>(qx,qy,QQ1,p1);
+            const double dist = ld; // GetValues, default comp set to 0
+            const double id2 = 0.5 / (dist*dist);
+            const double dsq = kernels::DistanceSquared<2>(p1,p0) * id2;
+            E(qx,qy,e) = weight * lim_normal * dsq * coeff0;
+         }
+      }
+   });
+   return energy * ones;
+}
+
+double TMOP_Integrator::GetGridFunctionEnergyPA_C0_2D(const Vector &X) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double ln = lim_normal;
+   const Vector &LD = PA.LD;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W   = PA.ir->GetWeights();
+   const Array<double> &B   = PA.maps->B;
+   const Array<double> &BLD = PA.maps_lim->B;
+   const Vector &X0 = PA.X0;
+   const Vector &C0 = PA.C0;
+   const Vector &O = PA.O;
+   Vector &E = PA.E;
+
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_C0_2D,id,ln,LD,C0,N,J,W,B,BLD,X0,X,O,E);
+}
+
+} // namespace mfem

fem/tmop_pa_w3.cpp

@@ -0,0 +1,165 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+#include "../linalg/dinvariants.hpp"
+
+namespace mfem
+{
+
+using Args = kernels::InvariantsEvaluator3D::Buffers;
+
+// mu_302 = I1b * I2b / 9 - 1
+static MFEM_HOST_DEVICE inline
+double EvalW_302(const double *J)
+{
+   double B[9];
+   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B));
+   return ie.Get_I1b()*ie.Get_I2b()/9. - 1.;
+}
+
+// mu_303 = I1b/3 - 1
+static MFEM_HOST_DEVICE inline
+double EvalW_303(const double *J)
+{
+   double B[9];
+   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B));
+   return ie.Get_I1b()/3. - 1.;
+}
+
+// mu_315 = (I3b - 1)^2
+static MFEM_HOST_DEVICE inline
+double EvalW_315(const double *J)
+{
+   double B[9];
+   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B));
+   const double a = ie.Get_I3b() - 1.0;
+   return a*a;
+}
+
+// mu_321 = I1 + I2/I3 - 6
+static MFEM_HOST_DEVICE inline
+double EvalW_321(const double *J)
+{
+   double B[9];
+   kernels::InvariantsEvaluator3D ie(Args().J(J).B(B));
+   return ie.Get_I1() + ie.Get_I2()/ie.Get_I3() - 6.0;
+}
+
+MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
+                           const double metric_normal,
+                           const int mid,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Vector &ones,
+                           const Vector &x_,
+                           Vector &energy,
+                           const int d1d,
+                           const int q1d)
+{
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
+               "3D metric not yet implemented!");
+
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto g = Reshape(g_.Read(), Q1D, D1D);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
+   const auto X = Reshape(x_.Read(), D1D, D1D, D1D, DIM, NE);
+
+   auto E = Reshape(energy.Write(), Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double BG[2][MQ1*MD1];
+      MFEM_SHARED double DDD[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[6][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[9][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[9][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+
+      kernels::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double *Jtr = &J(0,0,qx,qy,qz,e);
+               const double detJtr = kernels::Det<3>(Jtr);
+               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+
+               // Jrt = Jtr^{-1}
+               double Jrt[9];
+               kernels::CalcInverse<3>(Jtr, Jrt);
+
+               // Jpr = X^t.DSh
+               double Jpr[9];
+               kernels::PullGrad<MQ1>(qx,qy,qz, QQQ, Jpr);
+
+               // Jpt = X^t.DS = (X^t.DSh).Jrt = Jpr.Jrt
+               double Jpt[9];
+               kernels::Mult(3,3,3, Jpr, Jrt, Jpt);
+
+               // metric->EvalW(Jpt);
+               const double EvalW =
+               mid == 302 ? EvalW_302(Jpt) :
+               mid == 303 ? EvalW_303(Jpt) :
+               mid == 315 ? EvalW_315(Jpt) :
+               mid == 321 ? EvalW_321(Jpt) : 0.0;
+
+               E(qx,qy,qz,e) = weight * EvalW;
+            }
+         }
+      }
+   });
+   return energy * ones;
+}
+
+double TMOP_Integrator::GetGridFunctionEnergyPA_3D(const Vector &X) const
+{
+   const int N = PA.ne;
+   const int M = metric->Id();
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double mn = metric_normal;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &G = PA.maps->G;
+   const Vector &O = PA.O;
+   Vector &E = PA.E;
+
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,M,N,J,W,B,G,O,X,E);
+}
+
+} // namespace mfem

fem/tmop_pa_w3_c0.cpp

@@ -0,0 +1,148 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop.hpp"
+#include "tmop_pa.hpp"
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_3D,
+                           const double lim_normal,
+                           const Vector &lim_dist,
+                           const Vector &c0_,
+                           const int NE,
+                           const DenseTensor &j_,
+                           const Array<double> &w_,
+                           const Array<double> &b_,
+                           const Array<double> &bld_,
+                           const Vector &x0_,
+                           const Vector &x1_,
+                           const Vector &ones,
+                           Vector &energy,
+                           const int d1d,
+                           const int q1d)
+{
+   const bool const_c0 = c0_.Size() == 1;
+
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto C0 = const_c0 ?
+                   Reshape(c0_.Read(), 1, 1, 1, 1) :
+                   Reshape(c0_.Read(), Q1D, Q1D, Q1D, NE);
+   const auto LD = Reshape(lim_dist.Read(), D1D, D1D, D1D, NE);
+   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto bld = Reshape(bld_.Read(), Q1D, D1D);
+   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
+   const auto X0 = Reshape(x0_.Read(), D1D, D1D, D1D, DIM, NE);
+   const auto X1 = Reshape(x1_.Read(), D1D, D1D, D1D, DIM, NE);
+
+   auto E = Reshape(energy.Write(), Q1D, Q1D, Q1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX;
+
+      MFEM_SHARED double B[MQ1*MD1];
+      MFEM_SHARED double BLD[MQ1*MD1];
+
+      MFEM_SHARED double DDD[MD1*MD1*MD1];
+      MFEM_SHARED double DDQ[MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ[MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ[MQ1*MQ1*MQ1];
+
+      MFEM_SHARED double DDD0[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ0[3][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ0[3][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ0[3][MQ1*MQ1*MQ1];
+
+      MFEM_SHARED double DDD1[3][MD1*MD1*MD1];
+      MFEM_SHARED double DDQ1[3][MD1*MD1*MQ1];
+      MFEM_SHARED double DQQ1[3][MD1*MQ1*MQ1];
+      MFEM_SHARED double QQQ1[3][MQ1*MQ1*MQ1];
+
+      kernels::LoadX<MD1>(e,D1D,LD,DDD);
+      kernels::LoadX<MD1>(e,D1D,X0,DDD0);
+      kernels::LoadX<MD1>(e,D1D,X1,DDD1);
+
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::LoadB<MD1,MQ1>(D1D,Q1D,bld,BLD);
+
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,BLD,DDD,DDQ);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,BLD,DDQ,DQQ);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,BLD,DQQ,QQQ);
+
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD0,DDQ0);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ0,DQQ0);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ0,QQQ0);
+
+      kernels::EvalX<MD1,MQ1>(D1D,Q1D,B,DDD1,DDQ1);
+      kernels::EvalY<MD1,MQ1>(D1D,Q1D,B,DDQ1,DQQ1);
+      kernels::EvalZ<MD1,MQ1>(D1D,Q1D,B,DQQ1,QQQ1);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double D, p0[3], p1[3];
+               const double *Jtr = &J(0,0,qx,qy,qz,e);
+               const double detJtr = kernels::Det<3>(Jtr);
+               const double weight = W(qx,qy,qz) * detJtr;
+               const double coeff0 = const_c0 ? C0(0,0,0,0) : C0(qx,qy,qz,e);
+
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ,D);
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ0,p0);
+               kernels::PullEval<MQ1>(qx,qy,qz,QQQ1,p1);
+
+               const double dist = D; // GetValues, default comp set to 0
+               const double id2 = 0.5 / (dist*dist);
+
+               const double dsq = kernels::DistanceSquared<3>(p1,p0) * id2;
+               E(qx,qy,qz,e) = weight * lim_normal * dsq * coeff0;
+            }
+         }
+      }
+   });
+   return energy * ones;
+}
+
+double TMOP_Integrator::GetGridFunctionEnergyPA_C0_3D(const Vector &X) const
+{
+   const int N = PA.ne;
+   const int D1D = PA.maps->ndof;
+   const int Q1D = PA.maps->nqpt;
+   const int id = (D1D << 4 ) | Q1D;
+   const double ln = lim_normal;
+   const Vector &LD = PA.LD;
+   const DenseTensor &J = PA.Jtr;
+   const Array<double> &W = PA.ir->GetWeights();
+   const Array<double> &B = PA.maps->B;
+   const Array<double> &BLD = PA.maps_lim->B;
+   const Vector &X0 = PA.X0;
+   const Vector &C0 = PA.C0;
+   const Vector &O = PA.O;
+   Vector &E = PA.E;
+
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_C0_3D,id,ln,LD,C0,N,J,W,B,BLD,X0,X,O,E);
+}
+
+} // namespace mfem

fem/tmop_tools.cpp

@@ -33,10 +33,11 @@ void AdvectorCG::ComputeAtNewPosition(const Vector &new_nodes,
    const int pnt_cnt = new_field.Size()/ncomp;
 
    new_field = field0;
-
+   new_field.HostReadWrite();
+   Vector new_field_temp;
    for (int i = 0; i < ncomp; i++)
    {
-      Vector new_field_temp(new_field.GetData()+i*pnt_cnt, pnt_cnt);
+      new_field_temp.MakeRef(new_field, i*pnt_cnt, pnt_cnt);
       ComputeAtNewPositionScalar(new_nodes, new_field_temp);
    }
 
@@ -72,13 +73,13 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
    if (fes)
    {
       fess = new FiniteElementSpace(fes->GetMesh(), fes->FEColl(), 1);
-      oper = new SerialAdvectorCGOper(nodes0, u, *fess);
+      oper = new SerialAdvectorCGOper(nodes0, u, *fess, al);
    }
 #ifdef MFEM_USE_MPI
    else if (pfes)
    {
       pfess = new ParFiniteElementSpace(pfes->GetParMesh(), pfes->FEColl(), 1);
-      oper  = new ParAdvectorCGOper(nodes0, u, *pfess);
+      oper  = new ParAdvectorCGOper(nodes0, u, *pfess, al);
    }
 #endif
    MFEM_VERIFY(oper != NULL,
@@ -94,6 +95,7 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
    double v_max = 0.0;
    const int s = new_field.Size();
 
+   u.HostReadWrite();
    for (int i = 0; i < s; i++)
    {
       double vel = 0.;
@@ -149,6 +151,7 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
 #endif
 
    // Trim the overshoots and undershoots.
+   new_field.HostReadWrite();
    for (int i = 0; i < s; i++)
    {
       if (new_field(i) < glob_minv) { new_field(i) = glob_minv; }
@@ -164,18 +167,21 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
 
 SerialAdvectorCGOper::SerialAdvectorCGOper(const Vector &x_start,
                                            GridFunction &vel,
-                                           FiniteElementSpace &fes)
+                                           FiniteElementSpace &fes,
+                                           AssemblyLevel al)
    : TimeDependentOperator(fes.GetVSize()),
      x0(x_start), x_now(*fes.GetMesh()->GetNodes()),
-     u(vel), u_coeff(&u), M(&fes), K(&fes)
+     u(vel), u_coeff(&u), M(&fes), K(&fes), al(al)
 {
    ConvectionIntegrator *Kinteg = new ConvectionIntegrator(u_coeff);
    K.AddDomainIntegrator(Kinteg);
+   K.SetAssemblyLevel(al);
    K.Assemble(0);
    K.Finalize(0);
 
    MassIntegrator *Minteg = new MassIntegrator;
    M.AddDomainIntegrator(Minteg);
+   M.SetAssemblyLevel(al);
    M.Assemble();
    M.Finalize();
 }
@@ -186,6 +192,11 @@ void SerialAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
    const double t = GetTime();
    add(x0, t, u, x_now);
 
+   if (al == AssemblyLevel::PARTIAL)
+   {
+      K.FESpace()->GetMesh()->DeleteGeometricFactors();
+   }
+
    // Assemble on the new mesh.
    K.BilinearForm::operator=(0.0);
    K.Assemble();
@@ -196,30 +207,45 @@ void SerialAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
 
    di_dt = 0.0;
    CGSolver lin_solver;
-   DSmoother prec;
-   lin_solver.SetPreconditioner(prec);
-   lin_solver.SetOperator(M.SpMat());
+   Solver *prec = nullptr;
+   Array<int> ess_tdof_list;
+   if (al == AssemblyLevel::PARTIAL)
+   {
+      prec = new OperatorJacobiSmoother(M, ess_tdof_list);
+      lin_solver.SetOperator(M);
+   }
+   else
+   {
+      prec = new DSmoother(M.SpMat());
+      lin_solver.SetOperator(M.SpMat());
+   }
+   lin_solver.SetPreconditioner(*prec);
    lin_solver.SetRelTol(1e-12); lin_solver.SetAbsTol(0.0);
    lin_solver.SetMaxIter(100);
    lin_solver.SetPrintLevel(0);
    lin_solver.Mult(rhs, di_dt);
+
+   delete prec;
 }
 
 #ifdef MFEM_USE_MPI
 ParAdvectorCGOper::ParAdvectorCGOper(const Vector &x_start,
                                      GridFunction &vel,
-                                     ParFiniteElementSpace &pfes)
+                                     ParFiniteElementSpace &pfes,
+                                     AssemblyLevel al)
    : TimeDependentOperator(pfes.GetVSize()),
      x0(x_start), x_now(*pfes.GetMesh()->GetNodes()),
-     u(vel), u_coeff(&u), M(&pfes), K(&pfes)
+     u(vel), u_coeff(&u), M(&pfes), K(&pfes), al(al)
 {
    ConvectionIntegrator *Kinteg = new ConvectionIntegrator(u_coeff);
    K.AddDomainIntegrator(Kinteg);
+   K.SetAssemblyLevel(al);
    K.Assemble(0);
    K.Finalize(0);
 
    MassIntegrator *Minteg = new MassIntegrator;
    M.AddDomainIntegrator(Minteg);
+   M.SetAssemblyLevel(al);
    M.Assemble();
    M.Finalize();
 }
@@ -230,6 +256,11 @@ void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
    const double t = GetTime();
    add(x0, t, u, x_now);
 
+   if (al == AssemblyLevel::PARTIAL)
+   {
+      K.ParFESpace()->GetParMesh()->DeleteGeometricFactors();
+   }
+
    // Assemble on the new mesh.
    K.BilinearForm::operator=(0.0);
    K.Assemble();
@@ -241,13 +272,25 @@ void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
    HypreParVector *RHS = rhs.ParallelAssemble();
    HypreParVector X(K.ParFESpace());
    X = 0.0;
-   HypreParMatrix *Mh  = M.ParallelAssemble();
+
+   OperatorHandle Mop;
+   Solver *prec = nullptr;
+   Array<int> ess_tdof_list;
+   if (al == AssemblyLevel::PARTIAL)
+   {
+      M.FormSystemMatrix(ess_tdof_list, Mop);
+      prec = new OperatorJacobiSmoother(M, ess_tdof_list);
+   }
+   else
+   {
+      Mop.Reset(M.ParallelAssemble());
+      prec = new HypreSmoother;
+      static_cast<HypreSmoother*>(prec)->SetType(HypreSmoother::Jacobi, 1);
+   }
 
    CGSolver lin_solver(M.ParFESpace()->GetParMesh()->GetComm());
-   HypreSmoother prec;
-   prec.SetType(HypreSmoother::Jacobi, 1);
-   lin_solver.SetPreconditioner(prec);
-   lin_solver.SetOperator(*Mh);
+   lin_solver.SetPreconditioner(*prec);
+   lin_solver.SetOperator(*Mop);
    lin_solver.SetRelTol(1e-8);
    lin_solver.SetAbsTol(0.0);
    lin_solver.SetMaxIter(100);
@@ -255,8 +298,8 @@ void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
    lin_solver.Mult(*RHS, X);
    K.ParFESpace()->GetProlongationMatrix()->Mult(X, di_dt);
 
-   delete Mh;
    delete RHS;
+   delete prec;
 }
 #endif
 
@@ -374,18 +417,26 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 #endif
 
    double min_detJ = infinity();
-   for (int i = 0; i < NE; i++)
+   if (dim == 1)
    {
-      fes->GetElementVDofs(i, xdofs);
-      x_out_loc.GetSubVector(xdofs, posV);
-
-      for (int j = 0; j < nsp; j++)
+      for (int i = 0; i < NE; i++)
       {
-         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-         MultAtB(pos, dshape, Jpr);
-         min_detJ = std::min(min_detJ, Jpr.Det());
+         fes->GetElementVDofs(i, xdofs);
+         x_out_loc.GetSubVector(xdofs, posV);
+
+         for (int j = 0; j < nsp; j++)
+         {
+            fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+            MultAtB(pos, dshape, Jpr);
+            min_detJ = std::min(min_detJ, Jpr.Det());
+         }
       }
    }
+   else
+   {
+      min_detJ = dim == 2 ? MinDetJpr_2D(fes, x_out_loc) :
+                 dim == 3 ? MinDetJpr_3D(fes, x_out_loc) : 0.0;
+   }
    double min_detJ_all = min_detJ;
 #ifdef MFEM_USE_MPI
    if (parallel)
@@ -427,19 +478,26 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (!untangling)
       {
          int jac_ok = 1;
-         for (int i = 0; i < NE; i++)
+         if (dim == 1)
          {
-            fes->GetElementVDofs(i, xdofs);
-            x_out_loc.GetSubVector(xdofs, posV);
-            for (int j = 0; j < nsp; j++)
+            for (int i = 0; i < NE; i++)
             {
-               fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-               MultAtB(pos, dshape, Jpr);
-               if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
+               fes->GetElementVDofs(i, xdofs);
+               x_out_loc.GetSubVector(xdofs, posV);
+               for (int j = 0; j < nsp; j++)
+               {
+                  fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+                  MultAtB(pos, dshape, Jpr);
+                  if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
+               }
             }
+         break2:;
+         }
+         else
+         {
+            jac_ok = dim == 2 ? CheckDetJpr_2D(fes, x_out_loc) :
+                     dim == 3 ? CheckDetJpr_3D(fes, x_out_loc) : 0;
          }
-
-      break2:
          int jac_ok_all = jac_ok;
 #ifdef MFEM_USE_MPI
          if (parallel)
@@ -448,7 +506,6 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
                           p_nlf->ParFESpace()->GetComm());
          }
 #endif
-
          if (jac_ok_all == 0)
          {
             if (print_level >= 0)
@@ -458,6 +515,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       } // endif(!untangling)
 
       ProcessNewState(x_out);
+
       if (serial)
       {
          energy_out = nlf->GetGridFunctionEnergy(x_out_loc);
@@ -564,6 +622,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
             Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
             for (int j = 0; j < ati.Size(); j++)
             {
+               ati[j]->UpdateAfterMeshChange(x_loc);
                ati[j]->ComputeFDh(x_loc, *pfesc);
                UpdateDiscreteTC(*ati[j], x_loc);
             }
@@ -600,6 +659,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
             Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
             for (int j = 0; j < ati.Size(); j++)
             {
+               ati[j]->UpdateAfterMeshChange(x_loc);
                ati[j]->ComputeFDh(x_loc, *fesc);
                UpdateDiscreteTC(*ati[j], x_loc);
             }

fem/tmop_tools.hpp

@@ -28,12 +28,14 @@ private:
    Vector nodes0;
    Vector field0;
    const double dt_scale;
+   const AssemblyLevel al;
 
    void ComputeAtNewPositionScalar(const Vector &new_nodes, Vector &new_field);
 public:
-   AdvectorCG(double timestep_scale = 0.5)
+   AdvectorCG(AssemblyLevel al = AssemblyLevel::LEGACYFULL,
+              double timestep_scale = 0.5)
       : AdaptivityEvaluator(),
-        ode_solver(), nodes0(), field0(), dt_scale(timestep_scale) { }
+        ode_solver(), nodes0(), field0(), dt_scale(timestep_scale), al(al) { }
 
    virtual void SetInitialField(const Vector &init_nodes,
                                 const Vector &init_field);
@@ -78,12 +80,14 @@ protected:
    GridFunction &u;
    VectorGridFunctionCoefficient u_coeff;
    mutable BilinearForm M, K;
+   const AssemblyLevel al;
 
 public:
    /** Here @a fes is the FESpace of the function that will be moved. Note
        that Mult() moves the nodes of the mesh corresponding to @a fes. */
    SerialAdvectorCGOper(const Vector &x_start, GridFunction &vel,
-                        FiniteElementSpace &fes);
+                        FiniteElementSpace &fes,
+                        AssemblyLevel al = AssemblyLevel::LEGACYFULL);
 
    virtual void Mult(const Vector &ind, Vector &di_dt) const;
 };
@@ -98,12 +102,14 @@ protected:
    GridFunction &u;
    VectorGridFunctionCoefficient u_coeff;
    mutable ParBilinearForm M, K;
+   const AssemblyLevel al;
 
 public:
    /** Here @a pfes is the ParFESpace of the function that will be moved. Note
        that Mult() moves the nodes of the mesh corresponding to @a pfes. */
    ParAdvectorCGOper(const Vector &x_start, GridFunction &vel,
-                     ParFiniteElementSpace &pfes);
+                     ParFiniteElementSpace &pfes,
+                     AssemblyLevel al = AssemblyLevel::LEGACYFULL);
 
    virtual void Mult(const Vector &ind, Vector &di_dt) const;
 };
@@ -159,6 +165,10 @@ public:
       else { MFEM_ABORT("Invalid type"); }
    }
    virtual void SetPreconditioner(Solver &pr) { SetSolver(pr); }
+   int CheckDetJpr_2D(const FiniteElementSpace*, const Vector&) const;
+   int CheckDetJpr_3D(const FiniteElementSpace*, const Vector&) const;
+   double MinDetJpr_2D(const FiniteElementSpace*, const Vector&) const;
+   double MinDetJpr_3D(const FiniteElementSpace*, const Vector&) const;
 };
 
 void vis_tmop_metric_s(int order, TMOP_QualityMetric &qm,

general/CMakeLists.txt

@@ -35,6 +35,7 @@ list(APPEND HDRS
   backends.hpp
   binaryio.hpp
   cuda.hpp
+  debug.hpp
   device.hpp
   error.hpp
   gecko.hpp

general/backends.hpp

@@ -51,6 +51,7 @@
       (defined(MFEM_USE_HIP)  && defined(__HIP_DEVICE_COMPILE__)))
 #define MFEM_SHARED
 #define MFEM_SYNC_THREAD
+#define MFEM_BLOCK_ID(k) 0
 #define MFEM_THREAD_ID(k) 0
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)

general/cuda.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -61,6 +61,21 @@ void* CuMallocManaged(void** dptr, size_t bytes)
    return *dptr;
 }
 
+void* CuMemAllocHostPinned(void** ptr, size_t bytes)
+{
+#ifdef MFEM_USE_CUDA
+#ifdef MFEM_TRACK_CUDA_MEM
+   mfem::out << "CuMemAllocHostPinned(): allocating " << bytes << " bytes ... "
+             << std::flush;
+#endif
+   MFEM_GPU_CHECK(cudaMallocHost(ptr, bytes));
+#ifdef MFEM_TRACK_CUDA_MEM
+   mfem::out << "done: " << *ptr << std::endl;
+#endif
+#endif
+   return *ptr;
+}
+
 void* CuMemFree(void *dptr)
 {
 #ifdef MFEM_USE_CUDA
@@ -76,6 +91,21 @@ void* CuMemFree(void *dptr)
    return dptr;
 }
 
+void* CuMemFreeHostPinned(void *ptr)
+{
+#ifdef MFEM_USE_CUDA
+#ifdef MFEM_TRACK_CUDA_MEM
+   mfem::out << "CuMemFreeHostPinned(): deallocating memory @ " << ptr << " ... "
+             << std::flush;
+#endif
+   MFEM_GPU_CHECK(cudaFreeHost(ptr));
+#ifdef MFEM_TRACK_CUDA_MEM
+   mfem::out << "done." << std::endl;
+#endif
+#endif
+   return ptr;
+}
+
 void* CuMemcpyHtoD(void* dst, const void* src, size_t bytes)
 {
 #ifdef MFEM_USE_CUDA

general/cuda.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -63,9 +63,15 @@ void* CuMemAlloc(void **d_ptr, size_t bytes);
 /// Allocates managed device memory
 void* CuMallocManaged(void **d_ptr, size_t bytes);
 
+/// Allocates page-locked (pinned) host memory
+void* CuMemAllocHostPinned(void **ptr, size_t bytes);
+
 /// Frees device memory and returns destination ptr.
 void* CuMemFree(void *d_ptr);
 
+/// Frees page-locked (pinned) host memory and returns destination ptr.
+void* CuMemFreeHostPinned(void *ptr);
+
 /// Copies memory from Host to Device and returns destination ptr.
 void* CuMemcpyHtoD(void *d_dst, const void *h_src, size_t bytes);
 

general/debug.hpp

@@ -0,0 +1,178 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DEBUG_HPP
+#define MFEM_DEBUG_HPP
+
+#include <string>
+#include <cstring>
+#include <iomanip>
+#include <iostream>
+
+#include "globals.hpp"
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif
+
+namespace mfem
+{
+
+class Debug
+{
+   const bool debug = false;
+public:
+   inline Debug() {}
+
+   inline Debug(const int mpi_rank,
+                const char *FILE, const int LINE,
+                const char *FUNC, int COLOR): debug(true)
+   {
+      if (!debug) { return; }
+      const char *base = Strrnchr(FILE,'/', 2);
+      const char *file = base ? base + 1 : FILE;
+      const uint8_t color = COLOR ? COLOR : 20 + Checksum8(FILE) % 210;
+      mfem::out << "\033[38;5;" << std::to_string(color) << "m";
+      mfem::out << mpi_rank << std::setw(30) << file << ":";
+      mfem::out << "\033[2m" << std::setw(4) << LINE << "\033[22m: ";
+      if (FUNC) { mfem::out << "[" << FUNC << "] "; }
+      mfem::out << "\033[1m";
+   }
+
+   ~Debug()
+   {
+      if (!debug) { return; }
+      mfem::out << "\033[m";
+      mfem::out << std::endl;
+   }
+
+   template <typename T>
+   inline void operator<<(const T &arg) const noexcept { mfem::out << arg; }
+
+   template<typename T, typename... Args>
+   inline void operator()(const char *fmt, const T &arg,
+                          Args... args) const noexcept
+   {
+      if (!debug) { return; }
+      for (; *fmt != '\0'; fmt++ )
+      {
+         if (*fmt == '%')
+         {
+            fmt++;
+            const char c = *fmt;
+            if (c == 'p') { operator<<(arg); }
+            if (c == 's' || c == 'd' || c == 'f') { operator<<(arg); }
+            if (c == 'x' || c == 'X')
+            {
+               mfem::out << std::hex;
+               if (c == 'X') { mfem::out << std::uppercase; }
+               operator<<(arg);
+               mfem::out << std::nouppercase << std::dec;
+            }
+            if (c == '.')
+            {
+               fmt++;
+               const char c = *fmt;
+               char num[8] = { 0 };
+               for (int k = 0; *fmt != '\0'; fmt++, k++)
+               {
+                  if (*fmt == 'e' || *fmt == 'f') { break; }
+                  if (*fmt < 0x30 || *fmt > 0x39) { break; }
+                  num[k] = *fmt;
+               }
+               const int fx = std::atoi(num);
+               if (c == 'e') { mfem::out << std::scientific; }
+               if (c == 'f') { mfem::out << std::fixed; }
+               mfem::out << std::setprecision(fx);
+               operator<<(arg);
+               mfem::out << std::setprecision(6);
+            }
+            return operator()(fmt + 1, args...);
+         }
+         operator<<(*fmt);
+      }
+   }
+
+   template<typename T>
+   inline void operator()(const T &arg) const noexcept
+   {
+      if (!debug) { return; }
+      operator<<(arg);
+   }
+
+   inline void operator()() const noexcept { }
+
+public:
+   static const Debug Set(const char *FILE, const int LINE, const char *FUNC,
+                          int COLOR = 0)
+   {
+      static int mpi_dbg = 0, mpi_rank = 0;
+      static bool env_mpi = false, env_dbg = false;
+      static bool ini_dbg = false;
+      if (!ini_dbg)
+      {
+         const char *DBG = getenv("MFEM_DEBUG");
+         const char *MPI = getenv("MFEM_DEBUG_MPI");
+         env_dbg = DBG != nullptr;
+         env_mpi = MPI != nullptr;
+#ifdef MFEM_USE_MPI
+         int mpi_ini = false;
+         MPI_Initialized(&mpi_ini);
+         if (mpi_ini) { MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); }
+         mpi_dbg = atoi(env_mpi ? MPI : "0");
+#endif
+         ini_dbg = true;
+      }
+      const bool debug = (env_dbg && (!env_mpi || mpi_rank == mpi_dbg));
+      return debug ? Debug(mpi_rank, FILE, LINE, FUNC, COLOR) : Debug();
+   }
+
+private:
+   inline uint8_t Checksum8(const char *bfr)
+   {
+      unsigned int chk = 0;
+      size_t len = strlen(bfr);
+      for (; len; len--,bfr++) { chk += static_cast<unsigned int>(*bfr); }
+      return (uint8_t) chk;
+   }
+
+   inline const char *Strrnchr(const char *s, const unsigned char c, int n)
+   {
+      size_t len = strlen(s);
+      char *p = const_cast<char*>(s) + len - 1;
+      for (; n; n--,p--,len--)
+      {
+         for (; len; p--,len--)
+            if (*p == c) { break; }
+         if (!len) { return nullptr; }
+         if (n == 1) { return p; }
+      }
+      return nullptr;
+   }
+
+};
+
+#ifndef MFEM_DEBUG_COLOR
+#define MFEM_DEBUG_COLOR 0
+#endif
+
+#define dbg(...) \
+    mfem::Debug::Set(__FILE__,__LINE__,__FUNCTION__,MFEM_DEBUG_COLOR).\
+    operator()(__VA_ARGS__)
+
+} // mfem namespace
+
+#define DBG(...) { printf("\033[33m");  \
+                   printf(__VA_ARGS__); \
+                   printf(" \n\033[m"); \
+                   fflush(0); }
+
+#endif // MFEM_DEBUG_HPP

general/device.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -12,9 +12,10 @@
 #include "forall.hpp"
 #include "occa.hpp"
 #ifdef MFEM_USE_CEED
-#include <ceed.h>
+#include "../fem/ceed/util.hpp"
 #endif
 
+#include <unordered_map>
 #include <string>
 #include <map>
 
@@ -33,13 +34,16 @@ occa::device occaDevice;
 
 #ifdef MFEM_USE_CEED
 Ceed ceed = NULL;
+
+ceed::BasisMap ceed_basis_map;
+ceed::RestrMap ceed_restr_map;
 #endif
 
 // Backends listed by priority, high to low:
 static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 {
    Backend::CEED_CUDA, Backend::OCCA_CUDA, Backend::RAJA_CUDA, Backend::CUDA,
-   Backend::HIP, Backend::DEBUG,
+   Backend::CEED_HIP, Backend::RAJA_HIP, Backend::HIP, Backend::DEBUG,
    Backend::OCCA_OMP, Backend::RAJA_OMP, Backend::OMP,
    Backend::CEED_CPU, Backend::OCCA_CPU, Backend::RAJA_CPU, Backend::CPU
 };
@@ -48,7 +52,7 @@ static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 static const char *backend_name[Backend::NUM_BACKENDS] =
 {
    "ceed-cuda", "occa-cuda", "raja-cuda", "cuda",
-   "hip", "debug",
+   "ceed-hip", "raja-hip", "hip", "debug",
    "occa-omp", "raja-omp", "omp",
    "ceed-cpu", "occa-cpu", "raja-cpu", "cpu"
 };
@@ -61,15 +65,9 @@ Device Device::device_singleton;
 bool Device::device_env = false;
 bool Device::mem_host_env = false;
 bool Device::mem_device_env = false;
+bool Device::mem_types_set = false;
 
-Device::Device() : mode(Device::SEQUENTIAL),
-   backends(Backend::CPU),
-   destroy_mm(false),
-   mpi_gpu_aware(false),
-   host_mem_type(MemoryType::HOST),
-   host_mem_class(MemoryClass::HOST),
-   device_mem_type(MemoryType::HOST),
-   device_mem_class(MemoryClass::HOST)
+Device::Device()
 {
    if (getenv("MFEM_MEMORY") && !mem_host_env && !mem_device_env)
    {
@@ -154,6 +152,18 @@ Device::~Device()
    {
       free(device_option);
 #ifdef MFEM_USE_CEED
+      // Destroy FES -> CeedBasis, CeedElemRestriction hash table contents
+      for (auto entry : internal::ceed_basis_map)
+      {
+         CeedBasisDestroy(&entry.second);
+      }
+      internal::ceed_basis_map.clear();
+      for (auto entry : internal::ceed_restr_map)
+      {
+         CeedElemRestrictionDestroy(&entry.second);
+      }
+      internal::ceed_restr_map.clear();
+      // Destroy Ceed context
       CeedDestroy(&internal::ceed);
 #endif
       mm.Destroy();
@@ -210,15 +220,24 @@ void Device::Configure(const std::string &device, const int dev)
       beg = end + 1;
    }
 
-   // OCCA_CUDA needs CUDA or RAJA_CUDA:
-   if (Allows(Backend::OCCA_CUDA) && !Allows(Backend::RAJA_CUDA))
+   // OCCA_CUDA and CEED_CUDA need CUDA or RAJA_CUDA:
+   if (Allows(Backend::OCCA_CUDA|Backend::CEED_CUDA) &&
+       !Allows(Backend::RAJA_CUDA))
    {
       Get().MarkBackend(Backend::CUDA);
    }
-   if (Allows(Backend::CEED_CUDA))
+   // CEED_HIP needs HIP:
+   if (Allows(Backend::CEED_HIP))
    {
-      Get().MarkBackend(Backend::CUDA);
+      Get().MarkBackend(Backend::HIP);
    }
+   // OCCA_OMP will use OMP or RAJA_OMP unless MFEM_USE_OPENMP=NO:
+#ifdef MFEM_USE_OPENMP
+   if (Allows(Backend::OCCA_OMP) && !Allows(Backend::RAJA_OMP))
+   {
+      Get().MarkBackend(Backend::OMP);
+   }
+#endif
 
    // Perform setup.
    Get().Setup(dev);
@@ -233,6 +252,30 @@ void Device::Configure(const std::string &device, const int dev)
    destroy_mm = true;
 }
 
+// static method
+void Device::SetMemoryTypes(MemoryType h_mt, MemoryType d_mt)
+{
+   // If the device and/or the MemoryTypes are configured through the
+   // environment (variables 'MFEM_DEVICE', 'MFEM_MEMORY'), ignore calls to this
+   // method.
+   if (mem_host_env || mem_device_env || device_env) { return; }
+
+   MFEM_VERIFY(!IsConfigured(), "the default MemoryTypes can only be set before"
+               " Device construction and configuration");
+   MFEM_VERIFY(IsHostMemory(h_mt),
+               "invalid host MemoryType, h_mt = " << (int)h_mt);
+   MFEM_VERIFY(IsDeviceMemory(d_mt) || d_mt == h_mt,
+               "invalid device MemoryType, d_mt = " << (int)d_mt
+               << " (h_mt = " << (int)h_mt << ')');
+
+   Get().host_mem_type = h_mt;
+   Get().device_mem_type = d_mt;
+   mem_types_set = true;
+
+   // h_mt and d_mt will be set as dual to each other during configuration by
+   // the call mm.Configure(...) in UpdateMemoryTypeAndClass()
+}
+
 void Device::Print(std::ostream &out)
 {
    out << "Device configuration: ";
@@ -272,7 +315,14 @@ void Device::UpdateMemoryTypeAndClass()
 
 #ifdef MFEM_USE_UMPIRE
    // If MFEM has been compiled with Umpire support, use it as the default
-   if (!mem_host_env) { host_mem_type = MemoryType::HOST_UMPIRE; }
+   if (!mem_host_env && !mem_types_set)
+   {
+      host_mem_type = MemoryType::HOST_UMPIRE;
+      if (!mem_device_env)
+      {
+         device_mem_type = MemoryType::HOST_UMPIRE;
+      }
+   }
 #endif
 
    // Enable the device memory type
@@ -294,7 +344,7 @@ void Device::UpdateMemoryTypeAndClass()
                   device_mem_type = MemoryType::DEVICE;
             }
          }
-         else
+         else if (!mem_types_set)
          {
 #ifndef MFEM_USE_UMPIRE
             device_mem_type = MemoryType::DEVICE;
@@ -320,6 +370,9 @@ void Device::UpdateMemoryTypeAndClass()
       device_mem_type = MemoryType::DEVICE_DEBUG;
    }
 
+   MFEM_VERIFY(!device || IsDeviceMemory(device_mem_type),
+               "invalid device memory configuration!");
+
    // Update the memory manager with the new settings
    mm.Configure(host_mem_type, device_mem_type);
 }
@@ -369,6 +422,8 @@ static void RajaDeviceSetup(const int dev, int &ngpu)
 {
 #ifdef MFEM_USE_CUDA
    if (ngpu <= 0) { DeviceSetup(dev, ngpu); }
+#elif defined(MFEM_USE_HIP)
+   HipDeviceSetup(dev, ngpu);
 #else
    MFEM_CONTRACT_VAR(dev);
    MFEM_CONTRACT_VAR(ngpu);
@@ -435,7 +490,8 @@ static void CeedDeviceSetup(const char* ceed_spec)
    CeedInit(ceed_spec, &internal::ceed);
    const char *ceed_backend;
    CeedGetResource(internal::ceed, &ceed_backend);
-   if (strcmp(ceed_spec, ceed_backend) && strcmp(ceed_spec, "/cpu/self"))
+   if (strcmp(ceed_spec, ceed_backend) && strcmp(ceed_spec, "/cpu/self") &&
+       strcmp(ceed_spec, "/gpu/hip"))
    {
       mfem::out << std::endl << "WARNING!!!\n"
                 "libCEED is not using the requested backend!!!\n"
@@ -473,12 +529,16 @@ void Device::Setup(const int device)
    MFEM_VERIFY(!Allows(Backend::CEED_MASK),
                "the CEED backends require MFEM built with MFEM_USE_CEED=YES");
 #else
-   MFEM_VERIFY(!Allows(Backend::CEED_CPU) || !Allows(Backend::CEED_CUDA),
+   int ceed_cpu  = Allows(Backend::CEED_CPU);
+   int ceed_cuda = Allows(Backend::CEED_CUDA);
+   int ceed_hip  = Allows(Backend::CEED_HIP);
+   MFEM_VERIFY(ceed_cpu + ceed_cuda + ceed_hip <= 1,
                "Only one CEED backend can be enabled at a time!");
 #endif
    if (Allows(Backend::CUDA)) { CudaDeviceSetup(dev, ngpu); }
    if (Allows(Backend::HIP)) { HipDeviceSetup(dev, ngpu); }
-   if (Allows(Backend::RAJA_CUDA)) { RajaDeviceSetup(dev, ngpu); }
+   if (Allows(Backend::RAJA_CUDA) || Allows(Backend::RAJA_HIP))
+   { RajaDeviceSetup(dev, ngpu); }
    // The check for MFEM_USE_OCCA is in the function OccaDeviceSetup().
    if (Allows(Backend::OCCA_MASK)) { OccaDeviceSetup(dev); }
    if (Allows(Backend::CEED_CPU))
@@ -504,6 +564,17 @@ void Device::Setup(const int device)
          CeedDeviceSetup(device_option);
       }
    }
+   if (Allows(Backend::CEED_HIP))
+   {
+      if (!device_option)
+      {
+         CeedDeviceSetup("/gpu/hip");
+      }
+      else
+      {
+         CeedDeviceSetup(device_option);
+      }
+   }
    if (Allows(Backend::DEBUG)) { ngpu = 1; }
 }
 

general/device.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -46,26 +46,33 @@ struct Backend
       /** @brief [device] RAJA CUDA backend. Enabled when MFEM_USE_RAJA = YES
           and MFEM_USE_CUDA = YES. */
       RAJA_CUDA = 1 << 6,
+      /** @brief [device] RAJA HIP backend. Enabled when MFEM_USE_RAJA = YES
+          and MFEM_USE_HIP = YES. */
+      RAJA_HIP = 1 << 7,
       /** @brief [host] OCCA CPU backend: sequential execution on each MPI rank.
           Enabled when MFEM_USE_OCCA = YES. */
-      OCCA_CPU = 1 << 7,
+      OCCA_CPU = 1 << 8,
       /// [host] OCCA OpenMP backend. Enabled when MFEM_USE_OCCA = YES.
-      OCCA_OMP = 1 << 8,
+      OCCA_OMP = 1 << 9,
       /** @brief [device] OCCA CUDA backend. Enabled when MFEM_USE_OCCA = YES
           and MFEM_USE_CUDA = YES. */
-      OCCA_CUDA = 1 << 9,
+      OCCA_CUDA = 1 << 10,
       /** @brief [host] CEED CPU backend. GPU backends can still be used, but
           with expensive memory transfers. Enabled when MFEM_USE_CEED = YES. */
-      CEED_CPU  = 1 << 10,
+      CEED_CPU  = 1 << 11,
       /** @brief [device] CEED CUDA backend working together with the CUDA
           backend. Enabled when MFEM_USE_CEED = YES and MFEM_USE_CUDA = YES.
-          NOTE: The current default libCEED GPU backend is non-deterministic! */
-      CEED_CUDA = 1 << 11,
+          NOTE: The current default libCEED CUDA backend is non-deterministic! */
+      CEED_CUDA = 1 << 12,
+      /** @brief [device] CEED HIP backend working together with the HIP
+          backend. Enabled when MFEM_USE_CEED = YES and MFEM_USE_HIP = YES. */
+      CEED_HIP = 1 << 13,
       /** @brief [device] Debug backend: host memory is READ/WRITE protected
           while a device is in use. It allows to test the "device" code-path
           (using separate host/device memory pools and host <-> device
-          transfers) without any GPU hardware. */
-      DEBUG = 1 << 12
+          transfers) without any GPU hardware. As 'DEBUG' is sometimes used
+          as a macro, `_DEVICE` has been added to avoid conflicts. */
+      DEBUG = 1 << 14
    };
 
    /** @brief Additional useful constants. For example, the *_MASK constants can
@@ -73,23 +80,23 @@ struct Backend
    enum
    {
       /// Number of backends: from (1 << 0) to (1 << (NUM_BACKENDS-1)).
-      NUM_BACKENDS = 13,
+      NUM_BACKENDS = 15,
 
       /// Biwise-OR of all CPU backends
       CPU_MASK = CPU | RAJA_CPU | OCCA_CPU | CEED_CPU,
       /// Biwise-OR of all CUDA backends
       CUDA_MASK = CUDA | RAJA_CUDA | OCCA_CUDA | CEED_CUDA,
       /// Biwise-OR of all HIP backends
-      HIP_MASK = HIP,
+      HIP_MASK = HIP | RAJA_HIP | CEED_HIP,
       /// Biwise-OR of all OpenMP backends
       OMP_MASK = OMP | RAJA_OMP | OCCA_OMP,
       /// Bitwise-OR of all CEED backends
-      CEED_MASK = CEED_CPU | CEED_CUDA,
+      CEED_MASK = CEED_CPU | CEED_CUDA | CEED_HIP,
       /// Biwise-OR of all device backends
       DEVICE_MASK = CUDA_MASK | HIP_MASK | DEBUG,
 
       /// Biwise-OR of all RAJA backends
-      RAJA_MASK = RAJA_CPU | RAJA_OMP | RAJA_CUDA,
+      RAJA_MASK = RAJA_CPU | RAJA_OMP | RAJA_CUDA | RAJA_HIP,
       /// Biwise-OR of all OCCA backends
       OCCA_MASK = OCCA_CPU | OCCA_OMP | OCCA_CUDA
    };
@@ -117,22 +124,25 @@ private:
    friend class MemoryManager;
    enum MODES {SEQUENTIAL, ACCELERATED};
 
-   static bool device_env, mem_host_env, mem_device_env;
+   static bool device_env, mem_host_env, mem_device_env, mem_types_set;
    static Device device_singleton;
 
-   MODES mode;
-   int dev = 0; ///< Device ID of the configured device.
+   MODES mode = Device::SEQUENTIAL;
+   int dev = 0;   ///< Device ID of the configured device.
    int ngpu = -1; ///< Number of detected devices; -1: not initialized.
-   unsigned long backends; ///< Bitwise-OR of all configured backends.
+   /// Bitwise-OR of all configured backends.
+   unsigned long backends = Backend::CPU;
    /// Set to true during configuration, except in 'device_singleton'.
-   bool destroy_mm;
-   bool mpi_gpu_aware;
+   bool destroy_mm = false;
+   bool mpi_gpu_aware = false;
 
-   MemoryType host_mem_type;      ///< Current Host MemoryType
-   MemoryClass host_mem_class;    ///< Current Host MemoryClass
+   MemoryType host_mem_type = MemoryType::HOST;    ///< Current Host MemoryType
+   MemoryClass host_mem_class = MemoryClass::HOST; ///< Current Host MemoryClass
 
-   MemoryType device_mem_type;    ///< Current Device MemoryType
-   MemoryClass device_mem_class;  ///< Current Device MemoryClass
+   /// Current Device MemoryType
+   MemoryType device_mem_type = MemoryType::HOST;
+   /// Current Device MemoryClass
+   MemoryClass device_mem_class = MemoryClass::HOST;
 
    char *device_option = NULL;
    Device(Device const&);
@@ -173,14 +183,6 @@ public:
        @note This object should be destroyed after all other MFEM objects that
        use the Device are destroyed. */
    Device(const std::string &device, const int dev = 0)
-      : mode(Device::SEQUENTIAL),
-        backends(Backend::CPU),
-        destroy_mm(false),
-        mpi_gpu_aware(false),
-        host_mem_type(MemoryType::HOST),
-        host_mem_class(MemoryClass::HOST),
-        device_mem_type(MemoryType::HOST),
-        device_mem_class(MemoryClass::HOST)
    { Configure(device, dev); }
 
    /// Destructor.
@@ -193,25 +195,44 @@ public:
        * The available backends are described by the Backend class.
        * The string name of a backend is the lowercase version of the
          Backend::Id enumeration constant with '_' replaced by '-', e.g. the
-         string name of 'RAJA_CPU' is 'raja-cpu'.
+         string name of 'RAJA_CPU' is 'raja-cpu'. The string name of the debug
+         backend (Backend::Id 'DEBUG_DEVICE') is exceptionally set to 'debug'.
        * The 'cpu' backend is always enabled with lowest priority.
        * The current backend priority from highest to lowest is:
-         'ceed-cuda', 'occa-cuda', 'raja-cuda', 'cuda', 'hip', 'debug',
+         'ceed-cuda', 'occa-cuda', 'raja-cuda', 'cuda',
+         'ceed-hip', 'hip', 'debug',
          'occa-omp', 'raja-omp', 'omp',
          'ceed-cpu', 'occa-cpu', 'raja-cpu', 'cpu'.
        * Multiple backends can be configured at the same time.
        * Only one 'occa-*' backend can be configured at a time.
        * The backend 'occa-cuda' enables the 'cuda' backend unless 'raja-cuda'
          is already enabled.
+       * The backend 'occa-omp' enables the 'omp' backend (if MFEM was built
+         with MFEM_USE_OPENMP=YES) unless 'raja-omp' is already enabled.
+       * Only one 'ceed-*' backend can be configured at a time.
        * The backend 'ceed-cpu' delegates to a libCEED CPU backend the setup and
          evaluation of the operator.
        * The backend 'ceed-cuda' delegates to a libCEED CUDA backend the setup
-         and evaluation of the operator and enables the 'cuda' backend to avoid
-         transfer between host and device.
+         and evaluation of operators and enables the 'cuda' backend to avoid
+         transfers between host and device.
+       * The backend 'ceed-hip' delegates to a libCEED HIP backend the setup
+         and evaluation of operators and enables the 'hip' backend to avoid
+         transfers between host and device.
        * The 'debug' backend should not be combined with other device backends.
    */
    void Configure(const std::string &device, const int dev = 0);
 
+   /// Set the default host and device MemoryTypes, @a h_mt and @a d_mt.
+   /** The host and device MemoryTypes are also set to be dual to each other.
+
+       These two MemoryType%s are used by most MFEM classes when allocating
+       memory used on host and device, respectively.
+
+       This method can only be called before Device construction and
+       configuration, and the specified memory types must be compatible with
+       the subsequent Device configuration. */
+   static void SetMemoryTypes(MemoryType h_mt, MemoryType d_mt);
+
    /// Print the configuration of the MFEM virtual device object.
    void Print(std::ostream &out = mfem::out);
 

general/forall.hpp

@@ -23,8 +23,8 @@ namespace mfem
 {
 
 // Maximum size of dofs and quads in 1D.
-const int MAX_D1D = 14;
-const int MAX_Q1D = 14;
+const int MAX_D1D = 8;
+const int MAX_Q1D = 8;
 
 // MFEM pragma macros that can be used inside MFEM_FORALL macros.
 #define MFEM_PRAGMA(X) _Pragma(#X)
@@ -89,7 +89,11 @@ void OmpWrap(const int N, HBODY &&h_body)
 /// RAJA Cuda backend
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA)
 
+#if RAJA_VERSION_MAJOR == 0 && RAJA_VERSION_MINOR < 12
 using RAJA::statement::Segs;
+#else
+using RAJA::Segs;
+#endif
 
 template <const int BLOCKS = MFEM_CUDA_BLOCKS, typename DBODY>
 void RajaCudaWrap1D(const int N, DBODY &&d_body)
@@ -147,7 +151,12 @@ void RajaCudaWrap3D(const int N, DBODY &&d_body,
 /// RAJA OpenMP backend
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
 
+#if RAJA_VERSION_MAJOR == 0 && RAJA_VERSION_MINOR < 12
 using RAJA::statement::Segs;
+#else
+using RAJA::Segs;
+#endif
+
 
 template <typename HBODY>
 void RajaOmpWrap(const int N, HBODY &&h_body)

general/hip.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -61,6 +61,21 @@ void* HipMallocManaged(void** dptr, size_t bytes)
    return *dptr;
 }
 
+void* HipMemAllocHostPinned(void** ptr, size_t bytes)
+{
+#ifdef MFEM_USE_HIP
+#ifdef MFEM_TRACK_HIP_MEM
+   mfem::out << "HipMemAllocHostPinned(): allocating " << bytes << " bytes ... "
+             << std::flush;
+#endif
+   MFEM_GPU_CHECK(hipHostMalloc(ptr, bytes, hipHostMallocDefault));
+#ifdef MFEM_TRACK_HIP_MEM
+   mfem::out << "done: " << *ptr << std::endl;
+#endif
+#endif
+   return *ptr;
+}
+
 void* HipMemFree(void *dptr)
 {
 #ifdef MFEM_USE_HIP
@@ -76,6 +91,21 @@ void* HipMemFree(void *dptr)
    return dptr;
 }
 
+void* HipMemFreeHostPinned(void *ptr)
+{
+#ifdef MFEM_USE_HIP
+#ifdef MFEM_TRACK_HIP_MEM
+   mfem::out << "HipMemFreeHostPinned(): deallocating memory @ " << ptr << " ... "
+             << std::flush;
+#endif
+   MFEM_GPU_CHECK(hipHostFree(ptr));
+#ifdef MFEM_TRACK_HIP_MEM
+   mfem::out << "done." << std::endl;
+#endif
+#endif
+   return ptr;
+}
+
 void* HipMemcpyHtoD(void* dst, const void* src, size_t bytes)
 {
 #ifdef MFEM_USE_HIP
@@ -125,7 +155,7 @@ void* HipMemcpyDtoDAsync(void* dst, const void *src, size_t bytes)
 void* HipMemcpyDtoH(void *dst, const void *src, size_t bytes)
 {
 #ifdef MFEM_USE_HIP
-#ifdef MFEM_TRACK_HPI_MEM
+#ifdef MFEM_TRACK_HIP_MEM
    mfem::out << "HipMemcpyDtoH(): copying " << bytes << " bytes from "
              << src << " to " << dst << " ... " << std::flush;
 #endif

general/hip.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -64,9 +64,15 @@ void* HipMemAlloc(void **d_ptr, size_t bytes);
 /// Allocates managed device memory
 void* HipMallocManaged(void **d_ptr, size_t bytes);
 
+/// Allocates page-locked (pinned) host memory
+void* HipMemAllocHostPinned(void **ptr, size_t bytes);
+
 /// Frees device memory
 void* HipMemFree(void *d_ptr);
 
+/// Frees page-locked (pinned) host memory and returns destination ptr.
+void* HipMemFreeHostPinned(void *ptr);
+
 /// Copies memory from Host to Device
 void* HipMemcpyHtoD(void *d_dst, const void *h_src, size_t bytes);
 

general/mem_manager.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -62,38 +62,39 @@ MemoryType GetMemoryType(MemoryClass mc)
    return MemoryType::HOST;
 }
 
-// We want to keep this pairs, as it is checked in MFEM_VERIFY_TYPES
-MemoryType MemoryManager::GetDualMemoryType_(MemoryType mt)
-{
-   switch (mt)
-   {
-      case MemoryType::HOST:           return MemoryType::DEVICE;
-      case MemoryType::HOST_32:        return MemoryType::DEVICE;
-      case MemoryType::HOST_64:        return MemoryType::DEVICE;
-      case MemoryType::HOST_DEBUG:     return MemoryType::DEVICE_DEBUG;
-      case MemoryType::HOST_UMPIRE:    return MemoryType::DEVICE_UMPIRE;
-      case MemoryType::MANAGED:        return MemoryType::MANAGED;
-      case MemoryType::DEVICE:         return MemoryType::HOST;
-      case MemoryType::DEVICE_DEBUG:   return MemoryType::HOST_DEBUG;
-      case MemoryType::DEVICE_UMPIRE:  return MemoryType::HOST_UMPIRE;
-      default: mfem_error("Unknown memory type!");
-   }
-   MFEM_VERIFY(false,"");
-   return MemoryType::HOST;
-}
 
 static void MFEM_VERIFY_TYPES(const MemoryType h_mt, const MemoryType d_mt)
 {
-   MFEM_ASSERT(IsHostMemory(h_mt),"");
-   MFEM_ASSERT(IsDeviceMemory(d_mt),"");
+   MFEM_VERIFY(IsHostMemory(h_mt), "h_mt = " << (int)h_mt);
+   MFEM_VERIFY(IsDeviceMemory(d_mt) || d_mt == MemoryType::DEFAULT,
+               "d_mt = " << (int)d_mt);
+   // If h_mt == MemoryType::HOST_DEBUG, then d_mt == MemoryType::DEVICE_DEBUG
+   //                                      or d_mt == MemoryType::DEFAULT
+   MFEM_VERIFY(h_mt != MemoryType::HOST_DEBUG ||
+               d_mt == MemoryType::DEVICE_DEBUG ||
+               d_mt == MemoryType::DEFAULT,
+               "d_mt = " << MemoryTypeName[(int)d_mt]);
+   // If d_mt == MemoryType::DEVICE_DEBUG, then h_mt == MemoryType::HOST_DEBUG
+   MFEM_VERIFY(d_mt != MemoryType::DEVICE_DEBUG ||
+               h_mt == MemoryType::HOST_DEBUG,
+               "h_mt = " << MemoryTypeName[(int)h_mt]);
+#if 0
    const bool sync =
+      (h_mt == MemoryType::HOST_PINNED && d_mt == MemoryType::DEVICE) ||
+      (h_mt == MemoryType::HOST_PINNED && d_mt == MemoryType::DEVICE_UMPIRE) ||
+      (h_mt == MemoryType::HOST_PINNED && d_mt == MemoryType::DEVICE_UMPIRE_2) ||
+      (h_mt == MemoryType::HOST_UMPIRE && d_mt == MemoryType::DEVICE) ||
       (h_mt == MemoryType::HOST_UMPIRE && d_mt == MemoryType::DEVICE_UMPIRE) ||
+      (h_mt == MemoryType::HOST_UMPIRE && d_mt == MemoryType::DEVICE_UMPIRE_2) ||
       (h_mt == MemoryType::HOST_DEBUG && d_mt == MemoryType::DEVICE_DEBUG) ||
       (h_mt == MemoryType::MANAGED && d_mt == MemoryType::MANAGED) ||
       (h_mt == MemoryType::HOST_64 && d_mt == MemoryType::DEVICE) ||
       (h_mt == MemoryType::HOST_32 && d_mt == MemoryType::DEVICE) ||
-      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE);
+      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE) ||
+      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE_UMPIRE) ||
+      (h_mt == MemoryType::HOST && d_mt == MemoryType::DEVICE_UMPIRE_2);
    MFEM_VERIFY(sync, "");
+#endif
 }
 
 MemoryClass operator*(MemoryClass mc1, MemoryClass mc2)
@@ -135,9 +136,12 @@ struct Memory
    void *const h_ptr;
    void *d_ptr;
    const size_t bytes;
-   const MemoryType h_mt, d_mt;
+   const MemoryType h_mt;
+   MemoryType d_mt;
+   mutable bool h_rw, d_rw;
    Memory(void *p, size_t b, MemoryType h, MemoryType d):
-      h_ptr(p), d_ptr(nullptr), bytes(b), h_mt(h), d_mt(d) { }
+      h_ptr(p), d_ptr(nullptr), bytes(b), h_mt(h), d_mt(d),
+      h_rw(true), d_rw(true) { }
 };
 
 /// Alias class that holds the base memory region and the offset
@@ -173,8 +177,8 @@ public:
    virtual ~HostMemorySpace() { }
    virtual void Alloc(void **ptr, size_t bytes) { *ptr = std::malloc(bytes); }
    virtual void Dealloc(void *ptr) { std::free(ptr); }
-   virtual void Protect(const void*, size_t) { }
-   virtual void Unprotect(const void*, size_t) { }
+   virtual void Protect(const Memory&, size_t) { }
+   virtual void Unprotect(const Memory&, size_t) { }
    virtual void AliasProtect(const void*, size_t) { }
    virtual void AliasUnprotect(const void*, size_t) { }
 };
@@ -352,8 +356,10 @@ public:
    MmuHostMemorySpace(): HostMemorySpace() { MmuInit(); }
    void Alloc(void **ptr, size_t bytes) { MmuAlloc(ptr, bytes); }
    void Dealloc(void *ptr) { MmuDealloc(ptr, maps->memories.at(ptr).bytes); }
-   void Protect(const void *ptr, size_t bytes) { MmuProtect(ptr, bytes); }
-   void Unprotect(const void *ptr, size_t bytes) { MmuAllow(ptr, bytes); }
+   void Protect(const Memory& mem, size_t bytes)
+   { if (mem.h_rw) { mem.h_rw = false; MmuProtect(mem.h_ptr, bytes); } }
+   void Unprotect(const Memory &mem, size_t bytes)
+   { if (!mem.h_rw) { mem.h_rw = true; MmuAllow(mem.h_ptr, bytes); } }
    /// Aliases need to be restricted during protection
    void AliasProtect(const void *ptr, size_t bytes)
    { MmuProtect(MmuAddrR(ptr), MmuLengthR(ptr, bytes)); }
@@ -400,6 +406,31 @@ public:
    { return CuMemcpyDtoH(dst, src, bytes); }
 };
 
+/// The CUDA/HIP page-locked host memory space
+class HostPinnedMemorySpace: public HostMemorySpace
+{
+public:
+   HostPinnedMemorySpace(): HostMemorySpace() { }
+   void Alloc(void ** ptr, size_t bytes) override
+   {
+#ifdef MFEM_USE_CUDA
+      CuMemAllocHostPinned(ptr, bytes);
+#endif
+#ifdef MFEM_USE_HIP
+      HipMemAllocHostPinned(ptr, bytes);
+#endif
+   }
+   void Dealloc(void *ptr) override
+   {
+#ifdef MFEM_USE_CUDA
+      CuMemFreeHostPinned(ptr);
+#endif
+#ifdef MFEM_USE_HIP
+      HipMemFreeHostPinned(ptr);
+#endif
+   }
+};
+
 /// The HIP device memory space
 class HipDeviceMemorySpace: public DeviceMemorySpace
 {
@@ -442,8 +473,10 @@ public:
    MmuDeviceMemorySpace(): DeviceMemorySpace() { }
    void Alloc(Memory &m) { MmuAlloc(&m.d_ptr, m.bytes); }
    void Dealloc(Memory &m) { MmuDealloc(m.d_ptr, m.bytes); }
-   void Protect(const Memory &m) { MmuProtect(m.d_ptr, m.bytes); }
-   void Unprotect(const Memory &m) { MmuAllow(m.d_ptr, m.bytes); }
+   void Protect(const Memory &m)
+   { if (m.d_rw) { m.d_rw = false; MmuProtect(m.d_ptr, m.bytes); } }
+   void Unprotect(const Memory &m)
+   { if (!m.d_rw) { m.d_rw = true; MmuAllow(m.d_ptr, m.bytes); } }
    /// Aliases need to be restricted during protection
    void AliasProtect(const void *ptr, size_t bytes)
    { MmuProtect(MmuAddrR(ptr), MmuLengthR(ptr, bytes)); }
@@ -458,54 +491,64 @@ public:
    { return std::memcpy(dst, src, bytes); }
 };
 
-#ifndef MFEM_USE_UMPIRE
-class UmpireHostMemorySpace : public NoHostMemorySpace { };
-class UmpireDeviceMemorySpace : public NoDeviceMemorySpace { };
-#else
+#ifdef MFEM_USE_UMPIRE
+class UmpireMemorySpace
+{
+protected:
+   umpire::ResourceManager &rm;
+   umpire::Allocator allocator;
+   bool owns_allocator{false};
+
+public:
+   // TODO: this only releases unused memory
+   virtual ~UmpireMemorySpace() { if (owns_allocator) { allocator.release(); } }
+   UmpireMemorySpace(const char * name, const char * space)
+      : rm(umpire::ResourceManager::getInstance())
+   {
+      if (!rm.isAllocator(name))
+      {
+         allocator = rm.makeAllocator<umpire::strategy::DynamicPool>(
+                        name, rm.getAllocator(space));
+         owns_allocator = true;
+      }
+      else
+      {
+         allocator = rm.getAllocator(name);
+         owns_allocator = false;
+      }
+   }
+};
+
 /// The Umpire host memory space
-class UmpireHostMemorySpace : public HostMemorySpace
+class UmpireHostMemorySpace : public HostMemorySpace, public UmpireMemorySpace
 {
 private:
-   const char *name;
-   umpire::ResourceManager &rm;
-   umpire::Allocator h_allocator;
    umpire::strategy::AllocationStrategy *strat;
 public:
-   ~UmpireHostMemorySpace() { h_allocator.release(); }
-   UmpireHostMemorySpace():
-      HostMemorySpace(),
-      name(mm.GetUmpireAllocatorHostName()),
-      rm(umpire::ResourceManager::getInstance()),
-      h_allocator(rm.isAllocator(name)? rm.getAllocator(name):
-                  rm.makeAllocator<umpire::strategy::DynamicPool>
-                  (name, rm.getAllocator("HOST"))),
-      strat(h_allocator.getAllocationStrategy()) { }
-   void Alloc(void **ptr, size_t bytes) { *ptr = h_allocator.allocate(bytes); }
-   void Dealloc(void *ptr) { h_allocator.deallocate(ptr); }
+   UmpireHostMemorySpace(const char * name)
+      : HostMemorySpace(),
+        UmpireMemorySpace(name, "HOST"),
+        strat(allocator.getAllocationStrategy()) {}
+   void Alloc(void **ptr, size_t bytes) override
+   { *ptr = allocator.allocate(bytes); }
+   void Dealloc(void *ptr) override { allocator.deallocate(ptr); }
    void Insert(void *ptr, size_t bytes)
    { rm.registerAllocation(ptr, {ptr, bytes, strat}); }
 };
 
 /// The Umpire device memory space
-#ifdef MFEM_USE_CUDA
-class UmpireDeviceMemorySpace : public DeviceMemorySpace
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
+class UmpireDeviceMemorySpace : public DeviceMemorySpace,
+   public UmpireMemorySpace
 {
-private:
-   const char *name;
-   umpire::ResourceManager &rm;
-   umpire::Allocator d_allocator;
 public:
-   ~UmpireDeviceMemorySpace() { d_allocator.release(); }
-   UmpireDeviceMemorySpace():
-      DeviceMemorySpace(),
-      name(mm.GetUmpireAllocatorDeviceName()),
-      rm(umpire::ResourceManager::getInstance()),
-      d_allocator(rm.isAllocator(name)? rm.getAllocator(name):
-                  rm.makeAllocator<umpire::strategy::DynamicPool>
-                  (name, rm.getAllocator("DEVICE"))) { }
-   void Alloc(Memory &base) { base.d_ptr = d_allocator.allocate(base.bytes); }
-   void Dealloc(Memory &base) { d_allocator.deallocate(base.d_ptr); }
-   void *HtoD(void *dst, const void *src, size_t bytes)
+   UmpireDeviceMemorySpace(const char * name)
+      : DeviceMemorySpace(),
+        UmpireMemorySpace(name, "DEVICE") {}
+   void Alloc(Memory &base) override
+   { base.d_ptr = allocator.allocate(base.bytes); }
+   void Dealloc(Memory &base) override { rm.deallocate(base.d_ptr); }
+   void *HtoD(void *dst, const void *src, size_t bytes) override
    {
 #ifdef MFEM_USE_CUDA
       return CuMemcpyHtoD(dst, src, bytes);
@@ -515,7 +558,7 @@ public:
 #endif
       //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
-   void *DtoD(void* dst, const void* src, size_t bytes)
+   void *DtoD(void* dst, const void* src, size_t bytes) override
    {
 #ifdef MFEM_USE_CUDA
       return CuMemcpyDtoD(dst, src, bytes);
@@ -525,7 +568,7 @@ public:
 #endif
       //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
-   void *DtoH(void *dst, const void *src, size_t bytes)
+   void *DtoH(void *dst, const void *src, size_t bytes) override
    {
 #ifdef MFEM_USE_CUDA
       return CuMemcpyDtoH(dst, src, bytes);
@@ -537,8 +580,12 @@ public:
    }
 };
 #else
-class UmpireDeviceMemorySpace : public NoDeviceMemorySpace { };
-#endif // MFEM_USE_CUDA
+class UmpireDeviceMemorySpace : public NoDeviceMemorySpace
+{
+public:
+   UmpireDeviceMemorySpace(const char * /*unused*/) {}
+};
+#endif // MFEM_USE_CUDA || MFEM_USE_HIP
 #endif // MFEM_USE_UMPIRE
 
 /// Memory space controller class
@@ -568,7 +615,7 @@ public:
       host[static_cast<int>(MT::HOST_64)] = new Aligned64HostMemorySpace();
       // HOST_DEBUG is delayed, as it reroutes signals
       host[static_cast<int>(MT::HOST_DEBUG)] = nullptr;
-      host[static_cast<int>(MT::HOST_UMPIRE)] = new UmpireHostMemorySpace();
+      host[static_cast<int>(MT::HOST_UMPIRE)] = nullptr;
       host[static_cast<int>(MT::MANAGED)] = new UvmHostMemorySpace();
 
       // Filling the device memory backends, shifting with the device size
@@ -578,6 +625,7 @@ public:
       device[static_cast<int>(MemoryType::DEVICE)-shift] = nullptr;
       device[static_cast<int>(MT::DEVICE_DEBUG)-shift] = nullptr;
       device[static_cast<int>(MT::DEVICE_UMPIRE)-shift] = nullptr;
+      device[static_cast<int>(MT::DEVICE_UMPIRE_2)-shift] = nullptr;
    }
 
    HostMemorySpace* Host(const MemoryType mt)
@@ -610,8 +658,19 @@ public:
 private:
    HostMemorySpace* NewHostCtrl(const MemoryType mt)
    {
-      if (mt == MT::HOST_DEBUG) { return new MmuHostMemorySpace(); }
-      MFEM_ABORT("Unknown host memory controller!");
+      switch (mt)
+      {
+         case MT::HOST_DEBUG: return new MmuHostMemorySpace();
+#ifdef MFEM_USE_UMPIRE
+         case MT::HOST_UMPIRE:
+            return new UmpireHostMemorySpace(
+                      MemoryManager::GetUmpireHostAllocatorName());
+#else
+         case MT::HOST_UMPIRE: return new NoHostMemorySpace();
+#endif
+         case MT::HOST_PINNED: return new HostPinnedMemorySpace();
+         default: MFEM_ABORT("Unknown host memory controller!");
+      }
       return nullptr;
    }
 
@@ -619,7 +678,17 @@ private:
    {
       switch (mt)
       {
-         case MT::DEVICE_UMPIRE: return new UmpireDeviceMemorySpace();
+#ifdef MFEM_USE_UMPIRE
+         case MT::DEVICE_UMPIRE:
+            return new UmpireDeviceMemorySpace(
+                      MemoryManager::GetUmpireDeviceAllocatorName());
+         case MT::DEVICE_UMPIRE_2:
+            return new UmpireDeviceMemorySpace(
+                      MemoryManager::GetUmpireDevice2AllocatorName());
+#else
+         case MT::DEVICE_UMPIRE: return new NoDeviceMemorySpace();
+         case MT::DEVICE_UMPIRE_2: return new NoDeviceMemorySpace();
+#endif
          case MT::DEVICE_DEBUG: return new MmuDeviceMemorySpace();
          case MT::DEVICE:
          {
@@ -646,20 +715,50 @@ void *MemoryManager::New_(void *h_tmp, size_t bytes, MemoryType mt,
                           unsigned &flags)
 {
    MFEM_ASSERT(exists, "Internal error!");
-   MFEM_ASSERT(mt != MemoryType::HOST, "Internal error!");
-   const bool is_host_mem = IsHostMemory(mt);
-   const MemType dual_mt = GetDualMemoryType_(mt);
-   const MemType h_mt = is_host_mem ? mt : dual_mt;
-   const MemType d_mt = is_host_mem ? dual_mt : mt;
-   MFEM_VERIFY_TYPES(h_mt, d_mt);
-   void *h_ptr = h_tmp;
+   if (IsHostMemory(mt))
+   {
+      MFEM_ASSERT(mt != MemoryType::HOST && h_tmp == nullptr,
+                  "Internal error!");
+      // d_mt = MemoryType::DEFAULT means d_mt = GetDualMemoryType(h_mt),
+      // evaluated at the time when the device pointer is allocated, see
+      // GetDevicePtr() and GetAliasDevicePtr()
+      const MemoryType d_mt = MemoryType::DEFAULT;
+      // We rely on the next call using lazy dev alloc
+      return New_(h_tmp, bytes, mt, d_mt, Mem::VALID_HOST, flags);
+   }
+   else
+   {
+      const MemoryType h_mt = GetDualMemoryType(mt);
+      return New_(h_tmp, bytes, h_mt, mt, Mem::VALID_DEVICE, flags);
+   }
+}
+
+void *MemoryManager::New_(void *h_tmp, size_t bytes, MemoryType h_mt,
+                          MemoryType d_mt, unsigned valid_flags,
+                          unsigned &flags)
+{
+   MFEM_ASSERT(exists, "Internal error!");
+   MFEM_ASSERT(IsHostMemory(h_mt), "h_mt must be host type");
+   MFEM_ASSERT(IsDeviceMemory(d_mt) || d_mt == h_mt ||
+               d_mt == MemoryType::DEFAULT,
+               "d_mt must be device type, the same is h_mt, or DEFAULT");
+   MFEM_ASSERT((h_mt != MemoryType::HOST || h_tmp != nullptr) &&
+               (h_mt == MemoryType::HOST || h_tmp == nullptr),
+               "Internal error");
+   MFEM_ASSERT((valid_flags & ~(Mem::VALID_HOST | Mem::VALID_DEVICE)) == 0,
+               "Internal error");
+   void *h_ptr;
    if (h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
-   flags = Mem::REGISTERED;
-   flags |= Mem::OWNS_INTERNAL | Mem::OWNS_HOST | Mem::OWNS_DEVICE;
-   flags |= is_host_mem ? Mem::VALID_HOST : Mem::VALID_DEVICE;
-   if (is_host_mem) { mm.Insert(h_ptr, bytes, h_mt, d_mt); }
-   else { mm.InsertDevice(nullptr, h_ptr, bytes, h_mt, d_mt); }
+   else { h_ptr = h_tmp; }
+   flags = Mem::REGISTERED | Mem::OWNS_INTERNAL | Mem::OWNS_HOST |
+           Mem::OWNS_DEVICE | valid_flags;
+   // The other New_() method relies on this lazy allocation behavior.
+   mm.Insert(h_ptr, bytes, h_mt, d_mt); // lazy dev alloc
+   // mm.InsertDevice(nullptr, h_ptr, bytes, h_mt, d_mt); // non-lazy dev alloc
+
+   // MFEM_VERIFY_TYPES(h_mt, mt); // done by mm.Insert() above
    CheckHostMemoryType_(h_mt, h_ptr);
+
    return h_ptr;
 }
 
@@ -671,9 +770,12 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
    MFEM_ASSERT(exists, "Internal error!");
    MFEM_ASSERT(!alias, "Cannot register an alias!");
    const bool is_host_mem = IsHostMemory(mt);
-   const MemType dual_mt = GetDualMemoryType_(mt);
-   const MemType h_mt = is_host_mem ? mt : dual_mt;
-   const MemType d_mt = is_host_mem ? dual_mt : mt;
+   const MemType h_mt = is_host_mem ? mt : GetDualMemoryType(mt);
+   const MemType d_mt = is_host_mem ? MemoryType::DEFAULT : mt;
+   // d_mt = MemoryType::DEFAULT means d_mt = GetDualMemoryType(h_mt),
+   // evaluated at the time when the device pointer is allocated, see
+   // GetDevicePtr() and GetAliasDevicePtr()
+
    MFEM_VERIFY_TYPES(h_mt, d_mt);
 
    if (ptr == nullptr && h_tmp == nullptr)
@@ -694,17 +796,42 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
    }
    else // DEVICE TYPES
    {
-      h_ptr = h_tmp;
-      if (own && h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
+      MFEM_VERIFY(ptr, "cannot register NULL device pointer");
+      if (h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
+      else { h_ptr = h_tmp; }
       mm.InsertDevice(ptr, h_ptr, bytes, h_mt, d_mt);
       flags = own ? flags | Mem::OWNS_DEVICE : flags & ~Mem::OWNS_DEVICE;
-      flags = own ? flags | Mem::OWNS_HOST   : flags & ~Mem::OWNS_HOST;
-      flags |= Mem::VALID_DEVICE;
+      flags |= (Mem::OWNS_HOST | Mem::VALID_DEVICE);
    }
    CheckHostMemoryType_(h_mt, h_ptr);
    return h_ptr;
 }
 
+void MemoryManager::Register_(void *h_ptr, void *d_ptr, size_t bytes,
+                              MemoryType h_mt, MemoryType d_mt,
+                              bool own, bool alias, unsigned &flags)
+{
+   MFEM_CONTRACT_VAR(alias);
+   MFEM_ASSERT(exists, "Internal error!");
+   MFEM_ASSERT(!alias, "Cannot register an alias!");
+   MFEM_VERIFY_TYPES(h_mt, d_mt);
+
+   if (h_ptr == nullptr && d_ptr == nullptr)
+   {
+      MFEM_VERIFY(bytes == 0, "internal error");
+      return;
+   }
+
+   flags |= Mem::REGISTERED | Mem::OWNS_INTERNAL;
+
+   mm.InsertDevice(d_ptr, h_ptr, bytes, h_mt, d_mt);
+   flags = (own ? flags | (Mem::OWNS_HOST | Mem::OWNS_DEVICE) :
+            flags & ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE)) |
+           Mem::VALID_HOST;
+
+   CheckHostMemoryType_(h_mt, h_ptr);
+}
+
 void MemoryManager::Alias_(void *base_h_ptr, size_t offset, size_t bytes,
                            unsigned base_flags, unsigned &flags)
 {
@@ -714,6 +841,34 @@ void MemoryManager::Alias_(void *base_h_ptr, size_t offset, size_t bytes,
            ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE);
 }
 
+void MemoryManager::SetDeviceMemoryType_(void *h_ptr, unsigned flags,
+                                         MemoryType d_mt)
+{
+   MFEM_VERIFY(h_ptr, "cannot set the device memory type: Memory is empty!");
+   if (!(flags & Mem::ALIAS))
+   {
+      auto mem_iter = maps->memories.find(h_ptr);
+      MFEM_VERIFY(mem_iter != maps->memories.end(), "internal error");
+      internal::Memory &mem = mem_iter->second;
+      if (mem.d_mt == d_mt) { return; }
+      MFEM_VERIFY(mem.d_ptr == nullptr, "cannot set the device memory type:"
+                  " device memory is allocated!");
+      mem.d_mt = d_mt;
+   }
+   else
+   {
+      auto alias_iter = maps->aliases.find(h_ptr);
+      MFEM_VERIFY(alias_iter != maps->aliases.end(), "internal error");
+      internal::Alias &alias = alias_iter->second;
+      internal::Memory &base_mem = *alias.mem;
+      if (base_mem.d_mt == d_mt) { return; }
+      MFEM_VERIFY(base_mem.d_ptr == nullptr,
+                  "cannot set the device memory type:"
+                  " alias' base device memory is allocated!");
+      base_mem.d_mt = d_mt;
+   }
+}
+
 MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType mt, unsigned flags)
 {
    const bool alias = flags & Mem::ALIAS;
@@ -747,6 +902,16 @@ MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType mt, unsigned flags)
    return mt;
 }
 
+void MemoryManager::DeleteDevice_(void *h_ptr, unsigned & flags)
+{
+   const bool owns_device = flags & Mem::OWNS_DEVICE;
+   if (owns_device)
+   {
+      mm.EraseDevice(h_ptr);
+      flags = (flags | Mem::VALID_HOST) & ~Mem::VALID_DEVICE;
+   }
+}
+
 bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
                                       MemoryType h_mt, size_t bytes,
                                       unsigned flags)
@@ -760,11 +925,12 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
    const bool known = mm.IsKnown(h_ptr);
    const bool alias = mm.IsAlias(h_ptr);
    const bool check = known || ((flags & Mem::ALIAS) && alias);
-   MFEM_VERIFY(check,"");
+   MFEM_VERIFY(check, "Unknown host pointer: " << h_ptr);
    const internal::Memory &mem =
       (flags & Mem::ALIAS) ?
       *maps->aliases.at(h_ptr).mem : maps->memories.at(h_ptr);
-   const MemoryType &d_mt = mem.d_mt;
+   MemoryType d_mt = mem.d_mt;
+   if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
    switch (mc)
    {
       case MemoryClass::HOST_32:
@@ -783,6 +949,7 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
          MFEM_VERIFY(d_mt == MemoryType::DEVICE ||
                      d_mt == MemoryType::DEVICE_DEBUG ||
                      d_mt == MemoryType::DEVICE_UMPIRE ||
+                     d_mt == MemoryType::DEVICE_UMPIRE_2 ||
                      d_mt == MemoryType::MANAGED,"");
          return true;
       }
@@ -864,7 +1031,6 @@ void *MemoryManager::Write_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
       if (flags & Mem::ALIAS)
       { return mm.GetAliasDevicePtr(h_ptr, bytes, false); }
       else { return mm.GetDevicePtr(h_ptr, bytes, false); }
-
    }
 }
 
@@ -969,11 +1135,8 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
       {
          if (dst_h_ptr != src_d_ptr && bytes != 0)
          {
-            internal::Memory &dst_h_base = maps->memories.at(dst_h_ptr);
             internal::Memory &src_d_base = maps->memories.at(src_d_ptr);
-            MemoryType dst_h_mt = dst_h_base.h_mt;
             MemoryType src_d_mt = src_d_base.d_mt;
-            ctrl->Host(dst_h_mt)->Unprotect(dst_h_ptr, bytes);
             ctrl->Device(src_d_mt)->DtoH(dst_h_ptr, src_d_ptr, bytes);
          }
       }
@@ -1098,7 +1261,7 @@ void MemoryManager::Insert(void *h_ptr, size_t bytes,
 void MemoryManager::InsertDevice(void *d_ptr, void *h_ptr, size_t bytes,
                                  MemoryType h_mt, MemoryType d_mt)
 {
-   MFEM_VERIFY_TYPES(h_mt, d_mt);
+   // MFEM_VERIFY_TYPES(h_mt, d_mt); // done by Insert() below
    MFEM_ASSERT(h_ptr != NULL, "internal error");
    Insert(h_ptr, bytes, h_mt, d_mt);
    internal::Memory &mem = maps->memories.at(h_ptr);
@@ -1151,6 +1314,22 @@ void MemoryManager::Erase(void *h_ptr, bool free_dev_ptr)
    maps->memories.erase(mem_map_iter);
 }
 
+void MemoryManager::EraseDevice(void *h_ptr)
+{
+   if (!h_ptr) { return; }
+   auto mem_map_iter = maps->memories.find(h_ptr);
+   if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
+   auto it = maps->aliases.find(h_ptr);
+   if (it != maps->aliases.end())
+   {
+      fprintf(stderr, "count = %lu\n", it->second.counter);
+      mfem_error("cannot delete aliased obj!");
+   }
+   internal::Memory &mem = mem_map_iter->second;
+   if (mem.d_ptr) { ctrl->Device(mem.d_mt)->Dealloc(mem);}
+   mem.d_ptr = nullptr;
+}
+
 void MemoryManager::EraseAlias(void *alias_ptr)
 {
    if (!alias_ptr) { return; }
@@ -1171,16 +1350,21 @@ void *MemoryManager::GetDevicePtr(const void *h_ptr, size_t bytes,
    }
    internal::Memory &mem = maps->memories.at(h_ptr);
    const MemoryType &h_mt = mem.h_mt;
-   const MemoryType &d_mt = mem.d_mt;
+   MemoryType &d_mt = mem.d_mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
-   if (!mem.d_ptr) { ctrl->Device(d_mt)->Alloc(mem); }
+   if (!mem.d_ptr)
+   {
+      if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
+      ctrl->Device(d_mt)->Alloc(mem);
+   }
+   // Aliases might have done some protections
    ctrl->Device(d_mt)->Unprotect(mem);
    if (copy_data)
    {
       MFEM_ASSERT(bytes <= mem.bytes, "invalid copy size");
       ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes);
    }
-   ctrl->Host(h_mt)->Protect(h_ptr, bytes);
+   ctrl->Host(h_mt)->Protect(mem, bytes);
    return mem.d_ptr;
 }
 
@@ -1199,13 +1383,18 @@ void *MemoryManager::GetAliasDevicePtr(const void *alias_ptr, size_t bytes,
    const size_t offset = alias.offset;
    internal::Memory &mem = *alias.mem;
    const MemoryType &h_mt = mem.h_mt;
-   const MemoryType &d_mt = mem.d_mt;
+   MemoryType &d_mt = mem.d_mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
-   if (!mem.d_ptr) { ctrl->Device(d_mt)->Alloc(mem); }
+   if (!mem.d_ptr)
+   {
+      if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
+      ctrl->Device(d_mt)->Alloc(mem);
+   }
    void *alias_h_ptr = static_cast<char*>(mem.h_ptr) + offset;
    void *alias_d_ptr = static_cast<char*>(mem.d_ptr) + offset;
    MFEM_ASSERT(alias_h_ptr == alias_ptr, "internal error");
    MFEM_ASSERT(bytes <= alias.bytes, "internal error");
+   mem.d_rw = false;
    ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes);
    ctrl->Host(h_mt)->AliasUnprotect(alias_ptr, bytes);
    if (copy) { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
@@ -1221,8 +1410,8 @@ void *MemoryManager::GetHostPtr(const void *ptr, size_t bytes, bool copy)
    const MemoryType &h_mt = mem.h_mt;
    const MemoryType &d_mt = mem.d_mt;
    MFEM_VERIFY_TYPES(h_mt, d_mt);
-   ctrl->Host(h_mt)->Unprotect(mem.h_ptr, bytes);
    // Aliases might have done some protections
+   ctrl->Host(h_mt)->Unprotect(mem, bytes);
    if (mem.d_ptr) { ctrl->Device(d_mt)->Unprotect(mem); }
    if (copy && mem.d_ptr) { ctrl->Device(d_mt)->DtoH(mem.h_ptr, mem.d_ptr, bytes); }
    if (mem.d_ptr) { ctrl->Device(d_mt)->Protect(mem); }
@@ -1240,6 +1429,7 @@ void *MemoryManager::GetAliasHostPtr(const void *ptr, size_t bytes,
    void *alias_h_ptr = static_cast<char*>(mem->h_ptr) + alias.offset;
    void *alias_d_ptr = static_cast<char*>(mem->d_ptr) + alias.offset;
    MFEM_ASSERT(alias_h_ptr == ptr,  "internal error");
+   mem->h_rw = false;
    ctrl->Host(h_mt)->AliasUnprotect(alias_h_ptr, bytes);
    if (mem->d_ptr) { ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes); }
    if (copy_data && mem->d_ptr)
@@ -1261,23 +1451,48 @@ MemoryManager::MemoryManager() { Init(); }
 
 MemoryManager::~MemoryManager() { if (exists) { Destroy(); } }
 
+void MemoryManager::SetDualMemoryType(MemoryType mt, MemoryType dual_mt)
+{
+   MFEM_VERIFY(!configured, "changing the dual MemoryTypes is not allowed after"
+               " MemoryManager configuration!");
+   UpdateDualMemoryType(mt, dual_mt);
+}
+
+void MemoryManager::UpdateDualMemoryType(MemoryType mt, MemoryType dual_mt)
+{
+   MFEM_VERIFY((int)mt < MemoryTypeSize,
+               "invalid MemoryType, mt = " << (int)mt);
+   MFEM_VERIFY((int)dual_mt < MemoryTypeSize,
+               "invalid dual MemoryType, dual_mt = " << (int)dual_mt);
+
+   if ((IsHostMemory(mt) && IsDeviceMemory(dual_mt)) ||
+       (IsDeviceMemory(mt) && IsHostMemory(dual_mt)))
+   {
+      dual_map[(int)mt] = dual_mt;
+   }
+   else
+   {
+      // mt + dual_mt is not a pair of host + device types: this is only allowed
+      // when mt == dual_mt and mt is a host type; in this case we do not
+      // actually update the dual
+      MFEM_VERIFY(mt == dual_mt && IsHostMemory(mt),
+                  "invalid (mt, dual_mt) pair: ("
+                  << MemoryTypeName[(int)mt] << ", "
+                  << MemoryTypeName[(int)dual_mt] << ')');
+   }
+}
+
 void MemoryManager::Configure(const MemoryType host_mt,
                               const MemoryType device_mt)
 {
+   MemoryManager::UpdateDualMemoryType(host_mt, device_mt);
+   MemoryManager::UpdateDualMemoryType(device_mt, host_mt);
    Init();
    host_mem_type = host_mt;
    device_mem_type = device_mt;
+   configured = true;
 }
 
-#ifdef MFEM_USE_UMPIRE
-void MemoryManager::SetUmpireAllocatorNames(const char *h_name,
-                                            const char *d_name)
-{
-   h_umpire_name = h_name;
-   d_umpire_name = d_name;
-}
-#endif
-
 void MemoryManager::Destroy()
 {
    MFEM_VERIFY(exists, "MemoryManager has already been destroyed!");
@@ -1293,6 +1508,7 @@ void MemoryManager::Destroy()
    host_mem_type = MemoryType::HOST;
    device_mem_type = MemoryType::HOST;
    exists = false;
+   configured = false;
 }
 
 void MemoryManager::RegisterCheck(void *ptr)
@@ -1379,18 +1595,36 @@ void MemoryManager::CheckHostMemoryType_(MemoryType h_mt, void *h_ptr)
 MemoryManager mm;
 
 bool MemoryManager::exists = false;
-
-#ifdef MFEM_USE_UMPIRE
-const char* MemoryManager::h_umpire_name = "HOST";
-const char* MemoryManager::d_umpire_name = "DEVICE";
-#endif
+bool MemoryManager::configured = false;
 
 MemoryType MemoryManager::host_mem_type = MemoryType::HOST;
 MemoryType MemoryManager::device_mem_type = MemoryType::HOST;
 
+MemoryType MemoryManager::dual_map[MemoryTypeSize] =
+{
+   /* HOST            */  MemoryType::DEVICE,
+   /* HOST_32         */  MemoryType::DEVICE,
+   /* HOST_64         */  MemoryType::DEVICE,
+   /* HOST_DEBUG      */  MemoryType::DEVICE_DEBUG,
+   /* HOST_UMPIRE     */  MemoryType::DEVICE_UMPIRE,
+   /* HOST_PINNED     */  MemoryType::DEVICE,
+   /* MANAGED         */  MemoryType::MANAGED,
+   /* DEVICE          */  MemoryType::HOST,
+   /* DEVICE_DEBUG    */  MemoryType::HOST_DEBUG,
+   /* DEVICE_UMPIRE   */  MemoryType::HOST_UMPIRE,
+   /* DEVICE_UMPIRE_2 */  MemoryType::HOST_UMPIRE
+};
+
+#ifdef MFEM_USE_UMPIRE
+const char * MemoryManager::h_umpire_name = "MFEM_HOST";
+const char * MemoryManager::d_umpire_name = "MFEM_DEVICE";
+const char * MemoryManager::d_umpire_2_name = "MFEM_DEVICE_2";
+#endif
+
+
 const char *MemoryTypeName[MemoryTypeSize] =
 {
-   "host-std", "host-32", "host-64", "host-debug", "host-umpire",
+   "host-std", "host-32", "host-64", "host-debug", "host-umpire", "host-pinned",
 #if defined(MFEM_USE_CUDA)
    "cuda-uvm",
    "cuda",
@@ -1403,11 +1637,14 @@ const char *MemoryTypeName[MemoryTypeSize] =
 #endif
    "device-debug",
 #if defined(MFEM_USE_CUDA)
-   "cuda-umpire"
+   "cuda-umpire",
+   "cuda-umpire-2",
 #elif defined(MFEM_USE_HIP)
-   "hip-umpire"
+   "hip-umpire",
+   "hip-umpire-2",
 #else
-   "device-umpire"
+   "device-umpire",
+   "device-umpire-2",
 #endif
 };
 

general/mem_manager.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -31,14 +31,26 @@ enum class MemoryType
    HOST_32,        ///< Host memory; aligned at 32 bytes
    HOST_64,        ///< Host memory; aligned at 64 bytes
    HOST_DEBUG,     ///< Host memory; allocated from a "host-debug" pool
-   HOST_UMPIRE,    ///< Host memory; using Umpire
+   HOST_UMPIRE,    /**< Host memory; using an Umpire allocator which can be set
+                        with MemoryManager::SetUmpireHostAllocatorName */
+   HOST_PINNED,    ///< Host memory: pinned (page-locked)
    MANAGED,        /**< Managed memory; using CUDA or HIP *MallocManaged
                         and *Free */
    DEVICE,         ///< Device memory; using CUDA or HIP *Malloc and *Free
    DEVICE_DEBUG,   /**< Pseudo-device memory; allocated on host from a
                         "device-debug" pool */
-   DEVICE_UMPIRE,  ///< Device memory; using Umpire
-   SIZE            ///< Number of host and device memory types
+   DEVICE_UMPIRE,  /**< Device memory; using an Umpire allocator which can be
+                        set with MemoryManager::SetUmpireDeviceAllocatorName */
+   DEVICE_UMPIRE_2, /**< Device memory; using a second Umpire allocator settable
+                         with MemoryManager::SetUmpireDevice2AllocatorName */
+   SIZE,           ///< Number of host and device memory types
+
+   PRESERVE,       /**< Pseudo-MemoryType used as default value for MemoryType
+                        parameters to request preservation of existing
+                        MemoryType, e.g. in copy constructors. */
+   DEFAULT         /**< Pseudo-MemoryType used as default value for MemoryType
+                        parameters to request the use of the default host or
+                        device MemoryType. */
 };
 
 /// Static casts to 'int' and sizes of some useful memory types.
@@ -58,16 +70,22 @@ extern const char *MemoryTypeName[MemoryTypeSize];
 enum class MemoryClass
 {
    HOST,    /**< Memory types: { HOST, HOST_32, HOST_64, HOST_DEBUG,
-                                 HOST_UMPIRE, MANAGED } */
+                                 HOST_UMPIRE, HOST_PINNED, MANAGED } */
    HOST_32, ///< Memory types: { HOST_32, HOST_64, HOST_DEBUG }
    HOST_64, ///< Memory types: { HOST_64, HOST_DEBUG }
-   DEVICE,  ///< Memory types: { DEVICE, DEVICE_DEBUG, DEVICE_UMPIRE, MANAGED }
+   DEVICE,  /**< Memory types: { DEVICE, DEVICE_DEBUG, DEVICE_UMPIRE,
+                                 DEVICE_UMPIRE_2, MANAGED } */
    MANAGED  ///< Memory types: { MANAGED }
 };
 
 /// Return true if the given memory type is in MemoryClass::HOST.
 inline bool IsHostMemory(MemoryType mt) { return mt <= MemoryType::MANAGED; }
-inline bool IsDeviceMemory(MemoryType mt) { return mt >= MemoryType::MANAGED; }
+
+/// Return true if the given memory type is in MemoryClass::DEVICE
+inline bool IsDeviceMemory(MemoryType mt)
+{
+   return mt >= MemoryType::MANAGED && mt < MemoryType::SIZE;
+}
 
 /// Return a suitable MemoryType for a given MemoryClass.
 MemoryType GetMemoryType(MemoryClass mc);
@@ -177,6 +195,12 @@ public:
        MemoryType is still set as valid. */
    Memory(int size, MemoryType mt) { New(size, mt); }
 
+   /** @brief Allocate memory for @a size entries with the given host MemoryType
+       @a h_mt and device MemoryType @a d_mt. */
+   /** The newly allocated memory is not initialized. The host pointer is set as
+       valid. */
+   Memory(int size, MemoryType h_mt, MemoryType d_mt) { New(size, h_mt, d_mt); }
+
    /** @brief Wrap an externally allocated host pointer, @a ptr with the current
        host memory type returned by MemoryManager::GetHostMemoryType(). */
    /** The parameter @a own determines whether @a ptr will be deleted when the
@@ -261,9 +285,24 @@ public:
    /** The newly allocated memory is not initialized, however the given
        MemoryType is still set as valid.
 
+       When @a mt is a host type, the device MemoryType will be set later, if
+       requested, using the dual type of @a mt, see
+       MemoryManager::GetDualMemoryType().
+
+       When @a mt is a device type, the host MemoryType will be set immediately
+       to be the dual of @a mt, see MemoryManager::GetDualMemoryType().
+
        @note The current memory is NOT deleted by this method. */
    inline void New(int size, MemoryType mt);
 
+   /** @brief Allocate memory for @a size entries with the given host MemoryType
+       @a h_mt and device MemoryType @a d_mt. */
+   /** The newly allocated memory is not initialized. The host pointer is set as
+       valid.
+
+       @note The current memory is NOT deleted by this method. */
+   inline void New(int size, MemoryType h_mt, MemoryType d_mt);
+
    /** @brief Wrap an externally allocated host pointer, @a ptr with the current
        host memory type returned by MemoryManager::GetHostMemoryType(). */
    /** The parameter @a own determines whether @a ptr will be deleted when the
@@ -284,21 +323,18 @@ public:
        @note The current memory is NOT deleted by this method. */
    inline void Wrap(T *ptr, int size, MemoryType mt, bool own);
 
-   /** Wrap an externally pair of allocated pointers, @a h_ptr and @ d_ptr,
+   /** Wrap an externally pair of allocated pointers, @a h_ptr and @a d_ptr,
        of the given host MemoryType @a h_mt. */
    /** The new memory object will have the device MemoryType set as valid.
 
        The given @a h_ptr and @a d_ptr must be allocated appropriately for the
-       given host MemoryType and its associated device MemoryType:
-          - MANAGED => MANAGED,
-          - HOST_DEBUG => DEVICE_DEBUG,
-          - HOST_UMPIRE => DEVICE_UMPIRE,
-          - HOST, HOST_32, HOST_64 => DEVICE.
+       given host MemoryType and its dual device MemoryType as defined by
+       MemoryManager::GetDualMemoryType().
 
        The parameter @a own determines whether both @a h_ptr and @a d_ptr will
        be deleted when the method Delete() is called.
 
-       @note Ownership can also be controled by using the folowing methods:
+       @note Ownership can also be controlled by using the following methods:
          - ClearOwnerFlags,
          - SetHostPtrOwner,
          - SetDevicePtrOwner.
@@ -312,10 +348,29 @@ public:
        @note The current memory is NOT deleted by this method. */
    inline void MakeAlias(const Memory &base, int offset, int size);
 
+   /// Set the device MemoryType to be used by the Memory object.
+   /** If the specified @a d_mt is not a device MemoryType, i.e. not one of the
+       types in MemoryClass::DEVICE, then this method will return immediately.
+
+       If the device MemoryType has been previously set to a different type and
+       the actual device memory has been allocated, this method will trigger an
+       error. This method will not perform the actual device memory allocation,
+       however, the allocation may already exist if the MemoryType is the same
+       as the current one.
+
+       If the Memory is an alias Memory, the device MemoryType of its base will
+       be updated as described above. */
+   inline void SetDeviceMemoryType(MemoryType d_mt);
+
    /** @brief Delete the owned pointers. The Memory is not reset by this method,
        i.e. it will, generally, not be Empty() after this call. */
    inline void Delete();
 
+   /** @brief Delete the device pointer, if owned. If @a copy_to_host is true
+       and the data is valid only on device, move it to host before deleting.
+       Invalidates the device memory. */
+   inline void DeleteDevice(bool copy_to_host = true);
+
    /// Array subscript operator for host memory.
    inline T &operator[](int idx);
 
@@ -468,6 +523,12 @@ private:
       static inline T *New(std::size_t size) { return new T[size]; }
    };
 #endif
+
+   // Shortcut for Alloc<new_align_bytes>::New(size)
+   static inline T *NewHOST(std::size_t size)
+   {
+      return Alloc<new_align_bytes>::New(size);
+   }
 };
 
 
@@ -495,10 +556,22 @@ private:
    /// Return true if the global memory manager instance exists.
    static bool Exists() { return exists; }
 
+   /// Array defining the dual MemoryType for each MemoryType
+   /** The dual of a host MemoryType is a device MemoryType and vice versa: the
+       dual of a device MemoryType is a host MemoryType. */
+   static MemoryType dual_map[MemoryTypeSize];
+
+   /// Update the dual memory type of @a mt to be @a dual_mt.
+   static void UpdateDualMemoryType(MemoryType mt, MemoryType dual_mt);
+
+   /// True if Configure() was called.
+   static bool configured;
+
    /// Host and device allocator names for Umpire.
 #ifdef MFEM_USE_UMPIRE
-   static const char *h_umpire_name;
-   static const char *d_umpire_name;
+   static const char * h_umpire_name;
+   static const char * d_umpire_name;
+   static const char * d_umpire_2_name;
 #endif
 
 private: // Static methods used by the Memory<T> class
@@ -508,26 +581,37 @@ private: // Static methods used by the Memory<T> class
    /// memory type, e.g. CUDA (mt will not be HOST).
    static void *New_(void *h_tmp, size_t bytes, MemoryType mt, unsigned &flags);
 
+   static void *New_(void *h_tmp, size_t bytes, MemoryType h_mt,
+                     MemoryType d_mt, unsigned valid_flags, unsigned &flags);
+
    /// Register an external pointer of the given MemoryType.
    /// Return the host pointer.
    static void *Register_(void *ptr, void *h_ptr, size_t bytes, MemoryType mt,
                           bool own, bool alias, unsigned &flags);
 
+   /// Register a pair of external host and device pointers
+   static void Register_(void *h_ptr, void *d_ptr, size_t bytes,
+                         MemoryType h_mt, MemoryType d_mt,
+                         bool own, bool alias, unsigned &flags);
+
    /// Register an alias. Note: base_h_ptr may be an alias.
    static void Alias_(void *base_h_ptr, size_t offset, size_t bytes,
                       unsigned base_flags, unsigned &flags);
 
+   static void SetDeviceMemoryType_(void *h_ptr, unsigned flags,
+                                    MemoryType d_mt);
+
    /// Un-register and free memory identified by its host pointer. Returns the
    /// memory type of the host pointer.
    static MemoryType Delete_(void *h_ptr, MemoryType mt, unsigned flags);
 
+   /// Free device memory identified by its host pointer
+   static void DeleteDevice_(void *h_ptr, unsigned & flags);
+
    /// Check if the memory types given the memory class are valid
    static bool MemoryClassCheck_(MemoryClass mc, void *h_ptr,
                                  MemoryType h_mt, size_t bytes, unsigned flags);
 
-   /// Return the dual memory type of the given one.
-   static MemoryType GetDualMemoryType_(MemoryType mt);
-
    /// Return a pointer to the memory identified by the host pointer h_ptr for
    /// access with the given MemoryClass.
    static void *ReadWrite_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
@@ -595,6 +679,9 @@ private:
    /// Erase an address from the memory map, as well as all its aliases
    void Erase(void *h_ptr, bool free_dev_ptr = true);
 
+   /// Erase device memory for a given host address
+   void EraseDevice(void *h_ptr);
+
    /// Erase an alias from the aliases map
    void EraseAlias(void *alias_ptr);
 
@@ -621,15 +708,59 @@ public:
    /// Initialize the memory manager.
    void Init();
 
-   /// Configure the Memory manager with given default host and device types
-   /// This method will be called when configuring a device.
+   /// Return the dual MemoryType of the given one, @a mt.
+   /** The default dual memory types are:
+
+       memory type     | dual type
+       --------------- | ---------
+       HOST            | DEVICE
+       HOST_32         | DEVICE
+       HOST_64         | DEVICE
+       HOST_DEBUG      | DEVICE_DEBUG
+       HOST_UMPIRE     | DEVICE_UMPIRE
+       HOST_PINNED     | DEVICE
+       MANAGED         | MANAGED
+       DEVICE          | HOST
+       DEVICE_DEBUG    | HOST_DEBUG
+       DEVICE_UMPIRE   | HOST_UMPIRE
+       DEVICE_UMPIRE_2 | HOST_UMPIRE
+
+       The dual types can be modified before device configuration using the
+       method SetDualMemoryType() or by calling Device::SetMemoryTypes(). */
+   static inline MemoryType GetDualMemoryType(MemoryType mt)
+   { return dual_map[(int)mt]; }
+
+   /// Set the dual memory type of @a mt to be @a dual_mt.
+   /** This method can only be called before configuration, i.e. before calling
+       Configure(), which is typically done during Device construction.
+
+       One of the types must be a host MemoryType and the other must be a device
+       MemoryType or both types must be the same host memory type. The latter
+       case is only allowed for convenience in setting up pure host execution,
+       so the actual dual is not updated. */
+   static void SetDualMemoryType(MemoryType mt, MemoryType dual_mt);
+
+   /** @brief Configure the Memory manager with given default host and device
+       types. This method will be called when configuring a device.
+
+       The host and device MemoryType%s, @a h_mt and @a d_mt, are set to be dual
+       to each other. */
    void Configure(const MemoryType h_mt, const MemoryType d_mt);
 
 #ifdef MFEM_USE_UMPIRE
-   /// Set the host and device UMpire allocator names
-   void SetUmpireAllocatorNames(const char *h_name, const char *d_name);
-   const char *GetUmpireAllocatorHostName() { return h_umpire_name; }
-   const char *GetUmpireAllocatorDeviceName() { return d_umpire_name; }
+   /// Set the host Umpire allocator name used with MemoryType::HOST_UMPIRE
+   static void SetUmpireHostAllocatorName(const char * h_name) { h_umpire_name = h_name; }
+   /// Set the device Umpire allocator name used with MemoryType::DEVICE_UMPIRE
+   static void SetUmpireDeviceAllocatorName(const char * d_name) { d_umpire_name = d_name; }
+   /// Set the device Umpire allocator name used with MemoryType::DEVICE_UMPIRE_2
+   static void SetUmpireDevice2AllocatorName(const char * d_name) { d_umpire_2_name = d_name; }
+
+   /// Get the host Umpire allocator name used with MemoryType::HOST_UMPIRE
+   static const char * GetUmpireHostAllocatorName() { return h_umpire_name; }
+   /// Get the device Umpire allocator name used with MemoryType::DEVICE_UMPIRE
+   static const char * GetUmpireDeviceAllocatorName() { return d_umpire_name; }
+   /// Get the device Umpire allocator name used with MemoryType::DEVICE_UMPIRE_2
+   static const char * GetUmpireDevice2AllocatorName() { return d_umpire_2_name; }
 #endif
 
    /// Free all the device memories
@@ -663,7 +794,7 @@ template <typename T>
 inline void Memory<T>::Reset()
 {
    h_ptr = NULL;
-   h_mt = MemoryManager::host_mem_type;
+   h_mt = MemoryManager::GetHostMemoryType();
    capacity = 0;
    flags = 0;
 }
@@ -682,8 +813,8 @@ inline void Memory<T>::New(int size)
 {
    capacity = size;
    flags = OWNS_HOST | VALID_HOST;
-   h_mt = MemoryManager::host_mem_type;
-   h_ptr = (h_mt == MemoryType::HOST) ? Alloc<new_align_bytes>::New(size) :
+   h_mt = MemoryManager::GetHostMemoryType();
+   h_ptr = (h_mt == MemoryType::HOST) ? NewHOST(size) :
            (T*)MemoryManager::New_(nullptr, size*sizeof(T), h_mt, flags);
 }
 
@@ -694,12 +825,21 @@ inline void Memory<T>::New(int size, MemoryType mt)
    const size_t bytes = size*sizeof(T);
    const bool mt_host = mt == MemoryType::HOST;
    if (mt_host) { flags = OWNS_HOST | VALID_HOST; }
-   h_mt = IsHostMemory(mt) ? mt : MemoryManager::GetDualMemoryType_(mt);
-   T *h_tmp = (h_mt == MemoryType::HOST) ?
-              Alloc<new_align_bytes>::New(size) : nullptr;
+   h_mt = IsHostMemory(mt) ? mt : MemoryManager::GetDualMemoryType(mt);
+   T *h_tmp = (h_mt == MemoryType::HOST) ? NewHOST(size) : nullptr;
    h_ptr = (mt_host) ? h_tmp : (T*)MemoryManager::New_(h_tmp, bytes, mt, flags);
 }
 
+template <typename T>
+inline void Memory<T>::New(int size, MemoryType h_mt, MemoryType d_mt)
+{
+   capacity = size;
+   const size_t bytes = size*sizeof(T);
+   this->h_mt = h_mt;
+   T *h_tmp = (h_mt == MemoryType::HOST) ? NewHOST(size) : nullptr;
+   h_ptr = (T*)MemoryManager::New_(h_tmp, bytes, h_mt, d_mt, VALID_HOST, flags);
+}
+
 template <typename T>
 inline void Memory<T>::Wrap(T *ptr, int size, bool own)
 {
@@ -707,7 +847,7 @@ inline void Memory<T>::Wrap(T *ptr, int size, bool own)
    capacity = size;
    const size_t bytes = size*sizeof(T);
    flags = (own ? OWNS_HOST : 0) | VALID_HOST;
-   h_mt = MemoryManager::host_mem_type;
+   h_mt = MemoryManager::GetHostMemoryType();
 #ifdef MFEM_DEBUG
    if (own && MemoryManager::Exists())
    { MFEM_VERIFY(h_mt == MemoryManager::GetHostMemoryType_(h_ptr),""); }
@@ -726,14 +866,14 @@ inline void Memory<T>::Wrap(T *ptr, int size, MemoryType mt, bool own)
       h_ptr = ptr;
       if (mt == MemoryType::HOST || !own)
       {
-         // Skip restration
+         // Skip registration
          flags = (own ? OWNS_HOST : 0) | VALID_HOST;
          return;
       }
    }
    else
    {
-      h_mt = MemoryManager::GetDualMemoryType_(mt);
+      h_mt = MemoryManager::GetDualMemoryType(mt);
       h_ptr = (h_mt == MemoryType::HOST) ? new T[size] : nullptr;
    }
    flags = 0;
@@ -750,8 +890,8 @@ inline void Memory<T>::Wrap(T *ptr, T *d_ptr, int size, MemoryType mt, bool own)
    capacity = size;
    MFEM_ASSERT(IsHostMemory(h_mt),"");
    const size_t bytes = size*sizeof(T);
-   const MemoryType d_mt = MemoryManager::GetDualMemoryType_(h_mt);
-   MemoryManager::Register_(d_ptr, h_ptr, bytes, d_mt, own, false, flags);
+   const MemoryType d_mt = MemoryManager::GetDualMemoryType(h_mt);
+   MemoryManager::Register_(h_ptr, d_ptr, bytes, h_mt, d_mt, own, false, flags);
 }
 
 template <typename T>
@@ -770,6 +910,18 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
    }
 }
 
+template <typename T>
+inline void Memory<T>::SetDeviceMemoryType(MemoryType d_mt)
+{
+   if (!IsDeviceMemory(d_mt)) { return; }
+   if (!(flags & REGISTERED))
+   {
+      MemoryManager::Register_(h_ptr, nullptr, capacity*sizeof(T), h_mt,
+                               flags & OWNS_HOST, flags & ALIAS, flags);
+   }
+   MemoryManager::SetDeviceMemoryType_(h_ptr, flags, d_mt);
+}
+
 template <typename T>
 inline void Memory<T>::Delete()
 {
@@ -784,6 +936,16 @@ inline void Memory<T>::Delete()
    }
 }
 
+template <typename T>
+inline void Memory<T>::DeleteDevice(bool copy_to_host)
+{
+   if (flags & REGISTERED)
+   {
+      if (copy_to_host) { Read(MemoryClass::HOST, capacity); }
+      MemoryManager::DeleteDevice_((void*)h_ptr, flags);
+   }
+}
+
 template <typename T>
 inline T &Memory<T>::operator[](int idx)
 {
@@ -911,7 +1073,7 @@ inline void Memory<T>::CopyFrom(const Memory &src, int size)
    {
       if (h_ptr != src.h_ptr && size != 0)
       {
-         MFEM_ASSERT(h_ptr + size <= src || src + size <= h_ptr,
+         MFEM_ASSERT(h_ptr + size <= src.h_ptr || src.h_ptr + size <= h_ptr,
                      "data overlaps!");
          std::memcpy(h_ptr, src, size*sizeof(T));
       }

linalg/densemat.cpp

@@ -587,11 +587,7 @@ DenseMatrix &DenseMatrix::operator=(const DenseMatrix &m)
 
 DenseMatrix &DenseMatrix::operator+=(const double *m)
 {
-   const int hw = Height()*Width();
-   for (int i = 0; i < hw; i++)
-   {
-      data[i] += m[i];
-   }
+   kernels::Add(Height(), Width(), m, (double*)data);
    return *this;
 }
 
@@ -1934,12 +1930,7 @@ void Add(const DenseMatrix &A, const DenseMatrix &B,
 void Add(double alpha, const double *A,
          double beta,  const double *B, DenseMatrix &C)
 {
-   const int m = C.Height()*C.Width();
-   double *C_data = C.GetData();
-   for (int i = 0; i < m; i++)
-   {
-      C_data[i] = alpha*A[i] + beta*B[i];
-   }
+   kernels::Add(C.Height(), C.Width(), alpha, A, beta, B, C.Data());
 }
 
 void Add(double alpha, const DenseMatrix &A,
@@ -3595,4 +3586,114 @@ void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X)
 
 }
 
+void BatchLUFactor(Vector &Minv,const int m,const int NE, Array<int> &P)
+{
+   P.SetSize(m*NE);
+   auto data_all = mfem::Reshape(Minv.ReadWrite(), m, m, NE);
+   auto piv_all = mfem::Reshape(P.Write(), m, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+
+      double *data = &data_all(0,0,e);
+      int *ipiv = &piv_all(0,e);
+      for (int i = 0; i < m; i++)
+      {
+
+         // pivoting
+         {
+            int piv = i;
+            double a = fabs(data[piv+i*m]);
+            for (int j = i+1; j < m; j++)
+            {
+               const double b = fabs(data[j+i*m]);
+               if (b > a)
+               {
+                  a = b;
+                  piv = j;
+               }
+            }
+            ipiv[i] = piv;
+            if (piv != i)
+            {
+               // swap rows i and piv in both L and U parts
+               for (int j = 0; j < m; j++)
+               {
+                  mfem::kernels::internal::Swap<double>(data[i+j*m], data[piv+j*m]);
+               }
+            }
+         }//pivot end
+
+         //Q: How to check for errors?
+         //if (abs(data[i + i*m]) <= TOL)
+         //{
+         //return false; // failed
+         //}
+
+         const double a_ii_inv = 1.0 / data[i+i*m];
+         for (int j = i+1; j < m; j++)
+         {
+            data[j+i*m] *= a_ii_inv;
+         }
+
+         for (int k = i+1; k < m; k++)
+         {
+            const double a_ik = data[i+k*m];
+            for (int j = i+1; j < m; j++)
+            {
+               data[j+k*m] -= a_ik * data[j+i*m];
+            }
+         }
+
+      }//m loop
+
+   });
+
+}
+
+void BatchLUSolve(Vector &Minv, int m, int NE,
+                  Array<int> &P, Vector &X)
+{
+
+   auto data_all = mfem::Reshape(Minv.Read(), m, m, NE);
+   auto piv_all = mfem::Reshape(P.Read(), m, NE);
+   auto x_all = mfem::Reshape(X.ReadWrite(), m, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+
+      const double *data = &data_all(0,0,e);
+      const int *ipiv = &piv_all(0,e);
+      double *x = &x_all(0,e);
+
+      // X <- P X
+      for (int i = 0; i < m; i++)
+      {
+         mfem::kernels::internal::Swap<double>(x[i], x[ipiv[i]]);
+      }
+
+      // X <- L^{-1} X
+      for (int j = 0; j < m; j++)
+      {
+         const double x_j = x[j];
+         for (int i = j+1; i < m; i++)
+         {
+            x[i] -= data[i+j*m] * x_j;
+         }
+      }
+
+      // X <- U^{-1} X
+      for (int j = m-1; j >= 0; j--)
+      {
+         const double x_j = ( x[j] /= data[j+j*m] );
+         for (int i = 0; i < j; i++)
+         {
+            x[i] -= data[i+j*m] * x_j;
+         }
+      }
+   });
+
+}
+
+
 } // namespace mfem

linalg/densemat.hpp

@@ -746,6 +746,13 @@ public:
       tdata.New(i*j*k);
    }
 
+   DenseTensor(int i, int j, int k, MemoryType mt)
+      : Mk(NULL, i, j)
+   {
+      nk = k;
+      tdata.New(i*j*k, mt);
+   }
+
    /// Copy constructor: deep copy
    DenseTensor(const DenseTensor &other)
       : Mk(NULL, other.Mk.height, other.Mk.width), nk(other.nk)
@@ -768,9 +775,9 @@ public:
 
    int TotalSize() const { return SizeI()*SizeJ()*SizeK(); }
 
-   void SetSize(int i, int j, int k)
+   void SetSize(int i, int j, int k, MemoryType mt_ = MemoryType::SIZE)
    {
-      const MemoryType mt = tdata.GetMemoryType();
+      const MemoryType mt = mt_ == MemoryType::SIZE ? tdata.GetMemoryType() : mt_;
       tdata.Delete();
       Mk.UseExternalData(NULL, i, j);
       nk = k;
@@ -884,6 +891,10 @@ void BatchLUFactor(DenseTensor &Mlu, Array<int> &P, const double TOL = 0.0);
     dimension m x n. */
 void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X);
 
+void BatchLUFactor(Vector &Minv,int m,int NE, Array<int> &P);
+
+void BatchLUSolve(Vector &Minv, int m, int NE,
+                  Array<int> &P, Vector &X);
 
 // Inline methods
 

linalg/dinvariants.hpp

@@ -0,0 +1,699 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DINVARIANTS_HPP
+#define MFEM_DINVARIANTS_HPP
+
+#include "../config/config.hpp"
+#include "../general/cuda.hpp"
+#include "dtensor.hpp"
+#include <cmath>
+
+namespace mfem
+{
+
+namespace kernels
+{
+
+class InvariantsEvaluator2D
+{
+public:
+   class Buffers
+   {
+      friend class InvariantsEvaluator2D;
+   private:
+      const double * J_ = nullptr;
+      double * dI1_ = nullptr;
+      double * dI1b_ = nullptr;
+      double * ddI1_ = nullptr;
+      double * ddI1b_ = nullptr;
+      double * dI2_ = nullptr;
+      double * dI2b_ = nullptr;
+      double * ddI2_ = nullptr;
+      double * ddI2b_ = nullptr;
+   public:
+      MFEM_HOST_DEVICE Buffers() {}
+      MFEM_HOST_DEVICE Buffers &J(const double *b) { J_     = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI1(double *b)     { dI1_   = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI1b(double *b)    { dI1b_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI1(double *b)    { ddI1_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI1b(double *b)   { ddI1b_ = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI2(double *b)     { dI2_   = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI2b(double *b)    { dI2b_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI2(double *b)    { ddI2_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI2b(double *b)   { ddI2b_ = b; return *this; }
+   };
+
+private:
+   double const * const J;
+   double * const dI1, * const dI1b, * const ddI1, * const ddI1b;
+   double * const dI2, * const dI2b, * const ddI2, * const ddI2b;
+
+public:
+   MFEM_HOST_DEVICE
+   InvariantsEvaluator2D(Buffers b):
+      J(b.J_),
+      dI1(b.dI1_), dI1b(b.dI1b_), ddI1(b.ddI1_), ddI1b(b.ddI1b_),
+      dI2(b.dI2_), dI2b(b.dI2b_), ddI2(b.ddI2_), ddI2b(b.ddI2b_) { }
+
+   MFEM_HOST_DEVICE inline double Get_I2b(double &sign_detJ) // det(J) + sign
+   {
+      const double I2b = J[0]*J[3] - J[1]*J[2];
+      sign_detJ = I2b >= 0.0 ? 1.0 : -1.0;
+      return sign_detJ * I2b;
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I2b() // det(J)
+   {
+      double sign_detJ;
+      return Get_I2b(sign_detJ);
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I2() // det(J)^{2}
+   {
+      const double I2b = Get_I2b();
+      return I2b * I2b;
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I1() // I1 = ||J||_F^2
+   {
+      return J[0]*J[0] + J[1]*J[1] + J[2]*J[2] + J[3]*J[3];
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I1b() // I1b = I1/det(J)
+   {
+      return Get_I1() / Get_I2b();
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI1()
+   {
+      dI1[0] = 2*J[0]; dI1[2] = 2*J[2];
+      dI1[1] = 2*J[1]; dI1[3] = 2*J[3];
+      return dI1;
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI1b()
+   {
+      // I1b = I1/I2b
+      // dI1b = (1/I2b)*dI1 - (I1/I2b^2)*dI2b = (2/I2b)*[J - (I1b/2)*dI2b]
+      const double c1 = 2.0/Get_I2b();
+      const double c2 = Get_I1b()/2.0;
+      Get_dI2b();
+      dI1b[0] = c1*(J[0] - c2*dI2b[0]);
+      dI1b[1] = c1*(J[1] - c2*dI2b[1]);
+      dI1b[2] = c1*(J[2] - c2*dI2b[2]);
+      dI1b[3] = c1*(J[3] - c2*dI2b[3]);
+      return dI1b;
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI2()
+   {
+      // I2 = I2b^2
+      // dI2 = 2*I2b*dI2b = 2*det(J)*adj(J)^T
+      const double c1 = 2*Get_I2b();
+      Get_dI2b();
+      dI2[0] = c1*dI2b[0];
+      dI2[1] = c1*dI2b[1];
+      dI2[2] = c1*dI2b[2];
+      dI2[3] = c1*dI2b[3];
+      return dI2;
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI2b()
+   {
+      // I2b = det(J)
+      // dI2b = adj(J)^T
+      double sign_detJ;
+      Get_I2b(sign_detJ);
+      dI2b[0] =  sign_detJ*J[3];
+      dI2b[1] = -sign_detJ*J[2];
+      dI2b[2] = -sign_detJ*J[1];
+      dI2b[3] =  sign_detJ*J[0];
+      return dI2b;
+   }
+
+   // ddI1_ijkl = 2 I_ijkl = 2 δ_ik δ_jl
+   MFEM_HOST_DEVICE inline double *Get_ddI1(int i, int j)
+   {
+      // ddI1_ijkl = 2 I_ijkl = 2 δ_ik δ_jl
+      DeviceMatrix ddi1(ddI1,2,2);
+      for (int k=0; k<2; k++)
+      {
+         for (int l=0; l<2; l++)
+         {
+            ddi1(k,l) = (i==k && j==l) ? 2.0 : 0.0;
+         }
+      }
+      return ddI1;
+   }
+
+   // ddI1b = X1 + X2 + X3, where
+   // X1_ijkl = (I1b/I2) [ dI2b_ij dI2b_kl + dI2b_kj dI2b_il ]
+   // X2_ijkl = (1/I2b) ddI1_ijkl
+   // X3_ijkl = -(2/I2) (J_ij dI2b_kl + dI2b_ij J_kl)
+   MFEM_HOST_DEVICE inline double *Get_ddI1b(int i, int j)
+   {
+      double X1_p[4], X2_p[4], X3_p[4];
+
+      // X1_ijkl = (I1b/I2) [ dI2b_ij dI2b_kl + dI2b_kj dI2b_il ]
+      const double I2 = Get_I2();
+      const double I1b = Get_I1b();
+      ConstDeviceMatrix di2b(Get_dI2b(),2,2);
+      const double alpha = I1b / I2;
+      DeviceMatrix X1(X1_p,2,2);
+      for (int k=0; k<2; k++)
+      {
+         for (int l=0; l<2; l++)
+         {
+            X1(k,l) = alpha * (di2b(i,j)*di2b(k,l) + di2b(k,j)*di2b(i,l));
+         }
+      }
+      // X2_ijkl = (1/I2b) ddI1_ijkl
+      DeviceMatrix X2(X2_p,2,2);
+      const double beta = 1.0 / Get_I2b();
+      ConstDeviceMatrix ddi1(Get_ddI1(i,j),2,2);
+      for (int k=0; k<2; k++)
+      {
+         for (int l=0; l<2; l++)
+         {
+            X2(k,l) = beta * ddi1(k,l);
+         }
+      }
+      // X3_ijkl = -(2/I2) (J_ij dI2b_kl + dI2b_ij J_kl)
+      DeviceMatrix X3(X3_p,2,2);
+      const double gamma = -2.0/Get_I2();
+      ConstDeviceMatrix Jpt(J,2,2);
+      for (int k=0; k<2; k++)
+      {
+         for (int l=0; l<2; l++)
+         {
+            X3(k,l) = gamma * (Jpt(i,j)*di2b(k,l) + di2b(i,j)*Jpt(k,l));
+         }
+      }
+      DeviceMatrix ddi1b(ddI1b,2,2);
+      for (int k=0; k<2; k++)
+      {
+         for (int l=0; l<2; l++)
+         {
+            ddi1b(k,l) = X1(k,l) + X2(k,l) + X3(k,l);
+         }
+      }
+      return ddI1b;
+   }
+
+   // ddI2_ijkl = 2 dI2b_ij dI2b_kl + 2 (dI2b_ij dI2b_kl - dI2b_kj dI2b_il)
+   MFEM_HOST_DEVICE inline double *Get_ddI2(int i, int j)
+   {
+      DeviceMatrix ddi2(ddI2,2,2);
+      ConstDeviceMatrix di2b(Get_dI2b(),2,2);
+      for (int k=0; k<2; k++)
+      {
+         for (int l=0; l<2; l++)
+         {
+            ddi2(k,l) = 2*di2b(i,j)*di2b(k,l)
+                        + 2*(di2b(i,j)*di2b(k,l) - di2b(k,j)*di2b(i,l));
+         }
+      }
+      return ddI2;
+   }
+
+   // ddI2b_ijkl = (1/I2b) (δ_ks δ_it - δ_kt δ_si) dI2b_tj dI2b_sl
+   MFEM_HOST_DEVICE inline double *Get_ddI2b(int i, int j)
+   {
+      DeviceMatrix ddi2b(ddI2b,2,2);
+      const double alpha = 1.0/Get_I2b();
+      ConstDeviceMatrix di2b(Get_dI2b(),2,2);
+      for (int k=0; k<2; k++)
+      {
+         for (int l=0; l<2; l++)
+         {
+            ddi2b(k,l) = 0.0;
+            for (int s=0; s<2; s++)
+            {
+               for (int t=0; t<2; t++)
+               {
+                  const double ks_it = k==s && i==t ? 1.0 : 0.0;
+                  const double kt_si = k==t && s==i ? 1.0 : 0.0;
+                  ddi2b(k,l) += alpha * (ks_it - kt_si) * di2b(t,j) * di2b(s,l);
+               }
+            }
+         }
+      }
+      return ddI2b;
+   }
+};
+
+
+class InvariantsEvaluator3D
+{
+public:
+   class Buffers
+   {
+      friend class InvariantsEvaluator3D;
+   private:
+      const double * J_ = nullptr;
+      double * B_ = nullptr;
+      double * dI1_ = nullptr;
+      double * dI1b_ = nullptr;
+      double * ddI1_ = nullptr;
+      double * ddI1b_ = nullptr;
+      double * dI2_ = nullptr;
+      double * dI2b_ = nullptr;
+      double * ddI2_ = nullptr;
+      double * ddI2b_ = nullptr;
+      double * dI3b_ = nullptr;
+      double * ddI3b_ = nullptr;
+   public:
+      MFEM_HOST_DEVICE Buffers() {}
+      MFEM_HOST_DEVICE Buffers &J(const double *b) { J_     = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &B(double *b)       { B_     = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI1(double *b)     { dI1_   = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI1b(double *b)    { dI1b_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI1(double *b)    { ddI1_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI1b(double *b)   { ddI1b_ = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI2(double *b)     { dI2_   = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI2b(double *b)    { dI2b_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI2(double *b)    { ddI2_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI2b(double *b)   { ddI2b_ = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &dI3b(double *b)    { dI3b_  = b; return *this; }
+      MFEM_HOST_DEVICE Buffers &ddI3b(double *b)   { ddI3b_ = b; return *this; }
+   };
+
+private:
+   double const * const J;
+   double * const B;
+   double * const dI1, * const dI1b, * const ddI1, * const ddI1b;
+   double * const dI2, * const dI2b, * const ddI2, * const ddI2b;
+   double * const dI3b, * const ddI3b;
+
+public:
+   MFEM_HOST_DEVICE
+   InvariantsEvaluator3D(Buffers b):
+      J(b.J_), B(b.B_),
+      dI1(b.dI1_), dI1b(b.dI1b_), ddI1(b.ddI1_), ddI1b(b.ddI1b_),
+      dI2(b.dI2_), dI2b(b.dI2b_), ddI2(b.ddI2_), ddI2b(b.ddI2b_),
+      dI3b(b.dI3b_), ddI3b(b.ddI3b_) { }
+
+   MFEM_HOST_DEVICE inline double Get_I3b(double &sign_detJ) // det(J) + sign
+   {
+      const double I3b = + J[0]*(J[4]*J[8] - J[7]*J[5])
+                         - J[1]*(J[3]*J[8] - J[5]*J[6])
+                         + J[2]*(J[3]*J[7] - J[4]*J[6]);
+      sign_detJ = I3b >= 0.0 ? 1.0 : -1.0;
+      return sign_detJ * I3b;
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I3b() // det(J)
+   {
+      const double I3b = + J[0]*(J[4]*J[8] - J[7]*J[5])
+                         - J[1]*(J[3]*J[8] - J[5]*J[6])
+                         + J[2]*(J[3]*J[7] - J[4]*J[6]);
+      return I3b;
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I3() // det(J)^{2}
+   {
+      const double I3b = Get_I3b();
+      return I3b * I3b;
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I3b_p() // I3b^{-2/3}
+   {
+      double sign_detJ;
+      const double i3b = Get_I3b(sign_detJ);
+      return sign_detJ * std::pow(i3b, -2./3.);
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I3b_p(double &sign_detJ) // I3b^{-2/3}
+   {
+      const double i3b = Get_I3b(sign_detJ);
+      return sign_detJ * std::pow(i3b, -2./3.);
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I1()
+   {
+      B[0] = J[0]*J[0] + J[3]*J[3] + J[6]*J[6];
+      B[1] = J[1]*J[1] + J[4]*J[4] + J[7]*J[7];
+      B[2] = J[2]*J[2] + J[5]*J[5] + J[8]*J[8];
+      const double I1 = B[0] + B[1] + B[2];
+      return I1;
+   }
+
+   MFEM_HOST_DEVICE inline
+   double Get_I1b() // det(J)^{-2/3}*I_1 = I_1/I_3^{1/3}
+   {
+      const double I1b = Get_I1() * Get_I3b_p();
+      return I1b;
+   }
+
+   MFEM_HOST_DEVICE inline void Get_B_offd()
+   {
+      // B = J J^t
+      // B[3]=B(0,1), B[4]=B(0,2), B[5]=B(1,2)
+      B[3] = J[0]*J[1] + J[3]*J[4] + J[6]*J[7]; // B(0,1)
+      B[4] = J[0]*J[2] + J[3]*J[5] + J[6]*J[8]; // B(0,2)
+      B[5] = J[1]*J[2] + J[4]*J[5] + J[7]*J[8]; // B(1,2)
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I2()
+   {
+      Get_B_offd();
+      const double I1 = Get_I1();
+      const double BF2 = B[0]*B[0] + B[1]*B[1] + B[2]*B[2] +
+                         2*(B[3]*B[3] + B[4]*B[4] + B[5]*B[5]);
+      const double I2 = (I1*I1 - BF2)/2;
+      return I2;
+   }
+
+   MFEM_HOST_DEVICE inline double Get_I2b() // I2b = I2*I3b^{-4/3}
+   {
+      const double I3b_p = Get_I3b_p();
+      return Get_I2() * I3b_p * I3b_p;
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI1()
+   {
+      for (int i = 0; i < 9; i++) { dI1[i] = 2*J[i]; }
+      return dI1;
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI1b()
+   {
+      // I1b = I3b^{-2/3}*I1
+      // dI1b = 2*I3b^{-2/3}*(J - (1/3)*I1/I3b*dI3b)
+      double sign_detJ;
+      const double I3b = Get_I3b(sign_detJ);
+      const double I3b_p = Get_I3b_p();
+      const double c1 = 2.0 * I3b_p;
+      const double c2 = Get_I1()/(3.0 * I3b);
+      Get_dI3b(sign_detJ);
+      for (int i = 0; i < 9; i++) { dI1b[i] = c1*(J[i] - c2*dI3b[i]); }
+      return dI1b;
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI2()
+   {
+      // dI2 = 2 I_1 J - 2 J J^t J = 2 (I_1 I - B) J
+      const double I1 = Get_I1();
+      Get_B_offd();
+      // B[0]=B(0,0), B[1]=B(1,1), B[2]=B(2,2)
+      // B[3]=B(0,1), B[4]=B(0,2), B[5]=B(1,2)
+      const double C[6] =
+      {
+         2*(I1 - B[0]), 2*(I1 - B[1]), 2*(I1 - B[2]),
+         -2*B[3], -2*B[4], -2*B[5]
+      };
+      //       | C[0]  C[3]  C[4] |  | J[0]  J[3]  J[6] |
+      // dI2 = | C[3]  C[1]  C[5] |  | J[1]  J[4]  J[7] |
+      //       | C[4]  C[5]  C[2] |  | J[2]  J[5]  J[8] |
+      dI2[0] = C[0]*J[0] + C[3]*J[1] + C[4]*J[2];
+      dI2[1] = C[3]*J[0] + C[1]*J[1] + C[5]*J[2];
+      dI2[2] = C[4]*J[0] + C[5]*J[1] + C[2]*J[2];
+
+      dI2[3] = C[0]*J[3] + C[3]*J[4] + C[4]*J[5];
+      dI2[4] = C[3]*J[3] + C[1]*J[4] + C[5]*J[5];
+      dI2[5] = C[4]*J[3] + C[5]*J[4] + C[2]*J[5];
+
+      dI2[6] = C[0]*J[6] + C[3]*J[7] + C[4]*J[8];
+      dI2[7] = C[3]*J[6] + C[1]*J[7] + C[5]*J[8];
+      dI2[8] = C[4]*J[6] + C[5]*J[7] + C[2]*J[8];
+      return dI2;
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI2b()
+   {
+      // I2b = det(J)^{-4/3}*I2 = I3b^{-4/3}*I2
+      // dI2b = (-4/3)*I3b^{-7/3}*I2*dI3b + I3b^{-4/3}*dI2
+      //      = I3b^{-4/3} * [ dI2 - (4/3)*I2/I3b*dI3b ]
+      double sign_detJ;
+      const double I2 = Get_I2();
+      const double I3b_p = Get_I3b_p();
+      const double I3b = Get_I3b(sign_detJ);
+      const double c1 = I3b_p*I3b_p;
+      const double c2 = (4*I2/I3b)/3;
+      Get_dI2();
+      Get_dI3b(sign_detJ);
+      for (int i = 0; i < 9; i++) { dI2b[i] = c1*(dI2[i] - c2*dI3b[i]); }
+      return dI2b;
+   }
+
+   MFEM_HOST_DEVICE inline double *Get_dI3b(const double sign_detJ)
+   {
+      // I3b = det(J)
+      // dI3b = adj(J)^T
+      dI3b[0] = sign_detJ*(J[4]*J[8] - J[5]*J[7]); // 0  3  6
+      dI3b[1] = sign_detJ*(J[5]*J[6] - J[3]*J[8]); // 1  4  7
+      dI3b[2] = sign_detJ*(J[3]*J[7] - J[4]*J[6]); // 2  5  8
+      dI3b[3] = sign_detJ*(J[2]*J[7] - J[1]*J[8]);
+      dI3b[4] = sign_detJ*(J[0]*J[8] - J[2]*J[6]);
+      dI3b[5] = sign_detJ*(J[1]*J[6] - J[0]*J[7]);
+      dI3b[6] = sign_detJ*(J[1]*J[5] - J[2]*J[4]);
+      dI3b[7] = sign_detJ*(J[2]*J[3] - J[0]*J[5]);
+      dI3b[8] = sign_detJ*(J[0]*J[4] - J[1]*J[3]);
+      return dI3b;
+   }
+
+   // ddI1_ijkl = 2 I_ijkl = 2 δ_ik δ_jl
+   MFEM_HOST_DEVICE inline double *Get_ddI1(int i, int j)
+   {
+      DeviceMatrix ddi1(ddI1,3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            const double I_ijkl = (i==k && j==l) ? 1.0 : 0.0;
+            ddi1(k,l) = 2.0 * I_ijkl;
+         }
+      }
+      return ddI1;
+   }
+
+   // ddI1b = X1 + X2 + X3, where
+   // X1_ijkl = (2/3*I1b/I3) [ 2/3 dI3b_ij dI3b_kl + dI3b_kj dI3b_il ]
+   // X2_ijkl = (I3b^{-2/3}) ddI1_ijkl
+   // X3_ijkl = -(4/3*I3b^{-5/3}) (J_ij dI3b_kl + dI3b_ij J_kl)
+   MFEM_HOST_DEVICE inline double *Get_ddI1b(int i, int j)
+   {
+      // X1_ijkl = (2/3*I1b/I3) [ 2/3 dI3b_ij dI3b_kl + dI3b_kj dI3b_il ]
+      double sign_detJ;
+      Get_I3b(sign_detJ);
+      double X1_p[9], X2_p[9], X3_p[9];
+      DeviceMatrix X1(X1_p,3,3);
+      const double I3 = Get_I3();
+      const double I1b = Get_I1b();
+      const double alpha = (2./3.)*I1b/I3;
+      ConstDeviceMatrix di3b(Get_dI3b(sign_detJ),3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            X1(k,l) = alpha * ((2./3.)*di3b(i,j) * di3b(k,l) +
+                               di3b(k,j)*di3b(i,l));
+         }
+      }
+      // ddI1_ijkl = 2 δ_ik δ_jl
+      // X2_ijkl = (I3b^{-2/3}) ddI1_ijkl
+      DeviceMatrix X2(X2_p,3,3);
+      const double beta = Get_I3b_p();
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            const double ddI1_ijkl = (i==k && j==l) ? 2.0 : 0.0;
+            X2(k,l) = beta * ddI1_ijkl;
+         }
+      }
+      // X3_ijkl = -(4/3*I3b^{-5/3}) (J_ij dI3b_kl + dI3b_ij J_kl)
+      DeviceMatrix X3(X3_p,3,3);
+      const double I3b = Get_I3b();
+      const double gamma = -(4./3.)*Get_I3b_p()/I3b;
+      ConstDeviceMatrix Jpt(J,3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            X3(k,l) = gamma * (Jpt(i,j) * di3b(k,l) + di3b(i,j) * Jpt(k,l));
+         }
+      }
+      DeviceMatrix ddi1b(ddI1b,3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            ddi1b(k,l) = X1(k,l) + X2(k,l) + X3(k,l);
+         }
+      }
+      return ddI1b;
+   }
+
+   // ddI2 = x1 + x2 + x3
+   //    x1_ijkl = (2 I1) δ_ik δ_jl
+   //    x2_ijkl = 2 ( 2 δ_ku δ_iv - δ_ik δ_uv - δ_kv δ_iu ) J_vj J_ul
+   //    x3_ijkl = -2 (J J^t)_ik δ_jl = -2 B_ik δ_jl
+   MFEM_HOST_DEVICE inline double *Get_ddI2(int i, int j)
+   {
+      double x1_p[9], x2_p[9], x3_p[9];
+      DeviceMatrix x1(x1_p,3,3), x2(x2_p,3,3), x3(x3_p,3,3);
+      // x1_ijkl = (2 I1) δ_ik δ_jl
+      const double I1 = Get_I1();
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            const double ik_jl = (i==k && j==l) ? 1.0 : 0.0;
+            x1(k,l) = 2.0 * I1 * ik_jl;
+         }
+      }
+      // x2_ijkl = 2 ( 2 δ_ku δ_iv - δ_ik δ_uv - δ_kv δ_iu ) J_vj J_ul
+      ConstDeviceMatrix Jpt(J,3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            x2(k,l) = 0.0;
+            for (int u=0; u<3; u++)
+            {
+               for (int v=0; v<3; v++)
+               {
+                  const double ku_iv = k==u && i==v ? 1.0 : 0.0;
+                  const double ik_uv = i==k && u==v ? 1.0 : 0.0;
+                  const double kv_iu = k==v && i==u ? 1.0 : 0.0;
+                  x2(k,l) += 2.0*(2.*ku_iv-ik_uv-kv_iu)*Jpt(v,j)*Jpt(u,l);
+               }
+            }
+         }
+      }
+      // x3_ijkl = -2 B_ik δ_jl
+      B[0] = J[0]*J[0] + J[3]*J[3] + J[6]*J[6];
+      B[1] = J[1]*J[1] + J[4]*J[4] + J[7]*J[7];
+      B[2] = J[2]*J[2] + J[5]*J[5] + J[8]*J[8];
+      B[3] = J[0]*J[1] + J[3]*J[4] + J[6]*J[7]; // B(0,1)
+      B[4] = J[0]*J[2] + J[3]*J[5] + J[6]*J[8]; // B(0,2)
+      B[5] = J[1]*J[2] + J[4]*J[5] + J[7]*J[8]; // B(1,2)
+      const double b_p[9] =
+      {
+         B[0], B[3], B[4],
+         B[3], B[1], B[5],
+         B[4], B[5], B[2]
+      };
+      ConstDeviceMatrix b(b_p,3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            const double jl = j==l ? 1.0 : 0.0;
+            x3(k,l) = -2.0 * b(i,k) * jl;
+         }
+      }
+      // ddI2 = x1 + x2 + x3
+      DeviceMatrix ddi2(ddI2,3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            ddi2(k,l) = x1(k,l) + x2(k,l) + x3(k,l);
+         }
+      }
+      return ddI2;
+   }
+
+   // ddI2b = X1 + X2 + X3
+   //    X1_ijkl = 16/9 det(J)^{-10/3} I2 dI3b_ij dI3b_kl +
+   //               4/3 det(J)^{-10/3} I2 dI3b_il dI3b_kj
+   //    X2_ijkl = -4/3 det(J)^{-7/3} (dI2_ij dI3b_kl + dI2_kl dI3b_ij)
+   //    X3_ijkl =      det(J)^{-4/3} ddI2_ijkl
+   MFEM_HOST_DEVICE inline double *Get_ddI2b(int i, int j)
+   {
+      double X1_p[9], X2_p[9], X3_p[9];
+      // X1_ijkl = 16/9 det(J)^{-10/3} I2 dI3b_ij dI3b_kl +
+      //               4/3 det(J)^{-10/3} I2 dI3b_il dI3b_kj
+      double sign_detJ;
+      DeviceMatrix X1(X1_p,3,3);
+      const double I3b_p = Get_I3b_p(); // I3b^{-2/3}
+      const double I3b = Get_I3b(sign_detJ); // det(J)
+      const double I2 = Get_I2();
+      const double I3b_p43 = I3b_p*I3b_p;
+      const double I3b_p73 = I3b_p*I3b_p/I3b;
+      const double I3b_p103 = I3b_p*I3b_p/(I3b*I3b);
+      ConstDeviceMatrix di3b(Get_dI3b(sign_detJ),3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            const double up = (16./9.)*I3b_p103*I2*di3b(i,j)*di3b(k,l);
+            const double down = (4./3.)*I3b_p103*I2*di3b(i,l)*di3b(k,j);
+            X1(k,l) = up + down;
+         }
+      }
+      // X2_ijkl = -4/3 det(J)^{-7/3} (dI2_ij dI3b_kl + dI2_kl dI3b_ij)
+      DeviceMatrix X2(X2_p,3,3);
+      ConstDeviceMatrix di2(Get_dI2(),3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            X2(k,l) = -(4./3.)*I3b_p73*(di2(i,j)*di3b(k,l)+di2(k,l)*di3b(i,j));
+         }
+      }
+      // X3_ijkl =  det(J)^{-4/3} ddI2_ijkl
+      DeviceMatrix X3(X3_p,3,3);
+      ConstDeviceMatrix ddi2(Get_ddI2(i,j),3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            X3(k,l) = I3b_p43 * ddi2(k,l);
+         }
+      }
+      // ddI2b = X1 + X2 + X3
+      DeviceMatrix ddi2b(ddI2b,3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            ddi2b(k,l) = X1(k,l) + X2(k,l) + X3(k,l);
+         }
+      }
+      return ddI2b;
+   }
+
+   // dI3b = adj(J)^T
+   // ddI3b_ijkl = (1/I3b) (δ_ks δ_it - δ_kt δ_si) dI3b_tj dI3b_sl
+   MFEM_HOST_DEVICE inline double *Get_ddI3b(int i, int j)
+   {
+      const double c1 = 1./Get_I3b();
+      ConstDeviceMatrix di3b(dI3b,3,3);
+      DeviceMatrix ddi3b(ddI3b,3,3);
+      for (int k=0; k<3; k++)
+      {
+         for (int l=0; l<3; l++)
+         {
+            ddi3b(k,l) = 0.0;
+            for (int s=0; s<3; s++)
+            {
+               for (int t=0; t<3; t++)
+               {
+                  const double ks_it = k==s && i==t ? 1.0 : 0.0;
+                  const double kt_si = k==t && s==i ? 1.0 : 0.0;
+                  ddi3b(k,l) += c1*(ks_it-kt_si)*di3b(t,j)*di3b(s,l);
+               }
+            }
+         }
+      }
+      return ddI3b;
+   }
+};
+
+} // namespace kernels
+
+} // namespace mfem
+
+#endif // MFEM_DINVARIANTS_HPP

linalg/dtensor.hpp

@@ -54,6 +54,7 @@ template <int N, int Dim, typename T, typename... Args>
 class Init
 {
 public:
+   MFEM_HOST_DEVICE
    static inline int result(int* sizes, T first, Args... args)
    {
       sizes[N - 1] = first;
@@ -66,6 +67,7 @@ template <int Dim, typename T, typename... Args>
 class Init<Dim, Dim, T, Args...>
 {
 public:
+   MFEM_HOST_DEVICE
    static inline int result(int* sizes, T first, Args... args)
    {
       sizes[Dim - 1] = first;
@@ -88,7 +90,7 @@ public:
    DeviceTensor() = delete;
 
    /// Constructor to initialize a tensor from the Scalar array _data
-   template <typename... Args>
+   template <typename... Args> MFEM_HOST_DEVICE
    DeviceTensor(Scalar* _data, Args... args)
    {
       static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
@@ -110,7 +112,7 @@ public:
    }
 
    /// Conversion to `Scalar *`.
-   inline operator Scalar *() const { return data; }
+   MFEM_HOST_DEVICE inline operator Scalar *() const { return data; }
 
    /// Const accessor for the data
    template <typename... Args> MFEM_HOST_DEVICE inline
@@ -138,8 +140,16 @@ inline DeviceTensor<sizeof...(Dims),T> Reshape(T *ptr, Dims... dims)
 
 
 typedef DeviceTensor<1,int> DeviceArray;
+typedef DeviceTensor<1,const int> ConstDeviceArray;
+
 typedef DeviceTensor<1,double> DeviceVector;
+typedef DeviceTensor<1,const double> ConstDeviceVector;
+
 typedef DeviceTensor<2,double> DeviceMatrix;
+typedef DeviceTensor<2,const double> ConstDeviceMatrix;
+
+typedef DeviceTensor<3,double> DeviceCube;
+typedef DeviceTensor<3,const double> ConstDeviceCube;
 
 } // mfem namespace
 

linalg/hypre.cpp

@@ -1249,6 +1249,7 @@ void HypreParMatrix::operator*=(double s)
 static void get_sorted_rows_cols(const Array<int> &rows_cols,
                                  Array<HYPRE_Int> &hypre_sorted)
 {
+   rows_cols.HostRead();
    hypre_sorted.SetSize(rows_cols.Size());
    bool sorted = true;
    for (int i = 0; i < rows_cols.Size(); i++)

linalg/kernels.hpp

@@ -9,8 +9,8 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#ifndef MFEM_KERNELS_HPP
-#define MFEM_KERNELS_HPP
+#ifndef MFEM_LINALG_KERNELS_HPP
+#define MFEM_LINALG_KERNELS_HPP
 
 #ifdef _WIN32
 #define _USE_MATH_DEFINES
@@ -40,6 +40,98 @@ namespace mfem
 namespace kernels
 {
 
+/// Compute the square of the Euclidean distance to another vector
+template<int dim>
+MFEM_HOST_DEVICE inline double DistanceSquared(const double *x, const double *y)
+{
+   double d = 0.0;
+   for (int i = 0; i < dim; i++) { d += (x[i]-y[i])*(x[i]-y[i]); }
+   return d;
+}
+
+/// Creates n x n diagonal matrix with diagonal elements c
+template<int dim>
+MFEM_HOST_DEVICE inline void Diag(const double c, double *data)
+{
+   const int N = dim*dim;
+   for (int i = 0; i < N; i++) { data[i] = 0.0; }
+   for (int i = 0; i < dim; i++) { data[i*(dim+1)] = c; }
+}
+
+/// Vector subtraction operation: z = a * (x - y)
+template<int dim>
+MFEM_HOST_DEVICE inline void Subtract(const double a,
+                                      const double *x, const double *y,
+                                      double *z)
+{
+   for (int i = 0; i < dim; i++) { z[i] = a * (x[i] - y[i]); }
+}
+
+/// Dense matrix operation: VWt += v w^t
+template<int dim>
+MFEM_HOST_DEVICE inline void AddMultVWt(const double *v, const double *w,
+                                        double *VWt)
+{
+   for (int i = 0; i < dim; i++)
+   {
+      const double vi = v[i];
+      for (int j = 0; j < dim; j++) { VWt[i*dim+j] += vi * w[j]; }
+   }
+}
+
+template<int H, int W, typename T>
+MFEM_HOST_DEVICE inline
+void FNorm(double &scale_factor, double &scaled_fnorm2, const T *data)
+{
+   int i, hw = H * W;
+   T max_norm = 0.0, entry, fnorm2;
+
+   for (i = 0; i < hw; i++)
+   {
+      entry = fabs(data[i]);
+      if (entry > max_norm)
+      {
+         max_norm = entry;
+      }
+   }
+
+   if (max_norm == 0.0)
+   {
+      scale_factor = scaled_fnorm2 = 0.0;
+      return;
+   }
+
+   fnorm2 = 0.0;
+   for (i = 0; i < hw; i++)
+   {
+      entry = data[i] / max_norm;
+      fnorm2 += entry * entry;
+   }
+
+   scale_factor = max_norm;
+   scaled_fnorm2 = fnorm2;
+}
+
+/// Compute the Frobenius norm of the matrix
+template<int H, int W, typename T>
+MFEM_HOST_DEVICE inline
+double FNorm(const T *data)
+{
+   double s, n2;
+   kernels::FNorm<H,W>(s, n2, data);
+   return s*sqrt(n2);
+}
+
+/// Compute the square of the Frobenius norm of the matrix
+template<int H, int W, typename T>
+MFEM_HOST_DEVICE inline
+double FNorm2(const T *data)
+{
+   double s, n2;
+   kernels::FNorm<H,W>(s, n2, data);
+   return s*s*n2;
+}
+
 /// Returns the l2 norm of the Vector with given @a size and @a data.
 template<typename T>
 MFEM_HOST_DEVICE inline
@@ -123,8 +215,8 @@ MFEM_HOST_DEVICE inline T Det(const T *data)
    return TDetHD<T>(ColumnMajorLayout2D<dim,dim>(), data);
 }
 
-/** @brief Return the inverse a matrix with given @a size and @a data into the
-    matrix with data @a inv_data. */
+/** @brief Return the inverse of a matrix with given @a size and @a data into
+   the matrix with data @a inv_data. */
 template<int dim, typename T>
 MFEM_HOST_DEVICE inline
 void CalcInverse(const T *data, T *inv_data)
@@ -134,6 +226,15 @@ void CalcInverse(const T *data, T *inv_data)
    TAssignHD<AssignOp::Mult>(layout_t(), inv_data, static_cast<T>(1.0)/det);
 }
 
+/** @brief Return the adjugate of a matrix */
+template<int dim, typename T>
+MFEM_HOST_DEVICE inline
+void CalcAdjugate(const T *data, T *adj_data)
+{
+   typedef ColumnMajorLayout2D<dim,dim> layout_t;
+   TAdjugateHD<T>(layout_t(), data, layout_t(), adj_data);
+}
+
 /** @brief Compute C = A + alpha*B, where the matrices A, B and C are of size @a
     height x @a width with data @a Adata, @a Bdata and @a Cdata. */
 template<typename TALPHA, typename TA, typename TB, typename TC>
@@ -151,6 +252,50 @@ void Add(const int height, const int width, const TALPHA alpha,
    }
 }
 
+/** @brief Compute C = alpha*A + beta*B, where the matrices A, B and C are of
+    size @a height x @a width with data @a Adata, @a Bdata and @a Cdata. */
+template<typename TALPHA, typename TBETA, typename TA, typename TB, typename TC>
+MFEM_HOST_DEVICE inline
+void Add(const int height, const int width,
+         const TALPHA alpha, const TA *Adata,
+         const TBETA beta, const TB *Bdata,
+         TC *Cdata)
+{
+   const int m = height * width;
+   for (int i = 0; i < m; i++)
+   {
+      Cdata[i] = alpha * Adata[i] + beta * Bdata[i];
+   }
+}
+
+/** @brief Compute B += A, where the matrices A and B are of size
+    @a height x @a width with data @a Adata and @a Bdata. */
+template<typename TA, typename TB>
+MFEM_HOST_DEVICE inline
+void Add(const int height, const int width, const TA *Adata, TB *Bdata)
+{
+   const int m = height * width;
+   for (int i = 0; i < m; i++)
+   {
+      Bdata[i] += Adata[i];
+   }
+}
+
+/** @brief Compute B += alpha*A, where the matrices A and B are of size
+    @a height x @a width with data @a Adata and @a Bdata. */
+template<typename TA, typename TB>
+MFEM_HOST_DEVICE inline
+void Set(const int height, const int width,
+         const double alpha, const TA *Adata, TB *Bdata)
+{
+   const int m = height * width;
+   for (int i = 0; i < m; i++)
+   {
+      Bdata[i] = alpha * Adata[i];
+   }
+}
+
+
 /** @brief Matrix-matrix multiplication: A = B * C, where the matrices A, B and
     C are of sizes @a Aheight x @a Awidth, @a Aheight x @a Bwidth and @a Bwidth
     x @a Awidth, respectively. */
@@ -1419,4 +1564,4 @@ inline void LUSolve(const double *data, const int m, const int *ipiv,
 
 } // namespace mfem
 
-#endif // MFEM_KERNELS_HPP
+#endif // MFEM_LINALG_KERNELS_HPP

linalg/operator.cpp

@@ -597,7 +597,10 @@ double PowerMethod::EstimateLargestEigenvalue(Operator& opr, Vector& v0,
                                               int numSteps, double tolerance, int seed)
 {
    v1.SetSize(v0.Size());
-   v0.Randomize(seed);
+   if (seed != -1)
+   {
+      v0.Randomize(seed);
+   }
 
    double eigenvalue = 1.0;
 

linalg/operator.hpp

@@ -931,7 +931,8 @@ public:
        the eigenvector corresponding to the largest eigenvalue after convergence.
        The maximum number of iterations may set with \p numSteps, the relative
        tolerance with \p tolerance and the seed of the random initialization of
-       \p v0 with \p seed. */
+       \p v0 with \p seed, unless \p is -1, in which case \p v0 will be used
+       as-is. */
    double EstimateLargestEigenvalue(Operator& opr, Vector& v0,
                                     int numSteps = 10, double tolerance = 1e-8,
                                     int seed = 12345);

linalg/solvers.cpp

@@ -13,6 +13,7 @@
 #include "../general/forall.hpp"
 #include "../general/globals.hpp"
 #include "../fem/bilinearform.hpp"
+#include "../fem/nonlinearform.hpp"
 #include <iostream>
 #include <iomanip>
 #include <algorithm>
@@ -132,6 +133,19 @@ OperatorJacobiSmoother::OperatorJacobiSmoother(const BilinearForm &a,
    Setup(diag);
 }
 
+// In this case oper and will be set by SetOperator().
+OperatorJacobiSmoother::OperatorJacobiSmoother(const NonlinearForm &nlform,
+                                               const Array<int> &ess_tdofs,
+                                               const double dmpng)
+   :
+   Solver(nlform.FESpace()->GetTrueVSize()),
+   N(height),
+   dinv(N),
+   damping(dmpng),
+   ess_tdof_list(ess_tdofs),
+   residual(N),
+   dynamic(true), oper(NULL), nlf(&nlform) { }
+
 OperatorJacobiSmoother::OperatorJacobiSmoother(const Vector &d,
                                                const Array<int> &ess_tdofs,
                                                const double dmpng)
@@ -146,13 +160,29 @@ OperatorJacobiSmoother::OperatorJacobiSmoother(const Vector &d,
    Setup(d);
 }
 
+void OperatorJacobiSmoother::SetOperator(const Operator &op)
+{
+   oper = &op;
+
+   if (dynamic)
+   {
+      Vector diag(N);
+      nlf->AssembleGradientDiagonal(diag);
+      Setup(diag);
+   }
+}
+
 void OperatorJacobiSmoother::Setup(const Vector &diag)
 {
    residual.UseDevice(true);
    const double delta = damping;
    auto D = diag.Read();
    auto DI = dinv.Write();
-   MFEM_FORALL(i, N, DI[i] = delta / D[i]; );
+   MFEM_FORALL(i, N,
+   {
+      const double d_i = (D[i] < 0.0 && abs_values) ? -D[i] : D[i];
+      DI[i] = delta / d_i;
+   });
    auto I = ess_tdof_list.Read();
    MFEM_FORALL(i, ess_tdof_list.Size(), DI[I[i]] = delta; );
 }
@@ -536,7 +566,6 @@ void CGSolver::Mult(const Vector &b, Vector &x) const
 {
    int i;
    double r0, den, nom, nom0, betanom, alpha, beta;
-
    if (iterative_mode)
    {
       oper->Mult(x, r);

linalg/solvers.hpp

@@ -29,6 +29,7 @@ namespace mfem
 {
 
 class BilinearForm;
+class NonlinearForm;
 
 /// Abstract base class for an iterative solver monitor
 class IterativeSolverMonitor
@@ -119,6 +120,10 @@ public:
                           const Array<int> &ess_tdof_list,
                           const double damping=1.0);
 
+   OperatorJacobiSmoother(const NonlinearForm &nlform,
+                          const Array<int> &ess_tdof_list,
+                          const double damping=1.0);
+
    /** Application is by the *inverse* of the given vector. It is assumed that
        the underlying operator acts as the identity on entries in ess_tdof_list,
        corresponding to (assembled) DIAG_ONE policy or ConstrainedOperator in
@@ -126,12 +131,15 @@ public:
    OperatorJacobiSmoother(const Vector &d,
                           const Array<int> &ess_tdof_list,
                           const double damping=1.0);
+
    ~OperatorJacobiSmoother() {}
 
    void Mult(const Vector &x, Vector &y) const;
    void MultTranspose(const Vector &x, Vector &y) const { Mult(x, y); }
-   void SetOperator(const Operator &op) { oper = &op; }
+
+   void SetOperator(const Operator &op);
    void Setup(const Vector &diag);
+   void SetAbsValues(bool use_abs_values) { abs_values = use_abs_values; }
 
 private:
    const int N;
@@ -139,8 +147,11 @@ private:
    const double damping;
    const Array<int> &ess_tdof_list;
    mutable Vector residual;
+   const bool dynamic = false;
+   bool abs_values = false;
 
    const Operator *oper;
+   const NonlinearForm *nlf = NULL;
 };
 
 /// Chebyshev accelerated smoothing with given vector, no matrix necessary

linalg/sparsemat.cpp

@@ -28,6 +28,38 @@ namespace mfem
 
 using namespace std;
 
+#ifdef MFEM_USE_CUDA
+int SparseMatrix::SparseMatrixCount = 0;
+cusparseHandle_t SparseMatrix::handle;
+size_t SparseMatrix::bufferSize = 0;
+void * SparseMatrix::dBuffer = nullptr;
+#endif
+
+void SparseMatrix::InitCuSparse()
+{
+   /* Initialize CuSparse library */
+#ifdef MFEM_USE_CUDA
+   SparseMatrixCount++;
+   if (SparseMatrixCount == 1 && Device::Allows(Backend::CUDA_MASK))
+   {
+      cusparseCreate(&handle);
+   }
+#endif
+}
+
+void SparseMatrix::ClearCuSparseDescr()
+{
+#ifdef MFEM_USE_CUDA
+   if (initBuffers)
+   {
+      cusparseDestroySpMat(matA_descr);
+      cusparseDestroyDnVec(vecX_descr);
+      cusparseDestroyDnVec(vecY_descr);
+      initBuffers = false;
+   }
+#endif
+}
+
 SparseMatrix::SparseMatrix(int nrows, int ncols)
    : AbstractSparseMatrix(nrows, (ncols >= 0) ? ncols : nrows),
      Rows(new RowNode *[nrows]),
@@ -50,6 +82,8 @@ SparseMatrix::SparseMatrix(int nrows, int ncols)
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = new RowNodeAlloc;
 #endif
+
+   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
@@ -67,6 +101,8 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = NULL;
 #endif
+
+   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
@@ -98,6 +134,8 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
          A[i] = 0.0;
       }
    }
+
+   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
@@ -119,9 +157,12 @@ SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
    {
       I[i] = i * rowsize;
    }
+
+   InitCuSparse();
 }
 
-SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
+SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph,
+                           MemoryType mt)
    : AbstractSparseMatrix(mat.Height(), mat.Width())
 {
    if (mat.Finalized())
@@ -129,8 +170,8 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
       const int nnz = mat.I[height];
       if (copy_graph)
       {
-         I.New(height+1, mat.I.GetMemoryType());
-         J.New(nnz, mat.J.GetMemoryType());
+         I.New(height+1, mt == MemoryType::SIZE ? mat.I.GetMemoryType() : mt);
+         J.New(nnz, mt == MemoryType::SIZE ? mat.J.GetMemoryType() : mt);
          I.CopyFrom(mat.I, height+1);
          J.CopyFrom(mat.J, nnz);
       }
@@ -141,7 +182,7 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
          I.ClearOwnerFlags();
          J.ClearOwnerFlags();
       }
-      A.New(nnz, mat.A.GetMemoryType());
+      A.New(nnz, mt == MemoryType::SIZE ? mat.A.GetMemoryType() : mt);
       A.CopyFrom(mat.A, nnz);
 
       Rows = NULL;
@@ -174,9 +215,21 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
       }
 
       // We probably do not need to set the ownership flags here.
-      I.Reset(); I.SetHostPtrOwner(true);
-      J.Reset(); J.SetHostPtrOwner(true);
-      A.Reset(); A.SetHostPtrOwner(true);
+      if (mt != MemoryType::SIZE)
+      {
+         I.Reset(mt);
+         J.Reset(mt);
+         A.Reset(mt);
+      }
+      else
+      {
+         I.Reset();
+         J.Reset();
+         A.Reset();
+      }
+      I.SetHostPtrOwner(true);
+      J.SetHostPtrOwner(true);
+      A.SetHostPtrOwner(true);
    }
 
    current_row = -1;
@@ -184,6 +237,8 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
    ColPtrNode = NULL;
    At = NULL;
    isSorted = mat.isSorted;
+
+   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(const Vector &v)
@@ -211,6 +266,8 @@ SparseMatrix::SparseMatrix(const Vector &v)
       J[r] = r;
       A[r] = v[r];
    }
+
+   InitCuSparse();
 }
 
 SparseMatrix& SparseMatrix::operator=(const SparseMatrix &rhs)
@@ -250,6 +307,8 @@ void SparseMatrix::SetEmpty()
    NodesMem = NULL;
 #endif
    isSorted = false;
+
+   ClearCuSparseDescr();
 }
 
 int SparseMatrix::RowSize(const int i) const
@@ -494,9 +553,9 @@ void SparseMatrix::GetDiag(Vector & d) const
 
    d.SetSize(height);
 
-   auto I = this->ReadI();
-   auto J = this->ReadJ();
-   auto A = this->ReadData();
+   const auto I = this->ReadI();
+   const auto J = this->ReadJ();
+   const auto A = this->ReadData();
    auto dd = d.Write();
 
    MFEM_FORALL(i, height,
@@ -592,16 +651,72 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
    auto d_A = Read(A, nnz);
    auto d_x = x.Read();
    auto d_y = y.ReadWrite();
-   MFEM_FORALL(i, height,
+
+   //Skip if matrix has no non-zeros
+   if (nnz == 0) {return;}
+   if (Device::Allows(Backend::CUDA_MASK) && useCuSparse)
    {
-      double d = 0.0;
-      const int end = d_I[i+1];
-      for (int j = d_I[i]; j < end; j++)
+#ifdef MFEM_USE_CUDA
+      const double alpha = a;
+      const double beta  = 1.0;
+
+      //Setup descriptors
+      if (!initBuffers)
       {
-         d += d_A[j] * d_x[d_J[j]];
+         /* Setup matrix descriptor */
+         cusparseCreateCsr(&matA_descr,Height(), Width(), J.Capacity(),
+                           const_cast<int *>(d_I),
+                           const_cast<int *>(d_J), const_cast<double *>(d_A), CUSPARSE_INDEX_32I,
+                           CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F);
+
+         /*Create handles for input/output vectors */
+         cusparseCreateDnVec(&vecX_descr, x.Size(), const_cast<double *>(d_x),
+                             CUDA_R_64F);
+         cusparseCreateDnVec(&vecY_descr, y.Size(), d_y, CUDA_R_64F);
+
+         initBuffers = true;
       }
-      d_y[i] += a * d;
-   });
+
+      /*Allocate space for kernel. Buffer is shared between different sparsemats */
+      size_t newBufferSize = 0;
+      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
+                              matA_descr,
+                              vecX_descr, &beta, vecY_descr, CUDA_R_64F,
+                              CUSPARSE_CSRMV_ALG1, &newBufferSize);
+
+      //Check if need to resize
+      if (newBufferSize > bufferSize)
+      {
+         bufferSize = newBufferSize;
+         if (dBuffer != NULL) { CuMemFree(dBuffer); }
+         CuMemAlloc(&dBuffer, bufferSize);
+      }
+
+      //Update input/output vectors
+      cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
+      cusparseDnVecSetValues(vecY_descr, d_y);
+
+      // Y = alpha A * X + beta * Y
+      cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA_descr,
+                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, CUSPARSE_CSRMV_ALG1, dBuffer);
+#endif
+   }
+   else
+   {
+      //Native version
+      MFEM_FORALL(i, height,
+      {
+         double d = 0.0;
+         const int end = d_I[i+1];
+         for (int j = d_I[i]; j < end; j++)
+         {
+            d += d_A[j] * d_x[d_J[j]];
+         }
+         d_y[i] += a * d;
+      });
+
+   }
+
 #else
    const double *Ap = A, *xp = x.GetData();
    double *yp = y.GetData();
@@ -2150,21 +2265,21 @@ void SparseMatrix::DiagScale(const Vector &b, Vector &x, double sc) const
 {
    MFEM_VERIFY(Finalized(), "Matrix must be finalized.");
 
+   const int H = height;
    const int nnz = J.Capacity();
-
+   const bool scale = (sc != 1.0);
    const bool use_dev = b.UseDevice() || x.UseDevice();
 
-   auto bp = b.Read(use_dev);
+   const auto Ap = Read(A, nnz, use_dev);
+   const auto Ip = Read(I, height+1, use_dev);
+   const auto Jp = Read(J, nnz, use_dev);
+
+   const auto bp = b.Read(use_dev);
    auto xp = x.Write(use_dev);
 
-   auto Ap = Read(A, nnz);
-   auto Ip = Read(I, height+1);
-   auto Jp = Read(J, nnz);
-
-   bool scale = (sc != 1.0);
-   MFEM_FORALL(i, height,
+   MFEM_FORALL_SWITCH(use_dev, i, H,
    {
-      int end = Ip[i+1];
+      const int end = Ip[i+1];
       for (int j = Ip[i]; true; j++)
       {
          if (j == end)
@@ -2177,20 +2292,12 @@ void SparseMatrix::DiagScale(const Vector &b, Vector &x, double sc) const
             {
                MFEM_ABORT_KERNEL("Zero diagonal in SparseMatrix::DiagScale");
             }
-
-            if (scale)
-            {
-               xp[i] = sc * bp[i] / Ap[j];
-            }
-            else
-            {
-               xp[i] = bp[i] / Ap[j];
-            }
+            const double s = scale ? sc : 1.0;
+            xp[i] = s * bp[i] / Ap[j];
             break;
          }
       }
    });
-   return;
 }
 
 void SparseMatrix::Jacobi2(const Vector &b, const Vector &x0, Vector &x1,
@@ -2697,6 +2804,8 @@ SparseMatrix &SparseMatrix::operator=(double a)
 {
    if (Rows == NULL)
    {
+      HostReadWriteI();
+      HostReadWriteJ();
       const int nnz = J.Capacity();
       double *h_A = HostWrite(A, nnz);
       for (int i = 0; i < nnz; i++)
@@ -2962,6 +3071,10 @@ void SparseMatrix::Destroy()
    delete NodesMem;
 #endif
    delete At;
+
+#ifdef MFEM_USE_CUDA
+   ClearCuSparseDescr();
+#endif
 }
 
 int SparseMatrix::ActualWidth() const

linalg/sparsemat.hpp

@@ -21,6 +21,12 @@
 #include "../general/globals.hpp"
 #include "densemat.hpp"
 
+#ifdef MFEM_USE_CUDA
+#include <cusparse.h>
+#include <library_types.h>
+#include "../general/cuda.hpp"
+#endif
+
 namespace mfem
 {
 
@@ -80,9 +86,34 @@ protected:
    void Destroy();   // Delete all owned data
    void SetEmpty();  // Init all entries with empty values
 
+   bool useCuSparse{true}; //Use CuSparse if available
+
+   // Initialize CuSparse
+   void InitCuSparse();
+
+
+#ifdef MFEM_USE_CUDA
+   cusparseStatus_t status;
+   static cusparseHandle_t handle;
+   cusparseMatDescr_t descr=0;
+   static size_t bufferSize;
+   static void *dBuffer;
+   mutable bool initBuffers{false};
+
+   static int SparseMatrixCount;
+   mutable cusparseSpMatDescr_t matA_descr;
+   mutable cusparseDnVecDescr_t vecX_descr;
+   mutable cusparseDnVecDescr_t vecY_descr;
+#endif
+
 public:
    /// Create an empty SparseMatrix.
-   SparseMatrix() { SetEmpty(); }
+   SparseMatrix()
+   {
+      SetEmpty();
+
+      InitCuSparse();
+   }
 
    /** @brief Create a sparse matrix with flexible sparsity structure using a
        row-wise linked list (LIL) format. */
@@ -113,11 +144,15 @@ public:
    /** If @a mat is finalized and @a copy_graph is false, the #I and #J arrays
        will use a shallow copy (copy the pointers only) without transferring
        ownership. */
-   SparseMatrix(const SparseMatrix &mat, bool copy_graph = true);
+   SparseMatrix(const SparseMatrix &mat, bool copy_graph = true,
+                MemoryType mt = MemoryType::SIZE);
 
    /// Create a SparseMatrix with diagonal @a v, i.e. A = Diag(v)
    SparseMatrix(const Vector & v);
 
+   // Runtime option to use CuSparse
+   // Only valid when using a CUDA backend
+   void UseCuSparse(bool _useCuSparse = true) { useCuSparse = _useCuSparse;}
 
    /// Assignment operator: deep copy
    SparseMatrix& operator=(const SparseMatrix &rhs);
@@ -134,6 +169,9 @@ public:
    /// Clear the contents of the SparseMatrix.
    void Clear() { Destroy(); SetEmpty(); }
 
+   /// Release the cusparse descriptors.
+   void ClearCuSparseDescr();
+
    /// Check if the SparseMatrix is empty.
    bool Empty() const { return (A == NULL) && (Rows == NULL); }
 
@@ -573,7 +611,18 @@ public:
    void Swap(SparseMatrix &other);
 
    /// Destroys sparse matrix.
-   virtual ~SparseMatrix() { Destroy(); }
+   virtual ~SparseMatrix()
+   {
+      Destroy();
+#ifdef MFEM_USE_CUDA
+      if (handle && SparseMatrixCount==1 && Device::Allows(Backend::CUDA_MASK))
+      {
+         cusparseDestroy(handle);
+         CuMemFree(dBuffer);
+      }
+      SparseMatrixCount--;
+#endif
+   }
 
    Type GetType() const { return MFEM_SPARSEMAT; }
 };

linalg/tmatrix.hpp

@@ -626,6 +626,21 @@ inline void TAdjugate(const A_layout_t &a, const A_data_t &A,
    template Adjugate<scalar_t>(a, A, b, B);
 }
 
+// Compute the adjugate matrix of a (small) matrix: B = adj(A).
+// Host+device version.
+template <typename scalar_t,
+          typename A_layout_t, typename A_data_t,
+          typename B_layout_t, typename B_data_t>
+MFEM_HOST_DEVICE
+inline void TAdjugateHD(const A_layout_t &a, const A_data_t &A,
+                        const B_layout_t &b, B_data_t &B)
+{
+   MFEM_STATIC_ASSERT(A_layout_t::rank == 2 && B_layout_t::rank == 2,
+                      "invalid ranks");
+   internal::MatrixOps<A_layout_t::dim_1,A_layout_t::dim_2>::
+   template AdjugateHD<scalar_t>(a, A, b, B);
+}
+
 // Compute the adjugate and the determinant of a (small) matrix: B = adj(A),
 // return det(A).
 template <typename scalar_t,

linalg/vector.cpp

@@ -747,6 +747,7 @@ double Vector::Norml2() const
 
 double Vector::Normlinf() const
 {
+   HostRead();
    double max = 0.0;
    for (int i = 0; i < size; i++)
    {
@@ -757,6 +758,7 @@ double Vector::Normlinf() const
 
 double Vector::Norml1() const
 {
+   HostRead();
    double sum = 0.0;
    for (int i = 0; i < size; i++)
    {
@@ -819,6 +821,7 @@ double Vector::Max() const
 {
    if (size == 0) { return -infinity(); }
 
+   HostRead();
    double max = data[0];
 
    for (int i = 1; i < size; i++)
@@ -1026,6 +1029,7 @@ static double hipVectorDot(const int N, const double *X, const double *Y)
 double Vector::operator*(const Vector &v) const
 {
    MFEM_ASSERT(size == v.size, "incompatible Vectors!");
+   if (size == 0) { return 0.0; }
 
    const bool use_dev = UseDevice() || v.UseDevice();
 #if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) || defined(MFEM_USE_OPENMP)

makefile

@@ -261,7 +261,7 @@ endif
 # List of MFEM dependencies, that require the *_LIB variable to be non-empty
 MFEM_REQ_LIB_DEPS = SUPERLU METIS CONDUIT SIDRE LAPACK SUNDIALS MESQUITE\
  SUITESPARSE STRUMPACK GINKGO GNUTLS NETCDF PETSC SLEPC MPFR PUMI HIOP GSLIB\
- OCCA CEED RAJA UMPIRE
+ OCCA CEED RAJA UMPIRE CAMP
 PETSC_ERROR_MSG = $(if $(PETSC_FOUND),,. PETSC config not found: $(PETSC_VARS))
 SLEPC_ERROR_MSG = $(if $(SLEPC_FOUND),,. SLEPC config not found: $(SLEPC_VARS))
 
@@ -324,8 +324,8 @@ MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
  MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS\
  MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT\
  MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP\
- MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD\
- MFEM_USE_ADIOS2 MFEM_SOURCE_DIR MFEM_INSTALL_DIR
+ MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_CAMP\
+ MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_SOURCE_DIR MFEM_INSTALL_DIR
 
 # List of makefile variables that will be written to config.mk:
 MFEM_CONFIG_VARS = MFEM_CXX MFEM_HOST_CXX MFEM_CPPFLAGS MFEM_CXXFLAGS\
@@ -645,6 +645,7 @@ status info:
 	$(info MFEM_USE_OCCA          = $(MFEM_USE_OCCA))
 	$(info MFEM_USE_CEED          = $(MFEM_USE_CEED))
 	$(info MFEM_USE_UMPIRE        = $(MFEM_USE_UMPIRE))
+	$(info MFEM_USE_CAMP          = $(MFEM_USE_CAMP))
 	$(info MFEM_USE_SIMD          = $(MFEM_USE_SIMD))
 	$(info MFEM_USE_ADIOS2        = $(MFEM_USE_ADIOS2))
 	$(info MFEM_CXX               = $(value MFEM_CXX))

mesh/mesh.cpp

@@ -754,7 +754,10 @@ void Mesh::GetLocalQuadToWdgTransformation(
 }
 
 const GeometricFactors* Mesh::GetGeometricFactors(const IntegrationRule& ir,
-                                                  const int flags)
+                                                  const int flags,
+                                                  mfem::DofToQuad::Mode mode,
+                                                  MemoryType d_mt
+                                               )
 {
    for (int i = 0; i < geom_factors.Size(); i++)
    {
@@ -766,8 +769,7 @@ const GeometricFactors* Mesh::GetGeometricFactors(const IntegrationRule& ir,
    }
 
    this->EnsureNodes();
-
-   GeometricFactors *gf = new GeometricFactors(this, ir, flags);
+   GeometricFactors *gf = new GeometricFactors(this, ir, flags, mode, d_mt);
    geom_factors.Append(gf);
    return gf;
 }
@@ -5825,6 +5827,7 @@ int *Mesh::GeneratePartitioning(int nparts, int part_method)
    el_to_el = NULL;
 
    // Check for empty partitionings (a "feature" in METIS)
+   if (nparts > 1 && NumOfElements > nparts)
    {
       Array< Pair<int,int> > psize(nparts);
       int empty_parts;
@@ -10511,7 +10514,7 @@ int Mesh::FindPoints(DenseMatrix &point_mat, Array<int>& elem_ids,
 
 
 GeometricFactors::GeometricFactors(const Mesh *mesh, const IntegrationRule &ir,
-                                   int flags)
+                                   int flags, DofToQuad::Mode mode, MemoryType d_mt)
 {
    this->mesh = mesh;
    IntRule = &ir;
@@ -10526,34 +10529,101 @@ GeometricFactors::GeometricFactors(const Mesh *mesh, const IntegrationRule &ir,
    const int ND   = fe->GetDof();
    const int NQ   = ir.GetNPoints();
 
-   // For now, we are not using tensor product evaluation
-   const Operator *elem_restr = fespace->GetElementRestriction(
-                                   ElementDofOrdering::NATIVE);
-
    unsigned eval_flags = 0;
+   MemoryType my_d_mt = (d_mt != MemoryType::DEFAULT) ? d_mt :
+                         Device::GetDeviceMemoryType();
    if (flags & GeometricFactors::COORDINATES)
    {
-      X.SetSize(vdim*NQ*NE);
+      X.SetSize(vdim*NQ*NE, my_d_mt);
       eval_flags |= QuadratureInterpolator::VALUES;
    }
    if (flags & GeometricFactors::JACOBIANS)
    {
-      J.SetSize(dim*vdim*NQ*NE);
+      J.SetSize(dim*vdim*NQ*NE, my_d_mt);
       eval_flags |= QuadratureInterpolator::DERIVATIVES;
    }
    if (flags & GeometricFactors::DETERMINANTS)
    {
-      detJ.SetSize(NQ*NE);
+      detJ.SetSize(NQ*NE, my_d_mt);
       eval_flags |= QuadratureInterpolator::DETERMINANTS;
    }
 
-   const QuadratureInterpolator *qi = fespace->GetQuadratureInterpolator(ir);
-   // For now, we are not using tensor product evaluation (not implemented)
-   qi->DisableTensorProducts();
+   const QuadratureInterpolator *qi =
+      fespace->GetQuadratureInterpolator(ir, mode);
+   const bool use_tensor_products = qi->UseTensorProducts();
+
+   // GeometricFactors arrays use a column-major layout
    qi->SetOutputLayout(QVectorLayout::byNODES);
+
+   // Use LEXICOGRAPHIC ordering in case of tensor product evaluation
+   const ElementDofOrdering e_ordering = use_tensor_products ?
+                                         ElementDofOrdering::LEXICOGRAPHIC :
+                                         ElementDofOrdering::NATIVE;
+   const Operator *elem_restr = fespace->GetElementRestriction(e_ordering);
+
    if (elem_restr)
    {
-      Vector Enodes(vdim*ND*NE);
+      Vector Enodes(vdim*ND*NE, my_d_mt);
+      elem_restr->Mult(*nodes, Enodes);
+      qi->Mult(Enodes, eval_flags, X, J, detJ);
+   }
+   else
+   {
+      qi->Mult(*nodes, eval_flags, X, J, detJ);
+   }
+}
+
+GeometricFactors::GeometricFactors(const GridFunction *nodes_,
+                                   const IntegrationRule &ir,
+                                   int flags, DofToQuad::Mode mode)
+{
+   this->mesh = nullptr;
+   IntRule = &ir;
+   computed_factors = flags;
+
+   const GridFunction *nodes = nodes_;
+   const FiniteElementSpace *fespace = nodes->FESpace();
+   const FiniteElement *fe = fespace->GetFE(0);
+   const int dim  = fe->GetDim();
+   const int vdim = fespace->GetVDim();
+   const int NE   = fespace->GetNE();
+   const int ND   = fe->GetDof();
+   const int NQ   = ir.GetNPoints();
+
+   unsigned eval_flags = 0;
+   MemoryType d_mt = Device::GetDeviceMemoryType();
+   if (flags & GeometricFactors::COORDINATES)
+   {
+      X.SetSize(vdim*NQ*NE, d_mt);
+      eval_flags |= QuadratureInterpolator::VALUES;
+   }
+   if (flags & GeometricFactors::JACOBIANS)
+   {
+      J.SetSize(dim*vdim*NQ*NE, d_mt);
+      eval_flags |= QuadratureInterpolator::DERIVATIVES;
+   }
+   if (flags & GeometricFactors::DETERMINANTS)
+   {
+      detJ.SetSize(NQ*NE, d_mt);
+      eval_flags |= QuadratureInterpolator::DETERMINANTS;
+   }
+
+   const QuadratureInterpolator *qi =
+      fespace->GetQuadratureInterpolator(ir, mode);
+   const bool use_tensor_products = qi->UseTensorProducts();
+
+   // GeometricFactors arrays use a column-major layout
+   qi->SetOutputLayout(QVectorLayout::byNODES);
+
+   // Use LEXICOGRAPHIC ordering in case of tensor product evaluation
+   const ElementDofOrdering e_ordering = use_tensor_products ?
+                                         ElementDofOrdering::LEXICOGRAPHIC :
+                                         ElementDofOrdering::NATIVE;
+   const Operator *elem_restr = fespace->GetElementRestriction(e_ordering);
+
+   if (elem_restr)
+   {
+      Vector Enodes(vdim*ND*NE, d_mt);
       elem_restr->Mult(*nodes, Enodes);
       qi->Mult(Enodes, eval_flags, X, J, detJ);
    }

mesh/mesh.hpp

@@ -742,7 +742,9 @@ public:
    /** @brief Return the mesh geometric factors corresponding to the given
        integration rule. */
    const GeometricFactors* GetGeometricFactors(const IntegrationRule& ir,
-                                               const int flags);
+                                               const int flags,
+                                               DofToQuad::Mode = DofToQuad::FULL,
+                                               MemoryType d_mt = MemoryType::DEFAULT);
 
    /** @brief Return the mesh geometric factors for the faces corresponding
         to the given integration rule. */
@@ -1372,7 +1374,13 @@ public:
       DETERMINANTS = 1 << 2,
    };
 
-   GeometricFactors(const Mesh *mesh, const IntegrationRule &ir, int flags);
+   GeometricFactors(const Mesh *mesh, const IntegrationRule &ir, int flags,
+                    DofToQuad::Mode = DofToQuad::FULL,
+                    MemoryType d_mt = MemoryType::DEFAULT);
+
+   GeometricFactors(const GridFunction *nodes_, const IntegrationRule &ir,
+                    int flags,
+                    DofToQuad::Mode = DofToQuad::FULL);
 
    /// Mapped (physical) coordinates of all quadrature points.
    /** This array uses a column-major layout with dimensions (NQ x SDIM x NE)

mesh/pmesh.cpp

@@ -2394,9 +2394,18 @@ void ParMesh::GetGhostFaceTransformation(
 }
 
 FaceElementTransformations *ParMesh::
-GetSharedFaceTransformations(int sf, bool fill2)
+GetSharedFaceTransformations(int sf, bool fill2, bool direct)
 {
-   int FaceNo = GetSharedFace(sf);
+   //int FaceNo = GetSharedFace(sf);
+   int FaceNo;
+   if (direct)
+   {
+      FaceNo = sf;
+   }
+   else
+   {
+      FaceNo = GetSharedFace(sf);
+   }
 
    FaceInfo &face_info = faces_info[FaceNo];
 
@@ -4589,6 +4598,7 @@ void ParMesh::PrintAsOne(std::ostream &out)
       ParGridFunction *pnodes = dynamic_cast<ParGridFunction *>(Nodes);
       if (pnodes)
       {
+         pnodes->HostReadWrite();
          pnodes->SaveAsOne(out);
       }
       else
@@ -4599,6 +4609,7 @@ void ParMesh::PrintAsOne(std::ostream &out)
          {
             // create a wrapper ParGridFunction
             ParGridFunction ParNodes(pfes, Nodes);
+            ParNodes.HostRead();
             ParNodes.SaveAsOne(out);
          }
          else

mesh/pmesh.hpp

@@ -293,9 +293,9 @@ public:
 
    /** Get the FaceElementTransformations for the given shared face (edge 2D).
        In the returned object, 1 and 2 refer to the local and the neighbor
-       elements, respectively. */
+       elements, respectively. Use direct if sf is the face number */
    FaceElementTransformations *
-   GetSharedFaceTransformations(int sf, bool fill2 = true);
+   GetSharedFaceTransformations(int sf, bool fill2 = true, bool direct = false);
 
    ElementTransformation *
    GetFaceNbrElementTransformation(int i)

miniapps/meshing/mesh-optimizer.cpp

@@ -31,6 +31,9 @@
 //
 // Compile with: make mesh-optimizer
 //
+//  Preconditioner run:
+// ./mesh-optimizer -m blade.mesh -o 4 -rs 0 -mid 2 -tid 1 -ni 200 -bnd -qt 1 -qo 8 -ls 3 -nor -lc 0.1 -vl 2 -pa -d cpu
+//
 // Sample runs:
 //   Adapted analytic shape:
 //     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 2 -tid 4 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
@@ -52,6 +55,12 @@
 //   Adapted discrete aspect ratio (3D):
 //     mesh-optimizer -m cube.mesh -o 2 -rs 2 -mid 302 -tid 7 -ni 20  -ls 2 -li 100 -bnd -qt 1 -qo 8
 //
+//   Adapted discrete size (3D):
+//     mesh-optimizer -m cube.mesh -o 2 -rs 2 -mid 321 -tid 5 -ni 20  -ls 2 -li 100 -bnd -qt 1 -qo 8 -nor
+//   Adapted discrete shape+size explicit combo (3D):
+//     mesh-optimizer -m cube.mesh -o 2 -rs 2 -mid 302 -tid 5 -ni 20  -ls 2 -li 100 -bnd -qt 1 -qo 8 -cmb 2 -nor
+//
+//
 //   Adaptive limiting:
 //     mesh-optimizer -m stretched2D.mesh -o 2 -mid 2 -tid 1 -ni 50 -qo 5 -nor -vl 1 -alc 0.5
 //   Adaptive limiting through the L-BFGS solver:
@@ -83,6 +92,7 @@
 
 #include "mfem.hpp"
 #include "../common/mfem-common.hpp"
+#include <cfloat>
 #include <fstream>
 #include <iostream>
 #include "mesh-optimizer.hpp"
@@ -116,6 +126,8 @@ int main(int argc, char *argv[])
    bool fdscheme         = false;
    int adapt_eval        = 0;
    bool exactaction      = false;
+   const char *devopt    = "cpu";
+   bool pa               = false;
 
    // 1. Parse command-line options.
    OptionsParser args(argc, argv);
@@ -173,7 +185,12 @@ int main(int argc, char *argv[])
    args.AddOption(&solver_rtol, "-rtol", "--newton-rel-tolerance",
                   "Relative tolerance for the Newton solver.");
    args.AddOption(&lin_solver, "-ls", "--lin-solver",
-                  "Linear solver: 0 - l1-Jacobi, 1 - CG, 2 - MINRES.");
+                  "Linear solver:\n\t"
+                  "0: l1-Jacobi\n\t"
+                  "1: CG\n\t"
+                  "2: MINRES\n\t"
+                  "3: MINRES + Jacobi preconditioner"
+                  "4: MINRES + l1-Jacobi preconditioner");
    args.AddOption(&max_lin_iter, "-li", "--lin-iter",
                   "Maximum number of iterations in the linear solve.");
    args.AddOption(&move_bnd, "-bnd", "--move-boundary", "-fix-bnd",
@@ -200,6 +217,10 @@ int main(int argc, char *argv[])
                   "Set the verbosity level - 0, 1, or 2.");
    args.AddOption(&adapt_eval, "-ae", "--adaptivity-evaluator",
                   "0 - Advection based (DEFAULT), 1 - GSLIB.");
+   args.AddOption(&devopt, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
    args.Parse();
    if (!args.Good())
    {
@@ -208,6 +229,9 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
+   Device device(devopt);
+   device.Print();
+
    // 2. Initialize and refine the starting mesh.
    Mesh *mesh = new Mesh(mesh_file, 1, 1, false);
    for (int lev = 0; lev < rs_levels; lev++) { mesh->UniformRefinement(); }
@@ -276,6 +300,7 @@ int main(int argc, char *argv[])
    rdm.Randomize();
    rdm -= 0.25; // Shift to random values in [-0.5,0.5].
    rdm *= jitter;
+   rdm.HostReadWrite();
    // Scale the random values to be of order of the local mesh size.
    for (int i = 0; i < fespace->GetNDofs(); i++)
    {
@@ -342,7 +367,9 @@ int main(int argc, char *argv[])
    FiniteElementSpace ind_fes(mesh, &ind_fec);
    FiniteElementSpace ind_fesv(mesh, &ind_fec, dim);
    GridFunction size(&ind_fes), aspr(&ind_fes), disc(&ind_fes), ori(&ind_fes);
-   GridFunction aspr3d(&ind_fesv), size3d(&ind_fesv);
+   GridFunction aspr3d(&ind_fesv);
+   const AssemblyLevel al =
+      pa ? AssemblyLevel::PARTIAL : AssemblyLevel::LEGACYFULL;
    switch (target_id)
    {
       case 1: target_t = TargetConstructor::IDEAL_SHAPE_UNIT_SIZE; break;
@@ -357,13 +384,13 @@ int main(int argc, char *argv[])
          target_c = tc;
          break;
       }
-      case 5: // Discrete size 2D
+      case 5: // Discrete size 2D or 3D
       {
          target_t = TargetConstructor::IDEAL_SHAPE_GIVEN_SIZE;
          DiscreteAdaptTC *tc = new DiscreteAdaptTC(target_t);
          if (adapt_eval == 0)
          {
-            tc->SetAdaptivityEvaluator(new AdvectorCG);
+            tc->SetAdaptivityEvaluator(new AdvectorCG(al));
          }
          else
          {
@@ -373,8 +400,16 @@ int main(int argc, char *argv[])
             MFEM_ABORT("MFEM is not built with GSLIB.");
 #endif
          }
-         FunctionCoefficient ind_coeff(discrete_size_2d);
-         size.ProjectCoefficient(ind_coeff);
+         if (dim == 2)
+         {
+            FunctionCoefficient ind_coeff(discrete_size_2d);
+            size.ProjectCoefficient(ind_coeff);
+         }
+         else if (dim == 3)
+         {
+            FunctionCoefficient ind_coeff(discrete_size_3d);
+            size.ProjectCoefficient(ind_coeff);
+         }
          tc->SetSerialDiscreteTargetSize(size);
          target_c = tc;
          break;
@@ -389,7 +424,7 @@ int main(int argc, char *argv[])
          disc.ProjectCoefficient(ind_coeff);
          if (adapt_eval == 0)
          {
-            tc->SetAdaptivityEvaluator(new AdvectorCG);
+            tc->SetAdaptivityEvaluator(new AdvectorCG(al));
          }
          else
          {
@@ -480,7 +515,7 @@ int main(int argc, char *argv[])
          DiscreteAdaptTC *tc = new DiscreteAdaptTC(target_t);
          if (adapt_eval == 0)
          {
-            tc->SetAdaptivityEvaluator(new AdvectorCG);
+            tc->SetAdaptivityEvaluator(new AdvectorCG(al));
          }
          else
          {
@@ -503,7 +538,7 @@ int main(int argc, char *argv[])
          DiscreteAdaptTC *tc = new DiscreteAdaptTC(target_t);
          if (adapt_eval == 0)
          {
-            tc->SetAdaptivityEvaluator(new AdvectorCG);
+            tc->SetAdaptivityEvaluator(new AdvectorCG(al));
          }
          else
          {
@@ -543,7 +578,13 @@ int main(int argc, char *argv[])
    }
    target_c->SetNodes(x0);
    TMOP_Integrator *he_nlf_integ = new TMOP_Integrator(metric, target_c);
-   if (fdscheme) { he_nlf_integ->EnableFiniteDifferences(x); }
+
+   // Finite differences for computations of derivatives.
+   if (fdscheme)
+   {
+      MFEM_VERIFY(pa == false, "PA for finite differences is not imlemented.");
+      he_nlf_integ->EnableFiniteDifferences(x);
+   }
    he_nlf_integ->SetExactActionFlag(exactaction);
 
    // 12. Setup the quadrature rule for the non-linear form integrator.
@@ -577,10 +618,12 @@ int main(int argc, char *argv[])
    AdaptivityEvaluator *adapt_evaluator = NULL;
    if (adapt_lim_const > 0.0)
    {
+      MFEM_VERIFY(pa == false, "PA is not implemented for adaptive limiting");
+
       FunctionCoefficient alim_coeff(adapt_lim_fun);
       zeta_0.ProjectCoefficient(alim_coeff);
 
-      if (adapt_eval == 0) { adapt_evaluator = new AdvectorCG; }
+      if (adapt_eval == 0) { adapt_evaluator = new AdvectorCG(al); }
       else if (adapt_eval == 1)
       {
 #ifdef MFEM_USE_GSLIB
@@ -607,6 +650,7 @@ int main(int argc, char *argv[])
    //     command-line options for the weights and the type of the second
    //     metric; one should update those in the code.
    NonlinearForm a(fespace);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    ConstantCoefficient *coeff1 = NULL;
    TMOP_QualityMetric *metric2 = NULL;
    TargetConstructor *target_c2 = NULL;
@@ -619,7 +663,8 @@ int main(int argc, char *argv[])
       he_nlf_integ->SetCoefficient(*coeff1);
 
       // Second metric.
-      metric2 = new TMOP_Metric_077;
+      if (dim == 2) { metric2 = new TMOP_Metric_077; }
+      else          { metric2 = new TMOP_Metric_315; }
       TMOP_Integrator *he_nlf_integ2 = NULL;
       if (combomet == 1)
       {
@@ -643,7 +688,12 @@ int main(int argc, char *argv[])
 
       a.AddDomainIntegrator(combo);
    }
-   else { a.AddDomainIntegrator(he_nlf_integ); }
+   else
+   {
+      a.AddDomainIntegrator(he_nlf_integ);
+   }
+
+   if (pa) { a.Setup(); }
 
    const double init_energy = a.GetGridFunctionEnergy(x);
 
@@ -709,9 +759,9 @@ int main(int argc, char *argv[])
       a.SetEssentialVDofs(ess_vdofs);
    }
 
-   // 17. As we use the Newton method to solve the resulting nonlinear system,
-   //     here we setup the linear solver for the system's Jacobian.
-   Solver *S = NULL;
+   // As we use the Newton method to solve the resulting nonlinear system, here
+   // we setup the linear solver for the system's Jacobian.
+   Solver *S = NULL, *S_prec = NULL;
    const double linsol_rtol = 1e-12;
    if (lin_solver == 0)
    {
@@ -733,6 +783,16 @@ int main(int argc, char *argv[])
       minres->SetRelTol(linsol_rtol);
       minres->SetAbsTol(0.0);
       minres->SetPrintLevel(verbosity_level >= 2 ? 3 : -1);
+      if (lin_solver == 3 || lin_solver == 4)
+      {
+         if (pa)
+         {
+            MFEM_VERIFY(lin_solver != 4, "PA l1-Jacobi is not implemented");
+            S_prec = new OperatorJacobiSmoother(a, a.GetEssentialTrueDofs());
+         }
+         else { S_prec = new DSmoother((lin_solver == 3) ? 0 : 1, 1.0, 1); }
+         minres->SetPreconditioner(*S_prec);
+      }
       S = minres;
    }
 
@@ -828,6 +888,7 @@ int main(int argc, char *argv[])
    }
 
    // 24. Free the used memory.
+   delete S_prec;
    delete S;
    delete target_c2;
    delete metric2;

miniapps/meshing/mesh-optimizer.hpp

@@ -33,6 +33,23 @@ double discrete_size_2d(const Vector &x)
    return val * small + (1.0 - val) * big;
 }
 
+double discrete_size_3d(const Vector &x)
+{
+   const double small = 0.0001, big = 0.01;
+   double val = 0.;
+
+   // semi-circle
+   const double xc = x(0) - 0.0, yc = x(1) - 0.5, zc = x(2) - 0.5;
+   const double r = sqrt(xc*xc + yc*yc + zc*zc);
+   double r1 = 0.45; double r2 = 0.55; double sf=30.0;
+   val = 0.5*(1+std::tanh(sf*(r-r1))) - 0.5*(1+std::tanh(sf*(r-r2)));
+
+   val = std::max(0.,val);
+   val = std::min(1.,val);
+
+   return val * small + (1.0 - val) * big;
+}
+
 double material_indicator_2d(const Vector &x)
 {
    double xc = x(0)-0.5, yc = x(1)-0.5;
@@ -222,6 +239,15 @@ double adapt_lim_fun(const Vector &x)
    return val;
 }
 
+// Space-dependent limiting.
+double limit_func(const Vector &x)
+{
+   //if (x(0) < x(1) ) { return 1.0e-6; }
+   //else return 10000;
+
+   return 1.0;
+}
+
 void DiffuseField(GridFunction &field, int smooth_steps)
 {
    //Setup the Laplacian operator

miniapps/meshing/pmesh-optimizer.cpp

@@ -31,6 +31,9 @@
 //
 // Compile with: make pmesh-optimizer
 //
+// Diagonal PA test:
+//     mpirun -np 4 pmesh-optimizer -m blade.mesh -o 4 -rs 0 -mid 2 -tid 1 -ni 200 -bnd -qt 1 -qo 8 -ls 3 -nor -lc 0.1 -vl 2 -pa -d cpu
+//
 // Sample runs:
 //   Adapted analytic shape:
 //     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 2 -tid 4 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
@@ -122,6 +125,8 @@ int main (int argc, char *argv[])
    bool fdscheme         = false;
    int adapt_eval        = 0;
    bool exactaction      = false;
+   const char *devopt    = "cpu";
+   bool pa               = false;
 
    // 2. Parse command-line options.
    OptionsParser args(argc, argv);
@@ -180,7 +185,12 @@ int main (int argc, char *argv[])
    args.AddOption(&solver_rtol, "-rtol", "--newton-rel-tolerance",
                   "Relative tolerance for the Newton solver.");
    args.AddOption(&lin_solver, "-ls", "--lin-solver",
-                  "Linear solver: 0 - l1-Jacobi, 1 - CG, 2 - MINRES.");
+                  "Linear solver:\n\t"
+                  "0: l1-Jacobi\n\t"
+                  "1: CG\n\t"
+                  "2: MINRES\n\t"
+                  "3: MINRES + Jacobi preconditioner"
+                  "4: MINRES + l1-Jacobi preconditioner");
    args.AddOption(&max_lin_iter, "-li", "--lin-iter",
                   "Maximum number of iterations in the linear solve.");
    args.AddOption(&move_bnd, "-bnd", "--move-boundary", "-fix-bnd",
@@ -207,6 +217,10 @@ int main (int argc, char *argv[])
                   "Set the verbosity level - 0, 1, or 2.");
    args.AddOption(&adapt_eval, "-ae", "--adaptivity-evaluator",
                   "0 - Advection based (DEFAULT), 1 - GSLIB.");
+   args.AddOption(&devopt, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
    args.Parse();
    if (!args.Good())
    {
@@ -215,6 +229,9 @@ int main (int argc, char *argv[])
    }
    if (myid == 0) { args.PrintOptions(cout); }
 
+   Device device(devopt);
+   if (myid == 0) { device.Print();}
+
    // 3. Initialize and refine the starting mesh.
    Mesh *mesh = new Mesh(mesh_file, 1, 1, false);
    for (int lev = 0; lev < rs_levels; lev++)
@@ -299,6 +316,7 @@ int main (int argc, char *argv[])
    rdm.Randomize();
    rdm -= 0.25; // Shift to random values in [-0.5,0.5].
    rdm *= jitter;
+   rdm.HostReadWrite();
    // Scale the random values to be of order of the local mesh size.
    for (int i = 0; i < pfespace->GetNDofs(); i++)
    {
@@ -328,7 +346,8 @@ int main (int argc, char *argv[])
       mesh_name << "perturbed.mesh";
       ofstream mesh_ofs(mesh_name.str().c_str());
       mesh_ofs.precision(8);
-      pmesh->PrintAsOne(mesh_ofs);
+      //x.HostRead();
+      //pmesh->PrintAsOne(mesh_ofs);
    }
 
    // 11. Store the starting (prior to the optimization) positions.
@@ -372,8 +391,9 @@ int main (int argc, char *argv[])
    ParFiniteElementSpace ind_fes(pmesh, &ind_fec);
    ParFiniteElementSpace ind_fesv(pmesh, &ind_fec, dim);
    ParGridFunction size(&ind_fes), aspr(&ind_fes), disc(&ind_fes), ori(&ind_fes);
-   ParGridFunction aspr3d(&ind_fesv), size3d(&ind_fesv);
-
+   ParGridFunction aspr3d(&ind_fesv);
+   const AssemblyLevel al =
+      pa ? AssemblyLevel::PARTIAL : AssemblyLevel::LEGACYFULL;
    switch (target_id)
    {
       case 1: target_t = TargetConstructor::IDEAL_SHAPE_UNIT_SIZE; break;
@@ -394,7 +414,7 @@ int main (int argc, char *argv[])
          DiscreteAdaptTC *tc = new DiscreteAdaptTC(target_t);
          if (adapt_eval == 0)
          {
-            tc->SetAdaptivityEvaluator(new AdvectorCG);
+            tc->SetAdaptivityEvaluator(new AdvectorCG(al));
          }
          else
          {
@@ -420,7 +440,7 @@ int main (int argc, char *argv[])
          disc.ProjectCoefficient(ind_coeff);
          if (adapt_eval == 0)
          {
-            tc->SetAdaptivityEvaluator(new AdvectorCG);
+            tc->SetAdaptivityEvaluator(new AdvectorCG(al));
          }
          else
          {
@@ -518,7 +538,7 @@ int main (int argc, char *argv[])
          DiscreteAdaptTC *tc = new DiscreteAdaptTC(target_t);
          if (adapt_eval == 0)
          {
-            tc->SetAdaptivityEvaluator(new AdvectorCG);
+            tc->SetAdaptivityEvaluator(new AdvectorCG(al));
          }
          else
          {
@@ -540,7 +560,7 @@ int main (int argc, char *argv[])
          DiscreteAdaptTC *tc = new DiscreteAdaptTC(target_t);
          if (adapt_eval == 0)
          {
-            tc->SetAdaptivityEvaluator(new AdvectorCG);
+            tc->SetAdaptivityEvaluator(new AdvectorCG(al));
          }
          else
          {
@@ -583,7 +603,14 @@ int main (int argc, char *argv[])
    }
    target_c->SetNodes(x0);
    TMOP_Integrator *he_nlf_integ= new TMOP_Integrator(metric, target_c);
-   if (fdscheme) { he_nlf_integ->EnableFiniteDifferences(x); }
+
+   // Finite differences for computations of derivatives.
+   if (fdscheme)
+   {
+      MFEM_VERIFY(pa == false, "PA for finite differences is not imlemented.");
+
+      he_nlf_integ->EnableFiniteDifferences(x);
+   }
    he_nlf_integ->SetExactActionFlag(exactaction);
 
    // 13. Setup the quadrature rule for the non-linear form integrator.
@@ -606,8 +633,13 @@ int main (int argc, char *argv[])
 
    // 14. Limit the node movement.
    // The limiting distances can be given by a general function of space.
-   ParGridFunction dist(pfespace);
-   dist = 1.0;
+   // TODO come up with proper example for space-dependent limiting that shows
+   //   differences between Bernstein and Gauss-Lobatto.
+   H1_FECollection lim_fec(mesh_poly_deg, dim, BasisType::Positive);
+   ParFiniteElementSpace lim_fes(pmesh, &lim_fec);
+   ParGridFunction dist(&lim_fes);
+   FunctionCoefficient lim_dist_coeff(limit_func);
+   dist.ProjectCoefficient(lim_dist_coeff);
    // The small_phys_size is relevant only with proper normalization.
    if (normalization) { dist = small_phys_size; }
    ConstantCoefficient lim_coeff(lim_const);
@@ -619,10 +651,12 @@ int main (int argc, char *argv[])
    AdaptivityEvaluator *adapt_evaluator = NULL;
    if (adapt_lim_const > 0.0)
    {
+      MFEM_VERIFY(pa == false, "PA is not implemented for adaptive limiting");
+
       FunctionCoefficient alim_coeff(adapt_lim_fun);
       zeta_0.ProjectCoefficient(alim_coeff);
 
-      if (adapt_eval == 0) { adapt_evaluator = new AdvectorCG; }
+      if (adapt_eval == 0) { adapt_evaluator = new AdvectorCG(al); }
       else if (adapt_eval == 1)
       {
 #ifdef MFEM_USE_GSLIB
@@ -649,6 +683,7 @@ int main (int argc, char *argv[])
    //     no command-line options for the weights and the type of the second
    //     metric; one should update those in the code.
    ParNonlinearForm a(pfespace);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    ConstantCoefficient *coeff1 = NULL;
    TMOP_QualityMetric *metric2 = NULL;
    TargetConstructor *target_c2 = NULL;
@@ -687,6 +722,8 @@ int main (int argc, char *argv[])
    }
    else { a.AddDomainIntegrator(he_nlf_integ); }
 
+   if (pa) { a.Setup(); }
+
    const double init_energy = a.GetParGridFunctionEnergy(x);
 
    // 16. Visualize the starting mesh and metric values.
@@ -751,9 +788,9 @@ int main (int argc, char *argv[])
       a.SetEssentialVDofs(ess_vdofs);
    }
 
-   // 18. As we use the Newton method to solve the resulting nonlinear system,
-   //     here we setup the linear solver for the system's Jacobian.
-   Solver *S = NULL;
+   // As we use the Newton method to solve the resulting nonlinear system, here
+   // we setup the linear solver for the system's Jacobian.
+   Solver *S = NULL, *S_prec = NULL;
    const double linsol_rtol = 1e-12;
    if (lin_solver == 0)
    {
@@ -774,7 +811,27 @@ int main (int argc, char *argv[])
       minres->SetMaxIter(max_lin_iter);
       minres->SetRelTol(linsol_rtol);
       minres->SetAbsTol(0.0);
-      minres->SetPrintLevel(verbosity_level >= 2 ? 3 : -1);
+      if (verbosity_level > 2) { minres->SetPrintLevel(1); }
+      else { minres->SetPrintLevel(verbosity_level = 2 ? 3 : -1); }
+      if (lin_solver == 3 || lin_solver == 4)
+      {
+         if (pa)
+         {
+            MFEM_VERIFY(lin_solver != 4, "PA l1-Jacobi is not implemented");
+            OperatorJacobiSmoother *sm =
+               new OperatorJacobiSmoother(a, a.GetEssentialTrueDofs());
+            sm->SetAbsValues(true);
+            S_prec = sm;
+         }
+         else
+         {
+            HypreSmoother *hs = new HypreSmoother;
+            hs->SetType((lin_solver == 3) ? HypreSmoother::Jacobi
+                        : HypreSmoother::l1Jacobi, 1);
+            S_prec = hs;
+         }
+         minres->SetPreconditioner(*S_prec);
+      }
       S = minres;
    }
 
@@ -887,6 +944,7 @@ int main (int argc, char *argv[])
    }
 
    // 24. Free the used memory.
+   delete S_prec;
    delete S;
    delete target_c2;
    delete metric2;

tests/scripts/branch-history

@@ -60,7 +60,7 @@ if (!chdir $mfem_dir) {
 }
 
 # Maximum number of acceptable commits in the branch
-my $branch_max_commits = 200;
+my $branch_max_commits = 300;
 # Maximum number of acceptable files changed in any commit in the branch
 my $commit_max_files_changed = 50;
 # Maximum acceptable size (in K) of any git blob in the branch