Add some comments

vsx128 GNU fix [vsx]

.gitignore

@@ -15,6 +15,19 @@
 CMakeCache.txt
 CMakeFiles/
 
+# NSYS files
+*.nsys-rep
+
+# Benchmark files
+*.tex
+*.org
+*.aux
+*.log
+*.gnuplot
+*.table
+*.pdf
+*.png
+
 # Backup files
 *~
 
@@ -316,6 +329,9 @@ tests/unit/debug_device_tests
 
 # Benchmark binaries
 tests/benchmarks/bench_ceed
+tests/benchmarks/bench_solvers
+tests/benchmarks/bench_linext
+tests/benchmarks/bench_lor_assembly
 tests/benchmarks/bench_tmop
 tests/benchmarks/bench_vector
 tests/benchmarks/bench_virtuals

.gitlab/configs/lassen-config.yml

@@ -45,5 +45,5 @@ variables:
     - echo ${MFEM_DATA_DIR}
     - echo ${SPEC}
     # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 30 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
   needs: [setup]

CHANGELOG

@@ -10,10 +10,16 @@
 
 Version 4.3.1 (development)
 ===========================
-- Added support for automatic differentiation. Users can select between 
-  native implementation and external library implementation at the 
-  configuration phase. A parallel and two serial examples are implemented
-  in the autodiff miniapp directory.
+- More explicit and consistent formating of the output of iterative solvers
+  with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
+
+- Added support for automatic differentiation. Users can select between native
+  implementation and external library implementation during configuration. One
+  parallel and two serial examples are implemented in the miniapps/autodiff/
+  directory.
+
+- GridFunctionCoefficient (and the related vector, gradient, divergence, and
+  curl classes) now work properly with LORDiscretization and LORSolver.
 
 - Added support for mesh preprocessing to resolve fine scale problem data
   before simulation. This feature uses adaptive mesh refinement to control the
@@ -71,13 +77,23 @@ Version 4.3.1 (development)
 
 - Added initial TMOP-based capabilities for surface fitting and tangential
   relaxation in the mesh-optimizer and pmesh-optimizer miniapps.
- 
+
 - Added ParMesh Adjaceny Set (adjset) creation support to the Conduit Mesh
   Blueprint MFEM wrapper functions in ConduitDataCollection.
 
+- Added full assembly / device support for the LinearForm integrators:
+  * DomainLF: L(v) := (f, v)
+  * VectorDomainLF: L(v) := ((f1,...,fn), (v1,...,vn))
+  * DomainLFGrad: L(v) := (f,grad v)
+  * VectorDomainLFGrad:  L(v) := ((f1x,f1y,f1z,...,fnx,fny,fnz),grad(v1,...,vn))
+  It can be enabled by setting the linear assembly level to
+  'LinearAssemblyLevel::FULL' with the 'LinearForm::SetAssemblyLevel' method.
+
 - `HypreParVector` and `Vector` now support move semantics, and the copy
   constructor for `HypreParVector` now copies the local vector data.
 
+- The HPC versions of ex1 and ex1p (in miniapps/performance) now support
+  runtime selection of either 2D or 3D meshes.
 
 Version 4.3, released on July 29, 2021
 ======================================

CODE_OF_CONDUCT.md

@@ -0,0 +1,133 @@
+# MFEM Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at mfem@llnl.gov.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident. Anyone involved in the reported behavior will recuse
+themselves from the investigation and decision making about the resolution of
+the complaint.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -21,6 +21,9 @@ documentation; new examples and miniapps; HPC performance improvements; etc.
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license.
 
+Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
+in the MFEM community, you agree to abide by its rules.
+
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -138,6 +141,7 @@ The MFEM source code has the following structure:
   │   ├── tools
   │   └── toys
   └── tests
+      ├── benchmarks
       ├── convergence
       ├── gitlab
       ├── mem_manager

README

@@ -12,6 +12,9 @@ to enable high-performance scalable finite element discretization research and
 application development on a wide variety of platforms, ranging from laptops to
 supercomputers.
 
+We welcome contributions and feedback from the community. Please see the file
+CONTRIBUTING.md for additional details about our development process.
+
 * For building instructions, see the file INSTALL, or type "make help".
 
 * Copyright and licensing information can be found in files LICENSE and NOTICE.
@@ -19,9 +22,6 @@ supercomputers.
 * The best starting point for new users interested in MFEM's features is to
   review the examples and miniapps at https://mfem.org/examples.
 
-* Developers interested in contributing to the library, should read the
-  instructions and documentation in the CONTRIBUTING.md file.
-
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support

config/defaults.mk

@@ -428,7 +428,7 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse
+CUDA_LIB = -lcusparse -lnvToolsExt
 
 # HIP library configuration (currently not needed)
 HIP_OPT =

examples/ex1.cpp

@@ -120,8 +120,8 @@ int main(int argc, char *argv[])
    //    largest number that gives a final mesh with no more than 50,000
    //    elements.
    {
-      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+      const int NE = Device::IsEnabled() ? 1e5 : 50000.;
+      const int ref_levels = (int)floor(log(NE/mesh.GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh.UniformRefinement();
@@ -171,6 +171,7 @@ int main(int argc, char *argv[])
    LinearForm b(&fespace);
    ConstantCoefficient one(1.0);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
    b.Assemble();
 
    // 8. Define the solution vector x as a finite element grid function
@@ -184,7 +185,10 @@ int main(int argc, char *argv[])
    //    domain integrator.
    BilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   // 'Default' E2E kernel
+   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   // Can be E2E, L2L, libCeed, etc.
+   a.AddDomainIntegrator(new MassIntegrator(one));
 
    // 10. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: eliminating boundary
@@ -197,7 +201,7 @@ int main(int argc, char *argv[])
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-   cout << "Size of linear system: " << A->Height() << endl;
+   //cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
    if (!pa)
@@ -223,10 +227,70 @@ int main(int argc, char *argv[])
             ceed::AlgebraicSolver M(a, ess_tdof_list);
             PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
          }
-         else
+         /*else
          {
             OperatorJacobiSmoother M(a, ess_tdof_list);
             PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+         }*/
+         else
+         {
+            const int myid = 0;
+            const int max_it = 50;
+            const int print_lvl = -1;
+            const double rtol = 1e-12;
+
+            CGSolver cg;
+            cg.SetRelTol(rtol);
+            cg.SetOperator(*A);
+
+            // Warm-up CG solve (in case of JIT to avoid timing it)
+            {
+               Vector Y(X);
+               cg.SetMaxIter(2);
+               cg.SetPrintLevel(-1);
+               cg.Mult(B, Y);
+               MFEM_DEVICE_SYNC;
+            }
+
+            // benchmark this problem
+            {
+               tic_toc.Clear();
+               cg.SetMaxIter(max_it);
+               cg.SetPrintLevel(print_lvl);
+               {
+                  tic_toc.Start();
+                  cg.Mult(B, X);
+                  MFEM_DEVICE_SYNC;
+                  tic_toc.Stop();
+               }
+            }
+            // Final norm check
+            const double final_norm = cg.GetFinalNorm();
+            MFEM_VERIFY(final_norm < sqrt(rtol),
+                        "FinalNorm (" << final_norm << ") Error!");
+            // Number of iteration check
+            const int num_iter = cg.GetNumIterations();
+            MFEM_VERIFY(num_iter <= max_it,
+                        "NumIterations (" << num_iter << ") Error!");
+
+            const double rt = tic_toc.RealTime();
+            const double rt_min = rt, rt_max = rt;
+            const int dofs = fespace.GetVSize();
+            const int cg_iter = cg.GetNumIterations();
+            const double mdofs_max = ((1e-6 * dofs) * cg_iter) / rt_max;
+            const double mdofs_min = ((1e-6 * dofs) * cg_iter) / rt_min;
+            if (myid == 0)
+            {
+               mfem::out << "Total CG time:    " << rt_max << " (" << rt_min << ") sec."
+                         << std::endl;
+               mfem::out << "Time per CG step: "
+                         << rt_max / cg_iter << " ("
+                         << rt_min / cg_iter << ") sec." << std::endl;
+               mfem::out << "\033[32m";
+               mfem::out << "\"DOFs/sec\" in CG: " << mdofs_max << " ("
+                         << mdofs_min << ") million.";
+               mfem::out << "\033[m" << std::endl;
+            }
          }
       }
       else

examples/ex10.cpp

@@ -149,7 +149,7 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 void InitialVelocity(const Vector &x, Vector &v);
 
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -376,10 +376,10 @@ int main(int argc, char *argv[])
 }
 
 
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -389,24 +389,25 @@ void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "solution\n" << *mesh << *field;
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jl\n";  // turn off perspective and light
+         os << "view 0 0\n"; // view from top
+         os << "keys jl\n";  // turn off perspective and light
       }
-      out << "keys cm\n";         // show colorbar and mesh
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
-      out << "pause\n";
+      os << "keys cm\n";         // show colorbar and mesh
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
+      os << "pause\n";
    }
-   out << flush;
+   os << flush;
 }
 
 

examples/ex10p.cpp

@@ -154,7 +154,8 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 void InitialVelocity(const Vector &x, Vector &v);
 
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -438,10 +439,11 @@ int main(int argc, char *argv[])
    return 0;
 }
 
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -451,25 +453,27 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
-   out << "solution\n" << *mesh << *field;
+   os << "parallel " << mesh->GetNRanks()
+      << " " << mesh->GetMyRank() << "\n";
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jl\n";  // turn off perspective and light
+         os << "view 0 0\n"; // view from top
+         os << "keys jl\n";  // turn off perspective and light
       }
-      out << "keys cm\n";         // show colorbar and mesh
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
-      out << "pause\n";
+      os << "keys cm\n";         // show colorbar and mesh
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
+      os << "pause\n";
    }
-   out << flush;
+   os << flush;
 }
 
 

examples/ex18.hpp

@@ -32,7 +32,7 @@ private:
    mutable DenseTensor flux;
    mutable Vector z;
 
-   void GetFlux(const DenseMatrix &state, DenseTensor &flux) const;
+   void GetFlux(const DenseMatrix &state_, DenseTensor &flux_) const;
 
 public:
    FE_Evolution(FiniteElementSpace &vfes_,
@@ -256,26 +256,26 @@ inline double ComputeMaxCharSpeed(const Vector &state, const int dim)
 }
 
 // Compute the flux at solution nodes.
-void FE_Evolution::GetFlux(const DenseMatrix &x, DenseTensor &flux) const
+void FE_Evolution::GetFlux(const DenseMatrix &x_, DenseTensor &flux_) const
 {
-   const int dof = flux.SizeI();
-   const int dim = flux.SizeJ();
+   const int flux_dof = flux_.SizeI();
+   const int flux_dim = flux_.SizeJ();
 
-   for (int i = 0; i < dof; i++)
+   for (int i = 0; i < flux_dof; i++)
    {
-      for (int k = 0; k < num_equation; k++) { state(k) = x(i, k); }
-      ComputeFlux(state, dim, f);
+      for (int k = 0; k < num_equation; k++) { state(k) = x_(i, k); }
+      ComputeFlux(state, flux_dim, f);
 
-      for (int d = 0; d < dim; d++)
+      for (int d = 0; d < flux_dim; d++)
       {
          for (int k = 0; k < num_equation; k++)
          {
-            flux(i, d, k) = f(k, d);
+            flux_(i, d, k) = f(k, d);
          }
       }
 
       // Update max char speed
-      const double mcs = ComputeMaxCharSpeed(state, dim);
+      const double mcs = ComputeMaxCharSpeed(state, flux_dim);
       if (mcs > max_char_speed) { max_char_speed = mcs; }
    }
 }

examples/ex19.cpp

@@ -171,7 +171,7 @@ public:
 };
 
 // Visualization driver
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -542,10 +542,10 @@ RubberOperator::~RubberOperator()
 
 
 // Inline visualization
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -555,23 +555,25 @@ void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "solution\n" << *mesh << *field;
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
+         os << "view 0 0\n"; // view from top
+         // turn off perspective and light, +anti-aliasing
+         os << "keys jlA\n";
       }
-      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
    }
-   out << flush;
+   os << flush;
 }
 
 void ReferenceConfiguration(const Vector &x, Vector &y)

examples/ex19p.cpp

@@ -185,7 +185,8 @@ public:
 };
 
 // Visualization driver
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -617,10 +618,11 @@ RubberOperator::~RubberOperator()
 
 
 // Inline visualization
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
    {
       return;
    }
@@ -630,24 +632,27 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
-   out << "solution\n" << *mesh << *field;
+   os << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() <<
+      "\n";
+   os << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         out << "view 0 0\n"; // view from top
-         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
+         os << "view 0 0\n"; // view from top
+         // turn off perspective and light, +anti-aliasing
+         os << "keys jlA\n";
       }
-      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
    }
-   out << flush;
+   os << flush;
 }
 
 void ReferenceConfiguration(const Vector &x, Vector &y)

examples/ex1_es.cpp

@@ -1,266 +0,0 @@
-//                                MFEM Example 1
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int rs = -1;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   cout << "Calling RandomRefinement " << ra << " times." << endl;
-   for (int l = 0; l < ra; l++)
-   {
-      mesh->RandomRefinement(0.2);
-   }
-   cout << "Done with refinement" << endl;
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   if ( mesh->ncmesh )
-   {
-      mesh->ncmesh->PrintStats(cout);
-
-      ofstream ofsV("vp.out");
-      ofstream ofsE("ce.out");
-      mesh->ncmesh->PrintVertexParents(ofsV);
-      mesh->ncmesh->PrintCoarseElements(ofsE);
-   }
-
-   // 4. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (mesh->GetNodes())
-   {
-      fec = mesh->GetNodes()->OwnFEC();
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace->GetTrueVSize() << endl;
-
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (mesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(mesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-
-   cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 7. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
-   x = 0.0;
-
-   // 8. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 9. Assemble the bilinear form and the corresponding linear system,
-   //    applying any necessary transformations such as: eliminating boundary
-   //    conditions, applying conforming constraints for non-conforming AMR,
-   //    static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   SparseMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   cout << "Size of linear system: " << A.Height() << endl;
-
-#ifndef MFEM_USE_SUITESPARSE
-   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //     solve the system A X = B with PCG.
-   GSSmoother M(A);
-   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-   // 10. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(B, X);
-#endif
-
-   // 11. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 12. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 13. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 14. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete mesh;
-
-   return 0;
-}

examples/ex1p.cpp

@@ -128,7 +128,7 @@ int main(int argc, char *argv[])
    //    more than 10,000 elements.
    {
       int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(1000./mesh.GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh.UniformRefinement();
@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
    ParLinearForm b(&fespace);
    ConstantCoefficient one(1.0);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
    b.Assemble();
 
    // 10. Define the solution vector x as a parallel finite element grid
@@ -210,7 +211,8 @@ int main(int argc, char *argv[])
    //     Diffusion domain integrator.
    ParBilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddDomainIntegrator(new MassIntegrator(one));
 
    // 12. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
@@ -245,13 +247,67 @@ int main(int argc, char *argv[])
    {
       prec = new HypreBoomerAMG;
    }
+
+   const int max_it = 50;
+   const int print_lvl = -1;
+   const double rtol = 1e-12;
+
    CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
+   cg.SetRelTol(rtol);
+   //cg.SetMaxIter(max_it);
+   //cg.SetPrintLevel(print_lvl);
    if (prec) { cg.SetPreconditioner(*prec); }
    cg.SetOperator(*A);
-   cg.Mult(B, X);
+
+   // Warm-up CG solve (in case of JIT to avoid timing it)
+   {
+      Vector Y(X);
+      cg.SetMaxIter(2);
+      cg.SetPrintLevel(-1);
+      cg.Mult(B, Y);
+      MFEM_DEVICE_SYNC;
+   }
+
+   // benchmark this problem
+   {
+      tic_toc.Clear();
+      cg.SetMaxIter(max_it);
+      cg.SetPrintLevel(print_lvl);
+      {
+         tic_toc.Start();
+         cg.Mult(B, X);
+         MFEM_DEVICE_SYNC;
+         tic_toc.Stop();
+      }
+   }
+   // Final norm check
+   const double final_norm = cg.GetFinalNorm();
+   MFEM_VERIFY(final_norm < sqrt(rtol),
+               "FinalNorm (" << final_norm << ") Error!");
+   // Number of iteration check
+   const int num_iter = cg.GetNumIterations();
+   MFEM_VERIFY(num_iter <= max_it,
+               "NumIterations (" << num_iter << ") Error!");
+
+   const double rt = tic_toc.RealTime();
+   const double rt_min = rt, rt_max = rt;
+   HYPRE_BigInt dofs = fespace.GlobalTrueVSize();
+   const int cg_iter = cg.GetNumIterations();
+   const double mdofs_max = ((1e-6 * dofs) * cg_iter) / rt_max;
+   const double mdofs_min = ((1e-6 * dofs) * cg_iter) / rt_min;
+   if (myid == 0)
+   {
+      mfem::out << "Total CG time:    " << rt_max << " (" << rt_min << ") sec."
+                << std::endl;
+      mfem::out << "Time per CG step: "
+                << rt_max / cg_iter << " ("
+                << rt_min / cg_iter << ") sec." << std::endl;
+      mfem::out << "\033[32m";
+      mfem::out << "\"DOFs/sec\" in CG: " << mdofs_max << " ("
+                << mdofs_min << ") million.";
+      mfem::out << "\033[m" << std::endl;
+   }
+   //cg.Mult(B, X);
    delete prec;
 
    // 14. Recover the parallel grid function corresponding to X. This is the

examples/ex1p_es.cpp

@@ -1,388 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-//               The following are examples of using EntitySets to define
-//               homogeneous Dirichlet boundary condition.  These examples
-//               require a modified mesh file and a specialized version of
-//               example 1 called "ex1p_es".
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 0 -bs Origin
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 1 -bs Axes
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 1 -bs "Negative Axes"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 2 -bs "Interior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 2 -bs "Exterior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 3 -bs "Interior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 3 -bs "Exterior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 3 -bs "Steps"
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   cout << "creating ParMesh from serial mesh" << endl;
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   cout << "done creating ParMesh from serial mesh" << endl;
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   /*
-     We now have a ParEntitySets object which marshals the data stored
-     in EntitySets objects.  The data has now been pruned so that each
-     rank only contains indices of local entities.
-
-     The NCEntitySets object remains unchanged...
-
-     If we have an NC mesh a different path is taken and the
-     EntitySets are ignored.
-
-     1) ParNCMesh is created from NCMesh
-       a) Creates a ParNCEntitySets object from ncmesh (every rank contains
-          information to find every entity)
-     2) ParNCMesh is pruned which involves renumbering elements and vertices
-     3) ParMesh is initialized from ParNCMesh
-     4) ParNCMesh::OnMeshUpdated is called
-     5) Mesh::GenerateNCFaceInfo is called
-    */
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   /*
-     RandomRefinement will end up calling
-     ParMesh::NonconformingRefinement which will create a new ParMesh
-     object using the ParNCMesh object and then call
-     ParMesh::OnMeshUpdated on this new mesh.
-    */
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   for (int i=0; i<num_procs; i++)
-   {
-      if (myid == i)
-      {
-         cout << "Number of Dirichlet dofs on proc " << i << ": "
-              << ess_tdof_list.Size() << endl;
-      }
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-   //     preconditioner from hypre.
-   HypreSolver *amg = new HypreBoomerAMG(A);
-   HyprePCG *pcg = new HyprePCG(A);
-   pcg->SetTol(1e-12);
-   pcg->SetMaxIter(200);
-   pcg->SetPrintLevel(2);
-   pcg->SetPreconditioner(*amg);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 14. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 16. Free the used memory.
-   delete pcg;
-   delete amg;
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}

examples/ex27.cpp

@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
                    double alpha, double beta, double gamma,
-                   double &err);
+                   double &error);
 
 int main(int argc, char *argv[])
 {
@@ -302,26 +302,26 @@ int main(int argc, char *argv[])
    {
       // Integrate the solution on the Dirichlet boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
+      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
 
       bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
       mfem::out << "Average of solution on Gamma_dbc:\t"
                 << avg << ", \t"
                 << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
       // to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
+      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
 
       bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
       mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -330,25 +330,26 @@ int main(int argc, char *argv[])
       nbc0_bdr = 0;
       nbc0_bdr[3] = 1;
 
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
+      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
 
       bool hom_nbc = true;
       mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
+      double error;
+      double avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, error);
 
       bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
       mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                 << avg << ", \t"
                 << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
 
    // 14. Save the refined mesh and the solution. This output can be viewed
@@ -637,11 +638,11 @@ Mesh * GenerateSerialMesh(int ref)
 
 double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
                    double alpha, double beta, double gamma,
-                   double &err)
+                   double &error)
 {
    double nrm = 0.0;
    double avg = 0.0;
-   err = 0.0;
+   error = 0.0;
 
    const bool a_is_zero = alpha == 0.0;
    const bool b_is_zero = beta == 0.0;
@@ -705,20 +706,20 @@ double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
 
          // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
          val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
+         error += (val*val) * ip.weight * face_weight;
       }
    }
 
    // Normalize by the length of the boundary
    if (std::abs(nrm) > 0.0)
    {
-      err /= nrm;
+      error /= nrm;
       avg /= nrm;
    }
 
    // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'err')
-   err = (err >= 0.0) ? sqrt(err) : -sqrt(-err);
+   // quadrature weights may produce negative 'error')
+   error = (error >= 0.0) ? sqrt(error) : -sqrt(-error);
 
    // Return the average value of alpha * n.Grad(x) + beta * x
    return avg;

examples/ex27p.cpp

@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
                    double alpha, double beta, double gamma,
-                   double &err);
+                   double &error);
 
 int main(int argc, char *argv[])
 {
@@ -321,26 +321,26 @@ int main(int argc, char *argv[])
    {
       // Integrate the solution on the Dirichlet boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
+      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
 
       bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
       mfem::out << "Average of solution on Gamma_dbc:\t"
                 << avg << ", \t"
                 << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
       // to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
+      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
 
       bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
       mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -349,25 +349,26 @@ int main(int argc, char *argv[])
       nbc0_bdr = 0;
       nbc0_bdr[3] = 1;
 
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
+      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
 
       bool hom_nbc = true;
       mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
    {
       // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
       // expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
+      double error, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val,
+                                      error);
 
       bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
       mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                 << avg << ", \t"
                 << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
    }
 
    // 15. Save the refined mesh and the solution in parallel. This output can be
@@ -667,11 +668,11 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
    double loc_vals[3];
    double &nrm = loc_vals[0];
    double &avg = loc_vals[1];
-   double &err = loc_vals[2];
+   double &error = loc_vals[2];
 
    nrm = 0.0;
    avg = 0.0;
-   err = 0.0;
+   error = 0.0;
 
    const bool a_is_zero = alpha == 0.0;
    const bool b_is_zero = beta == 0.0;
@@ -735,7 +736,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
 
          // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
          val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
+         error += (val*val) * ip.weight * face_weight;
       }
    }
 
@@ -754,7 +755,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
    }
 
    // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'err')
+   // quadrature weights may produce negative 'error')
    glb_err = (glb_err >= 0.0) ? sqrt(glb_err) : -sqrt(-glb_err);
 
    // Return the average value of alpha * n.Grad(x) + beta * x

examples/ex29.cpp

@@ -167,9 +167,9 @@ int main(int argc, char *argv[])
 
    // 13. Compute error in the solution and its flux
    FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
+   double error = x.ComputeL2Error(uCoef);
 
-   cout << "|u - u_h|_2 = " << err << endl;
+   cout << "|u - u_h|_2 = " << error << endl;
 
    FiniteElementSpace flux_fespace(mesh, &fec, 3);
    GridFunction flux(&flux_fespace);

examples/ex29p.cpp

@@ -197,9 +197,9 @@ int main(int argc, char *argv[])
 
    // 15. Compute error in the solution and its flux
    FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
+   double error = x.ComputeL2Error(uCoef);
 
-   if (myid == 0) { cout << "|u - u_h|_2 = " << err << endl; }
+   if (myid == 0) { cout << "|u - u_h|_2 = " << error << endl; }
 
    ParFiniteElementSpace flux_fespace(&pmesh, &fec, 3);
    ParGridFunction flux(&flux_fespace);

examples/ex3p.cpp

@@ -255,10 +255,10 @@ int main(int argc, char *argv[])
 
    // 15. Compute and print the L^2 norm of the error.
    {
-      double err = x.ComputeL2Error(E);
+      double error = x.ComputeL2Error(E);
       if (myid == 0)
       {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+         cout << "\n|| E_h - E ||_{L^2} = " << error << '\n' << endl;
       }
    }
 

examples/ex3p_es.cpp

@@ -1,411 +0,0 @@
-//                       MFEM Example 3 - Parallel Version
-//
-// Compile with: make ex3p
-//
-// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
-//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
-//
-// Description:  This example code solves a simple electromagnetic diffusion
-//               problem corresponding to the second order definite Maxwell
-//               equation curl curl E + E = f with boundary condition
-//               E x n = <given tangential field>. Here, we use a given exact
-//               solution E and compute the corresponding r.h.s. f.
-//               We discretize with Nedelec finite elements in 2D or 3D.
-//
-//               The example demonstrates the use of H(curl) finite element
-//               spaces with the curl-curl and the (vector finite element) mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Static condensation is
-//               also illustrated.
-//
-//               We recommend viewing examples 1-2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, E, and r.h.s., f. See below for implementation.
-//void E_exact(const Vector &, Vector &);
-//void f_exact(const Vector &, Vector &);
-//double freq = 1.0, kappa;
-void f_const(const Vector &, Vector &);
-
-int dim;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/beam-tet.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   /*
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");
-   */
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-   // kappa = freq * M_PI;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   pmesh->ReorientTetMesh();
-   pmesh->ent_sets->PrintSetInfo(cout);
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   if (myid == 0)
-   {
-      cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-   }
-
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_const);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   // VectorFunctionCoefficient E(sdim, E_exact);
-   // x.ProjectCoefficient(E);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
-   //     mass domain integrators.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(1.0);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for AX=B with the AMS
-   //     preconditioner from hypre.
-   ParFiniteElementSpace *prec_fespace =
-      (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-   HypreSolver *ams = new HypreAMS(A, prec_fespace);
-   HyprePCG *pcg = new HyprePCG(A);
-   pcg->SetTol(1e-12);
-   pcg->SetMaxIter(500);
-   pcg->SetPrintLevel(2);
-   pcg->SetPreconditioner(*ams);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   /*
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(E);
-      if (myid == 0)
-      {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
-      }
-   }
-   */
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete pcg;
-   delete ams;
-   delete a;
-   delete sigma;
-   delete muinv;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-/*
-void E_exact(const Vector &x, Vector &E)
-{
-   if (dim == 3)
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-void f_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
-*/
-void f_const(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      f(2) = 1.0;
-   }
-   else
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}

examples/ex4p.cpp

@@ -256,10 +256,10 @@ int main(int argc, char *argv[])
 
    // 15. Compute and print the L^2 norm of the error.
    {
-      double err = x.ComputeL2Error(F);
+      double error = x.ComputeL2Error(F);
       if (myid == 0)
       {
-         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
+         cout << "\n|| F_h - F ||_{L^2} = " << error << '\n' << endl;
       }
    }
 

examples/ex4p_es.cpp

@@ -1,438 +0,0 @@
-//                       MFEM Example 4 - Parallel Version
-//
-// Compile with: make ex4p
-//
-// Sample runs:  mpirun -np 4 ex4p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex4p -m ../data/star.mesh
-//               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
-//               mpirun -np 4 ex4p -m ../data/beam-hex-nurbs.mesh -o 3
-//               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
-//               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
-//               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
-//
-// Description:  This example code solves a simple 2D/3D H(div) diffusion
-//               problem corresponding to the second order definite equation
-//               -grad(alpha div F) + beta F = f with boundary condition F dot n
-//               = <given normal field>. Here, we use a given exact solution F
-//               and compute the corresponding r.h.s. f.  We discretize with
-//               Raviart-Thomas finite elements.
-//
-//               The example demonstrates the use of H(div) finite element
-//               spaces with the grad-div and H(div) vector finite element mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Bilinear form
-//               hybridization and static condensation are also illustrated.
-//
-//               We recommend viewing examples 1-3 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, F, and r.h.s., f. See below for implementation.
-//void F_exact(const Vector &, Vector &);
-//void f_exact(const Vector &, Vector &);
-//double freq = 1.0, kappa;
-void f_const(const Vector &, Vector &);
-
-int dim;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool set_bc = true;
-   bool static_cond = false;
-   bool hybridization = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
-                  "Impose or not essential boundary conditions.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   // args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-   //                 " solution.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
-                  "--no-hybridization", "Enable hybridization.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-   // kappa = freq * M_PI;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume, as well as periodic meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->ReorientTetMesh();
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = set_bc ? 1 : 0;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   if (myid == 0)
-   {
-      cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_const);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary faces will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   // VectorFunctionCoefficient F(sdim, F_exact);
-   // x.ProjectCoefficient(F);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form corresponding to the H(div)
-   //     diffusion operator grad alpha div + beta I, by adding the div-div and
-   //     the mass domain integrators.
-   Coefficient *alpha = new ConstantCoefficient(1.0);
-   Coefficient *beta  = new ConstantCoefficient(1.0);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation,
-   //     hybridization, etc.
-   FiniteElementCollection *hfec = NULL;
-   ParFiniteElementSpace *hfes = NULL;
-   if (static_cond)
-   {
-      a->EnableStaticCondensation();
-   }
-   else if (hybridization)
-   {
-      hfec = new DG_Interface_FECollection(order-1, dim);
-      hfes = new ParFiniteElementSpace(pmesh, hfec);
-      a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
-                             ess_tdof_list);
-   }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   HYPRE_Int glob_size = A.GetGlobalNumRows();
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << glob_size << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
-   //     the 3D ADS preconditioners from hypre. If using hybridization, the
-   //     system is preconditioned with hypre's BoomerAMG.
-   HypreSolver *prec = NULL;
-   CGSolver *pcg = new CGSolver(A.GetComm());
-   pcg->SetOperator(A);
-   pcg->SetRelTol(1e-12);
-   pcg->SetMaxIter(500);
-   pcg->SetPrintLevel(1);
-   if (hybridization) { prec = new HypreBoomerAMG(A); }
-   else
-   {
-      ParFiniteElementSpace *prec_fespace =
-         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
-      else            { prec = new HypreADS(A, prec_fespace); }
-   }
-   pcg->SetPreconditioner(*prec);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-   /*
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(F);
-      if (myid == 0)
-      {
-         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
-      }
-   }
-   */
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete pcg;
-   delete prec;
-   delete hfes;
-   delete hfec;
-   delete a;
-   delete alpha;
-   delete beta;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-/*
-// The exact solution (for non-surface meshes)
-void F_exact(const Vector &p, Vector &F)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0;
-
-   F(0) = cos(kappa*x)*sin(kappa*y);
-   F(1) = cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      F(2) = 0.0;
-   }
-}
-
-// The right hand side
-void f_exact(const Vector &p, Vector &f)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0;
-
-   double temp = 1 + 2*kappa*kappa;
-
-   f(0) = temp*cos(kappa*x)*sin(kappa*y);
-   f(1) = temp*cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      f(2) = 0;
-   }
-}
-*/
-void f_const(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      f(2) = 1.0;
-   }
-   else
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}

examples/ex6.cpp

@@ -125,6 +125,7 @@ int main(int argc, char *argv[])
    BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
    a.AddDomainIntegrator(integ);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
 
    // 7. The solution vector x and the associated finite element grid function
    //    will be maintained over the AMR iterations. We initialize it to zero.

examples/ex6p.cpp

@@ -202,6 +202,7 @@ int main(int argc, char *argv[])
    BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
    a.AddDomainIntegrator(integ);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
 
    // 12. The solution vector x and the associated finite element grid function
    //     will be maintained over the AMR iterations. We initialize it to zero.

examples/ex6p_es.cpp

@@ -1,325 +0,0 @@
-//                       MFEM Example 6 - Parallel Version
-//
-// Compile with: make ex6p
-//
-// Sample runs:  mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 1
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/star.mesh -o 3
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/fichera.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/ball-nurbs.mesh
-//               mpirun -np 4 ex6p -m ../data/pipe-nurbs.mesh
-//               mpirun -np 4 ex6p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
-//
-// Description:  This is a version of Example 1 with a simple adaptive mesh
-//               refinement loop. The problem being solved is again the Laplace
-//               equation -Delta u = 1 with homogeneous Dirichlet boundary
-//               conditions. The problem is solved on a sequence of meshes which
-//               are locally refined in a conforming (triangles, tetrahedrons)
-//               or non-conforming (quadrilaterals, hexahedra) manner according
-//               to a simple ZZ error estimator.
-//
-//               The example demonstrates MFEM's capability to work with both
-//               conforming and nonconforming refinements, in 2D and 3D, on
-//               linear, curved and surface meshes. Interpolation of functions
-//               from coarse to fine meshes, as well as persistent GLVis
-//               visualization are also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static int max_dofs = 100000;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int bt = EntitySets::INVALID;
-   const char *bs = "";
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&max_dofs, "-md", "--max-dofs",
-                  "Maximum number of degrees of freedom.");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution.
-   //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
-   //    sure that the mesh is non-conforming.
-   if (mesh->NURBSext)
-   {
-      mesh->UniformRefinement();
-      mesh->SetCurvature(2);
-   }
-   mesh->EnsureNCMesh();
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-
-   // 5. Define a parallel mesh by partitioning the serial mesh.
-   //    Once the parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh.pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh.pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-
-   // 6. Define a finite element space on the mesh. The polynomial order is
-   //    one (linear) by default, but this can be changed on the command line.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec);
-
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh.bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace.GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                   ess_tdof_list);
-   }
-
-   // 7. As in Example 1p, we set up bilinear and linear forms corresponding to
-   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
-   //    problem yet, this will be done in the main loop.
-   ParBilinearForm a(&fespace);
-   ParLinearForm b(&fespace);
-
-   ConstantCoefficient one(1.0);
-
-   BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
-   a.AddDomainIntegrator(integ);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-
-   // 8. The solution vector x and the associated finite element grid function
-   //    will be maintained over the AMR iterations. We initialize it to zero.
-   ParGridFunction x(&fespace);
-   x = 0;
-
-   // 9. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-
-   socketstream sout;
-   if (visualization)
-   {
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         if (myid == 0)
-         {
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-            cout << "GLVis visualization disabled.\n";
-         }
-         visualization = false;
-      }
-
-      sout.precision(8);
-   }
-
-   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
-   //     with L2 projection in the smoothing step to better handle hanging
-   //     nodes and parallel partitioning. We need to supply a space for the
-   //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
-   //     used here).
-   L2_FECollection flux_fec(order, dim);
-   ParFiniteElementSpace flux_fes(&pmesh, &flux_fec, sdim);
-   RT_FECollection smooth_flux_fec(order-1, dim);
-   ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec);
-   // Another possible option for the smoothed flux space:
-   // H1_FECollection smooth_flux_fec(order, dim);
-   // ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec, dim);
-   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
-
-   // 11. A refiner selects and refines elements based on a refinement strategy.
-   //     The strategy here is to refine elements with errors larger than a
-   //     fraction of the maximum element error. Other strategies are possible.
-   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(estimator);
-   refiner.SetTotalErrorFraction(0.7);
-
-   // 12. The main AMR loop. In each iteration we solve the problem on the
-   //     current mesh, visualize the solution, and refine the mesh.
-   // const int max_dofs = 100000;
-   for (int it = 0; ; it++)
-   {
-      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
-      if (myid == 0)
-      {
-         cout << "\nAMR iteration " << it << endl;
-         cout << "Number of unknowns: " << global_dofs << endl;
-      }
-
-      // 13. Assemble the stiffness matrix and the right-hand side. Note that
-      //     MFEM doesn't care at this point that the mesh is nonconforming
-      //     and parallel. The FE space is considered 'cut' along hanging
-      //     edges/faces, and also across processor boundaries.
-      a.Assemble();
-      b.Assemble();
-
-      // 14. Create the parallel linear system: eliminate boundary conditions,
-      //     constrain hanging nodes and nodes across processor boundaries.
-      //     The system will be solved for true (unconstrained/unique) DOFs only.
-      // Array<int> ess_tdof_list;
-      if ( bt == EntitySets::INVALID )
-      {
-         if (pmesh.bdr_attributes.Size())
-         {
-            Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-            ess_bdr = 1;
-            fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-         }
-      }
-      else
-      {
-         fespace.GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                      ess_tdof_list);
-      }
-
-      HypreParMatrix A;
-      Vector B, X;
-      const int copy_interior = 1;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-
-      // 15. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-      //     preconditioner from hypre.
-      HypreBoomerAMG amg;
-      amg.SetPrintLevel(0);
-      CGSolver pcg(A.GetComm());
-      pcg.SetPreconditioner(amg);
-      pcg.SetOperator(A);
-      pcg.SetRelTol(1e-6);
-      pcg.SetMaxIter(200);
-      pcg.SetPrintLevel(3); // print the first and the last iterations only
-      pcg.Mult(B, X);
-
-      // 16. Extract the parallel grid function corresponding to the finite element
-      //     approximation X. This is the local solution on each processor.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 17. Send the solution by socket to a GLVis server.
-      if (visualization)
-      {
-         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout << "solution\n" << pmesh << x << flush;
-      }
-
-      if (global_dofs > max_dofs)
-      {
-         if (myid == 0)
-         {
-            cout << "Reached the maximum number of dofs. Stop." << endl;
-         }
-         break;
-      }
-
-      // 18. Call the refiner to modify the mesh. The refiner calls the error
-      //     estimator to obtain element errors, then it selects elements to be
-      //     refined and finally it modifies the mesh. The Stop() method can be
-      //     used to determine if a stopping criterion was met.
-      refiner.Apply(pmesh);
-      if (refiner.Stop())
-      {
-         if (myid == 0)
-         {
-            cout << "Stopping criterion satisfied. Stop." << endl;
-         }
-         break;
-      }
-
-      // 19. Update the finite element space (recalculate the number of DOFs,
-      //     etc.) and create a grid function update matrix. Apply the matrix
-      //     to any GridFunctions over the space. In this case, the update
-      //     matrix is an interpolation matrix so the updated GridFunction will
-      //     still represent the same function as before refinement.
-      fespace.Update();
-      x.Update();
-
-      // 20. Load balance the mesh, and update the space and solution. Currently
-      //     available only for nonconforming meshes.
-      if (pmesh.Nonconforming())
-      {
-         pmesh.Rebalance();
-
-         // Update the space and the GridFunction. This time the update matrix
-         // redistributes the GridFunction among the processors.
-         fespace.Update();
-         x.Update();
-      }
-
-      // 21. Inform also the bilinear and linear forms that the space has
-      //     changed.
-      a.Update();
-      b.Update();
-   }
-
-   MPI_Finalize();
-   return 0;
-}

examples/ex7p.cpp

@@ -282,10 +282,10 @@ int main(int argc, char *argv[])
    delete b;
 
    // 12. Compute and print the L^2 norm of the error.
-   double err = x.ComputeL2Error(sol_coef);
+   double error = x.ComputeL2Error(sol_coef);
    if (myid == 0)
    {
-      cout << "\nL2 norm of error: " << err << endl;
+      cout << "\nL2 norm of error: " << error << endl;
    }
 
    // 13. Save the refined mesh and the solution. This output can be viewed

examples/fichera-mixed-set.mesh

@@ -1,162 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-14
-1 4 13 15 21 25
-1 4 12 13 15 21
-1 4 13 21 22 25
-1 4 15 24 21 25
-1 4 13 15 25 16
-1 5  0  1  4  3  9 10 13 12
-1 5  8  9 12 11 17 18 21 20
-1 5  2  3  6  5 11 12 15 14
-1 6  3  4  6 12 13 15
-1 6  4  7  6 13 16 15
-1 6 12 13 21  9 10 18
-1 6 13 22 21 10 19 18
-1 6 11 14 20 12 15 21
-1 6 15 21 24 14 20 23
-
-boundary
-30
- 1 3  5  6  3  2
- 2 2  3  6  4
- 2 2  4  6  7
- 3 3  3  4  1  0
- 4 3 11 12  9  8
- 5 3  2  3 12 11
- 6 3  0  1 10  9
- 7 2  9 10 18
- 7 2 10 19 18
- 8 3  8  9 18 17
- 9 3  1  4 13 10
-10 3  4  7 16 13
-11 2 13 16 25
-11 2 13 25 22
-12 3 10 13 22 19
-13 3  7  6 15 16
-14 3  6  5 14 15
-15 3 15 14 23 24
-16 2 16 15 25
-16 2 15 24 25
-17 3  5  2 11 14
-18 3  3  0  9 12
-19 3 11  8 17 20
-20 2 11 20 14
-20 2 14 20 23
-21 3 17 18 21 20
-22 3 18 19 22 21
-23 2 21 22 25
-23 2 21 25 24
-24 3 20 21 24 23
-
-vertices
-26
-3
-0 -1 -1
-1 -1 -1
--1 0 -1
-0 0 -1
-1 0 -1
--1 1 -1
-0 1 -1
-1 1 -1
--1 -1 0
-0 -1 0
-1 -1 0
--1 0 0
-0 0 0
-1 0 0
--1 1 0
-0 1 0
-1 1 0
--1 -1 1
-0 -1 1
-1 -1 1
--1 0 1
-0 0 1
-1 0 1
--1 1 1
-0 1 1
-1 1 1
-
-MFEM sets v1.0
-
-vertex_sets
-1
-
-Origin
-1
-12
-
-edge_sets
-2
-
-Axes
-3
-12 13
-12 15
-12 21
-
-Negative Axes
-3
-12 9
-12 11
-12 3
-
-face_sets
-2
-
-Interior Corner
-3
-3 11 12  9 8
-3  2  3 12 11
-3  3  0  9 12
-
-Exterior Corner
-15
-2 13 16 25
-2 13 25 22
-2 16 15 25
-2 15 24 25
-2 21 22 25
-2 21 25 24
-3 10 13 22 19
-3 4 7 16 13
-3 1 4 13 10
-3 7 6 15 16
-3 6 5 14 15
-3 15 14 23 24
-3 20 21 24 23
-3 18 19 22 21
-3 17 18 21 20
-
-element_sets
-3
-
-Interior Corner
-3
-5 6 7
-
-Exterior Corner
-5
-0 1 2 3 4
-
-Steps
-3
-6 8 9

examples/fichera-set.mesh

@@ -1,145 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-#
-
-dimension
-3
-
-elements
-7
-1 5 0 1 4 3 9 10 13 12
-1 5 3 4 7 6 12 13 16 15
-1 5 2 3 6 5 11 12 15 14
-1 5 8 9 12 11 17 18 21 20
-1 5 9 10 13 12 18 19 22 21
-1 5 12 13 16 15 21 22 25 24
-1 5 11 12 15 14 20 21 24 23
-
-boundary
-24
-1 3 5 6 3 2
-2 3 6 7 4 3
-3 3 3 4 1 0
-4 3 11 12 9 8
-5 3 2 3 12 11
-6 3 0 1 10 9
-7 3 9 10 19 18
-8 3 8 9 18 17
-9 3 1 4 13 10
-10 3 4 7 16 13
-11 3 13 16 25 22
-12 3 10 13 22 19
-13 3 7 6 15 16
-14 3 6 5 14 15
-15 3 15 14 23 24
-16 3 16 15 24 25
-17 3 5 2 11 14
-18 3 3 0 9 12
-19 3 11 8 17 20
-20 3 14 11 20 23
-21 3 17 18 21 20
-22 3 18 19 22 21
-23 3 21 22 25 24
-24 3 20 21 24 23
-
-vertices
-26
-3
-0 -1 -1
-1 -1 -1
--1 0 -1
-0 0 -1
-1 0 -1
--1 1 -1
-0 1 -1
-1 1 -1
--1 -1 0
-0 -1 0
-1 -1 0
--1 0 0
-0 0 0
-1 0 0
--1 1 0
-0 1 0
-1 1 0
--1 -1 1
-0 -1 1
-1 -1 1
--1 0 1
-0 0 1
-1 0 1
--1 1 1
-0 1 1
-1 1 1
-
-MFEM sets v1.0
-
-vertex_sets
-1
-
-Origin
-1
-12
-
-edge_sets
-2
-
-Axes
-3
-12 13
-12 15
-12 21
-
-Negative Axes
-3
-12 9
-12 11
-12 3
-
-face_sets
-2
-
-Interior Corner
-3
-3 11 12  9 8
-3  2  3 12 11
-3  3  0  9 12
-
-Exterior Corner
-12
-3 13 16 25 22
-3 16 15 24 25
-3 21 22 25 24
-3 10 13 22 19
-3 4 7 16 13
-3 1 4 13 10
-3 7 6 15 16
-3 6 5 14 15
-3 15 14 23 24
-3 20 21 24 23
-3 18 19 22 21
-3 17 18 21 20
-
-element_sets
-3
-
-Interior Corner
-3
-0 2 3
-
-Exterior Corner
-1
-5
-
-Steps
-2
-1 3

examples/star-mixed-set.mesh

@@ -1,158 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-30
-1 3 0 11 26 14
-1 3 0 14 27 17
-1 3 0 17 28 20
-1 3 0 20 29 23
-1 3 0 23 30 11
-1 2 11 1 26
-1 2 1 12 26
-1 3 26 12 3 13
-1 2 26 13 2
-1 2 14 26 2
-1 2 14 2 27
-1 2 2 15 27
-1 3 27 15 5 16
-1 2 27 16 4
-1 2 17 27 4
-1 2 17 4 28
-1 2 4 18 28
-1 3 28 18 7 19
-1 2 28 19 6
-1 2 20 28 6
-1 2 20 6 29
-1 2 6 21 29
-1 3 29 21 9 22
-1 2 29 22 8
-1 2 23 29 8
-1 2 23 8 30
-1 2 8 24 30
-1 3 30 24 10 25
-1 2 30 25 1
-1 2 11 30 1
-
-boundary
-20
-1 1 13 2
-1 1 12 3
-1 1 16 4
-1 1 15 5
-1 1 19 6
-1 1 18 7
-1 1 22 8
-1 1 21 9
-1 1 25 1
-1 1 24 10
-1 1 3 13
-1 1 1 12
-1 1 5 16
-1 1 2 15
-1 1 7 19
-1 1 4 18
-1 1 9 22
-1 1 6 21
-1 1 10 25
-1 1 8 24
-
-vertices
-31
-2
-0 0
-1 0
-0.309017 0.951057
-1.30902 0.951057
--0.809017 0.587785
--0.5 1.53884
--0.809017 -0.587785
--1.61803 0
-0.309017 -0.951057
--0.5 -1.53884
-1.30902 -0.951057
-0.5 0
-1.15451 0.475529
-0.809019 0.951057
-0.154508 0.475529
--0.0954915 1.24495
--0.654508 1.06331
--0.404508 0.293893
--1.21352 0.293893
--1.21352 -0.293892
--0.404508 -0.293893
--0.654508 -1.06331
--0.0954915 -1.24495
-0.154508 -0.475529
-0.809019 -0.951057
-1.15451 -0.475529
-0.654509 0.475529
--0.25 0.769421
--0.809016 0
--0.25 -0.76942
-0.654509 -0.475529
-
-MFEM sets v1.0
-
-vertex_sets
-3
-
-Origin
-1
-0
-
-Tent
-5
-1 2 4 6 8
-
-Gazebo
-5
-3 5 7 9 10
-
-edge_sets
-2
-
-Columbine
-5
-1 11
-2 14
-4 17
-6 20
-8 23
-
-Lily
-5
-0 11
-0 14
-0 17
-0 20
-0 23
-
-element_sets
-3
-
-Flying Squirrel
-3
-7 17 27
-
-Sea Lion
-4
-12 17 22 27
-
-Pinwheel
-5
-8 13 18 23 28

examples/star-set.mesh

@@ -1,143 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-#
-
-dimension
-2
-
-elements
-20
-1 3 0 11 26 14
-1 3 0 14 27 17
-1 3 0 17 28 20
-1 3 0 20 29 23
-1 3 0 23 30 11
-1 3 11 1 12 26
-1 3 26 12 3 13
-1 3 14 26 13 2
-1 3 14 2 15 27
-1 3 27 15 5 16
-1 3 17 27 16 4
-1 3 17 4 18 28
-1 3 28 18 7 19
-1 3 20 28 19 6
-1 3 20 6 21 29
-1 3 29 21 9 22
-1 3 23 29 22 8
-1 3 23 8 24 30
-1 3 30 24 10 25
-1 3 11 30 25 1
-
-boundary
-20
-1 1 13 2
-1 1 12 3
-1 1 16 4
-1 1 15 5
-1 1 19 6
-1 1 18 7
-1 1 22 8
-1 1 21 9
-1 1 25 1
-1 1 24 10
-1 1 3 13
-1 1 1 12
-1 1 5 16
-1 1 2 15
-1 1 7 19
-1 1 4 18
-1 1 9 22
-1 1 6 21
-1 1 10 25
-1 1 8 24
-
-vertices
-31
-2
-0 0
-1 0
-0.309017 0.951057
-1.30902 0.951057
--0.809017 0.587785
--0.5 1.53884
--0.809017 -0.587785
--1.61803 0
-0.309017 -0.951057
--0.5 -1.53884
-1.30902 -0.951057
-0.5 0
-1.15451 0.475529
-0.809019 0.951057
-0.154508 0.475529
--0.0954915 1.24495
--0.654508 1.06331
--0.404508 0.293893
--1.21352 0.293893
--1.21352 -0.293892
--0.404508 -0.293893
--0.654508 -1.06331
--0.0954915 -1.24495
-0.154508 -0.475529
-0.809019 -0.951057
-1.15451 -0.475529
-0.654509 0.475529
--0.25 0.769421
--0.809016 0
--0.25 -0.76942
-0.654509 -0.475529
-
-MFEM sets v1.0
-
-vertex_sets
-3
-
-Origin
-1
-0
-
-Tent
-5
-1 2 4 6 8
-
-Gazebo
-5
-3 5 7 9 10
-
-edge_sets
-2
-
-Columbine
-5
-1 11
-2 14
-4 17
-6 20
-8 23
-
-Lily
-5
-0 11
-0 14
-0 17
-0 20
-0 23
-
-element_sets
-2
-
-Flying Squirrel
-3
-6 12 18
-
-Sea Lion
-4
-9 12 15 18

fem/CMakeLists.txt

@@ -21,6 +21,8 @@ set(SRCS
   bilininteg_dgtrace_ea.cpp
   bilininteg_diffusion_mf.cpp
   bilininteg_diffusion_pa.cpp
+  bilininteg_diffusion_pa_diag_fast.cpp
+  bilininteg_diffusion_pa_fast.cpp
   bilininteg_diffusion_ea.cpp
   bilininteg_divergence.cpp
   bilininteg_hcurl.cpp
@@ -29,6 +31,11 @@ set(SRCS
   bilininteg_gradient.cpp
   bilininteg_mass_mf.cpp
   bilininteg_mass_pa.cpp
+  bilininteg_mass_pa_diag.cpp
+  bilininteg_mass_pa_fast.cpp
+  bilininteg_mass_pa_fast_amd.cpp
+  bilininteg_mass_pa_fast_hip.cpp
+  bilininteg_mass_pa_fast_diag.cpp
   bilininteg_mass_ea.cpp
   bilininteg_transpose_ea.cpp
   bilininteg_vecdiffusion.cpp
@@ -68,7 +75,16 @@ set(SRCS
   ceed/operator.cpp
   ceed/util.cpp
   linearform.cpp
+  linearform_ext.cpp
   lininteg.cpp
+  lininteg_domain.cpp
+  lininteg_domain_grad.cpp
+  lininteg_vector_domain.cpp
+  lininteg_vector_domain_grad.cpp
+  lor.cpp
+  lor_assembly.cpp
+  lor_assembly_ker.cpp
+  lor_assembly_ni3.cpp
   multigrid.cpp
   nonlinearform.cpp
   nonlinearform_ext.cpp
@@ -161,7 +177,12 @@ set(HDRS
   ceed/operator.hpp
   ceed/util.hpp
   linearform.hpp
+  linearform_ext.hpp
   lininteg.hpp
+  lininteg_domain.hpp
+  lininteg_domain_grad.hpp
+  lor.hpp
+  lor_assembly.hpp
   multigrid.hpp
   nonlinearform.hpp
   nonlinearform_ext.hpp

fem/bilinearform_ext.cpp

@@ -318,14 +318,31 @@ void PABilinearFormExtension::Assemble()
 void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integrators_size = integrators.Size();
 
-   const int iSz = integrators.Size();
-   if (elem_restrict && !DeviceCanUseCeed())
+   // Scan the different action types we need to launch
+   bool E2E_actions = false,
+        L2L_actions = false;
+   for (int i = 0; i < integrators_size; ++i)
+   {
+      const ActionType action_type = integrators[i]->GetActionType();
+      E2E_actions |= action_type == ActionType::E2E;
+      L2L_actions |= action_type == ActionType::L2L;
+   }
+
+   // typically this is a large vector, so store on device
+   y.UseDevice(true);
+
+   // If E2E kernels are present, do their computation on local vectors
+   if (E2E_actions && elem_restrict)
    {
       localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      for (int i = 0; i < integrators_size; ++i)
       {
-         integrators[i]->AssembleDiagonalPA(localY);
+         if (integrators[i]->GetActionType() == ActionType::E2E)
+         {
+            integrators[i]->AssembleDiagonalPA(localY);
+         }
       }
       const ElementRestriction* H1elem_restrict =
          dynamic_cast<const ElementRestriction*>(elem_restrict);
@@ -340,11 +357,20 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
    }
    else
    {
-      y.UseDevice(true); // typically this is a large vector, so store on device
+      // otherwise, initialize the y output
       y = 0.0;
-      for (int i = 0; i < iSz; ++i)
+   }
+
+   // Continue with the computation of the L2L integrators
+   if (L2L_actions || !elem_restrict)
+   {
+      for (int i = 0; i < integrators_size; ++i)
       {
-         integrators[i]->AssembleDiagonalPA(y);
+         const ActionType action = integrators[i]->GetActionType();
+         if (action == ActionType::L2L || !elem_restrict)
+         {
+            integrators[i]->AssembleDiagonalPA(y);
+         }
       }
    }
 }
@@ -383,27 +409,52 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integrators_size = integrators.Size();
 
-   const int iSz = integrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
+   // Scan the different action types we need to launch
+   bool E2E_actions = false,
+        L2L_actions = false;
+   for (int i = 0; i < integrators_size; ++i)
    {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(x, y);
-      }
+      const ActionType action_type = integrators[i]->GetActionType();
+      E2E_actions |= action_type == ActionType::E2E;
+      L2L_actions |= action_type == ActionType::L2L;
    }
-   else
+
+   // typically this is a large vector, so store on device
+   y.UseDevice(true);
+
+   // If E2E kernels are present, do their computation on local vectors
+   if (E2E_actions && elem_restrict)
    {
       elem_restrict->Mult(x, localX);
       localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      for (int i = 0; i < integrators_size; ++i)
       {
-         integrators[i]->AddMultPA(localX, localY);
+         if (integrators[i]->GetActionType() == ActionType::E2E)
+         {
+            integrators[i]->AddMultPA(localX, localY);
+         }
       }
       elem_restrict->MultTranspose(localY, y);
    }
+   else
+   {
+      // otherwise, initialize the y output
+      y = 0.0;
+   }
+
+   // Continue with the computation of the L2L integrators
+   if (L2L_actions || !elem_restrict)
+   {
+      for (int i = 0; i < integrators_size; ++i)
+      {
+         if (integrators[i]->GetActionType() == ActionType::L2L || !elem_restrict)
+         {
+            integrators[i]->AddMultPA(x, y);
+         }
+      }
+   }
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int iFISz = intFaceIntegrators.Size();

fem/bilininteg.hpp

@@ -244,6 +244,8 @@ public:
                                     Vector &flux, Vector *d_energy = NULL)
    { return 0.0; }
 
+   virtual bool SupportsBatchedLOR() const { return false; }
+
    virtual ~BilinearFormIntegrator() { }
 };
 
@@ -1977,7 +1979,7 @@ private:
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
    int dim, ne, dofs1D, quad1D;
-   Vector pa_data;
+   Vector pa_data, CoG;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
@@ -2058,6 +2060,8 @@ public:
                                          const FiniteElement &test_fe);
 
    bool SupportsCeed() const { return DeviceCanUseCeed(); }
+
+   bool SupportsBatchedLOR() const { return true; }
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */

fem/bilininteg_convection_pa.cpp

@@ -768,8 +768,8 @@ void SmemPAConvectionApply3D(const int ne,
 
 void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : memory_type;
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetFE(0);

fem/bilininteg_diffusion_pa.cpp

@@ -9,18 +9,45 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#define MFEM_DEBUG_COLOR 87
+#include "../general/debug.hpp"
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
 #include "ceed/diffusion.hpp"
+#include <string>
+
+#define MFEM_NVTX_COLOR Olive
+#include "../general/nvtx.hpp"
 
 using namespace std;
 
 namespace mfem
 {
 
+void NDK_PADiffusionAssembleDiagonal(const int dim,
+                                     const int D1D,
+                                     const int Q1D,
+                                     const int NE,
+                                     const bool symm,
+                                     const FiniteElementSpace *fes,
+                                     const DofToQuad *maps,
+                                     const Vector &D,
+                                     Vector &Y);
+
 // PA Diffusion Integrator
 
+void NDK_PADiffusionApply(const int dim,
+                          const int D1D,
+                          const int Q1D,
+                          const int NE,
+                          const Vector &CoG,
+                          const FiniteElementSpace *fes,
+                          const DofToQuad *maps,
+                          const Vector &D,
+                          const Vector &X,
+                          Vector &Y);
+
 // OCCA 2D Assemble kernel
 #ifdef MFEM_USE_OCCA
 static void OccaPADiffusionSetup2D(const int D1D,
@@ -349,10 +376,115 @@ static void PADiffusionSetup(const int dim,
    }
 }
 
+////////////////////////////////////////////////////////////////////////////////
+template<typename T> MFEM_HOST_DEVICE inline
+void HouseholderReflect(T *A, const T *v,
+                        const T b, const int m, const int n,
+                        const int row, const int col)
+{
+   for (int j = 0; j < n; j++)
+   {
+      T w = A[0*row + j*col];
+      for (int i = 1; i < m; i++) { w += v[i] * A[i*row + j*col]; }
+      A[0*row + j*col] -= b * w;
+      for (int i = 1; i < m; i++) { A[i*row + j*col] -= b * w * v[i]; }
+   }
+}
+
+template<int Q1D, typename T> MFEM_HOST_DEVICE inline
+void HouseholderApplyQ(T *A, const T *Q, const T *tau,
+                       const int k, const int row, const int col)
+{
+   T v[Q1D];
+   for (int ii=0; ii<k; ii++)
+   {
+      const int i = k-1-ii;
+      for (int j = i+1; j < Q1D; j++) { v[j] = Q[j*k+i]; }
+      // Apply Householder reflector (I - tau v v^T) coG^T
+      HouseholderReflect(&A[i*row], &v[i], tau[i], Q1D-i, Q1D, row, col);
+   }
+}
+
+template<int D1D, int Q1D, typename T> MFEM_HOST_DEVICE inline
+void QRFactorization(T *mat, T *tau)
+{
+   T v[Q1D];
+   DeviceMatrix B(mat, D1D, Q1D);
+   for (int i = 0; i < D1D; i++)
+   {
+      // Calculate Householder vector, magnitude
+      T sigma = 0.0;
+      v[i] = B(i,i);
+      for (int j = i + 1; j < Q1D; j++)
+      {
+         v[j] = B(i,j);
+         sigma += v[j] * v[j];
+      }
+      T norm = std::sqrt(v[i]*v[i] + sigma); // norm of v[i:m]
+      T Rii = -copysign(norm, v[i]);
+      v[i] -= Rii;
+      // norm of v[i:m] after modification above and scaling below
+      //   norm = sqrt(v[i]*v[i] + sigma) / v[i];
+      //   tau = 2 / (norm*norm)
+      tau[i] = 2 * v[i]*v[i] / (v[i]*v[i] + sigma);
+      for (int j=i+1; j<Q1D; j++) { v[j] /= v[i]; }
+      // Apply Householder reflector to lower right panel
+      HouseholderReflect(&mat[i*D1D+i+1], &v[i], tau[i],
+                         Q1D-i, D1D-i-1, D1D, 1);
+      // Save v
+      B(i,i) = Rii;
+      for (int j=i+1; j<Q1D; j++) { B(i,j) = v[j]; }
+   }
+}
+
+template<int D1D, int Q1D>
+void GetCollocatedGrad(const ConstDeviceMatrix &b,
+                       const ConstDeviceMatrix &g,
+                       const DeviceMatrix &CoG)
+{
+   double tau[Q1D];
+   double B1d[Q1D*D1D];
+   double G1d[Q1D*D1D];
+   DeviceMatrix B(B1d, D1D, Q1D);
+   DeviceMatrix G(G1d, D1D, Q1D);
+
+   for (int d = 0; d < D1D; d++)
+   {
+      for (int q = 0; q < Q1D; q++)
+      {
+         B(d,q) = b(q,d);
+         G(d,q) = g(q,d);
+      }
+   }
+   QRFactorization<D1D,Q1D>(B1d, tau);
+   // Apply Rinv, colograd1d = grad1d Rinv
+   for (int i = 0; i < Q1D; i++)
+   {
+      CoG(0,i) = G(0,i)/B(0,0);
+      for (int j = 1; j < D1D; j++)
+      {
+         CoG(j,i) = G(j,i);
+         for (int k = 0; k < j; k++) { CoG(j,i) -= B(j,k)*CoG(k,i); }
+         CoG(j,i) /= B(j,j);
+      }
+      for (int j = D1D; j < Q1D; j++) { CoG(j,i) = 0.0; }
+   }
+   // Apply Qtranspose, colograd = colograd Qtranspose
+   HouseholderApplyQ<Q1D>((double*)CoG, B1d, tau, D1D, 1, Q1D);
+}
+
+////////////////////////////////////////////////////////////////////////////////
 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : memory_type;
+
+   // If device options allow fast kernels, set the action type to L2L
+   action_type =
+      (Device::FastKernelsEnabled())?
+      ActionType::L2L: // all fast kernel are L2L
+      ActionType::E2E; // default is E2E
+
    // Assuming the same element type
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
@@ -365,6 +497,7 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
       MFEM_VERIFY(!VQ && !MQ && !SMQ,
                   "Only scalar coefficient supported for DiffusionIntegrator"
                   " with libCEED");
+      action_type = ActionType::L2L;
       ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
       return;
    }
@@ -490,6 +623,40 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
          }
       }
    }
+
+   if (Device::FastKernelsEnabled())
+   {
+      NVTX("CoG");
+      const int D1D = dofs1D;
+      const int Q1D = quad1D;
+      const int id = (D1D << 4) | Q1D;
+
+      CoG.SetSize(Q1D*Q1D);
+      CoG.UseDevice(true);
+      assert(CoG.UseDevice());
+
+      void (*KoG)(const ConstDeviceMatrix &b,
+                  const ConstDeviceMatrix &g,
+                  const DeviceMatrix &CoG) = nullptr;
+
+      switch (id) // orders 1~8
+      {
+         case 0x23: KoG=GetCollocatedGrad<2,3>; break; // 1
+         case 0x34: KoG=GetCollocatedGrad<3,4>; break; // 2
+         case 0x45: KoG=GetCollocatedGrad<4,5>; break; // 3
+         case 0x56: KoG=GetCollocatedGrad<5,6>; break; // 4
+         case 0x67: KoG=GetCollocatedGrad<6,7>; break; // 5
+         case 0x78: KoG=GetCollocatedGrad<7,8>; break; // 6
+         //case 0x89: KoG=GetCollocatedGrad<8,9>; break; // 7
+         //case 0x9A: KoG=GetCollocatedGrad<9,10>; break; // 8
+         default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
+      }
+
+      KoG(ConstDeviceMatrix(maps->B.HostRead(),Q1D,D1D),
+          ConstDeviceMatrix(maps->G.HostRead(),Q1D,D1D),
+          DeviceMatrix(CoG.HostReadWrite(),Q1D,Q1D));
+   }
+
    pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
    PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
                     geom->J, coeff, pa_data);
@@ -869,6 +1036,7 @@ static void SmemPADiffusionDiagonal3D(const int NE,
                   }
                }
             }
+            MFEM_SYNC_THREAD;
          }
       }
    });
@@ -925,6 +1093,11 @@ void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
    {
       ceedOp->GetDiagonal(diag);
    }
+   else if (Device::FastKernelsEnabled())
+   {
+      NDK_PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
+                                      fespace, maps, pa_data, diag);
+   }
    else
    {
       if (pa_data.Size()==0) { AssemblePA(*fespace); }
@@ -1180,6 +1353,7 @@ static void SmemPADiffusionApply2D(const int NE,
    auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+
    MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
    {
       const int tidz = MFEM_THREAD_ID(z);
@@ -1220,6 +1394,7 @@ static void SmemPADiffusionApply2D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
+
       MFEM_FOREACH_THREAD(dy,y,D1D)
       {
          MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -1545,6 +1720,7 @@ static void SmemPADiffusionApply3D(const int NE,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
+   MFEM_NVTX;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -1595,6 +1771,7 @@ static void SmemPADiffusionApply3D(const int NE,
             }
          }
       }
+      MFEM_SYNC_THREAD;
       if (MFEM_THREAD_ID(z) == 0)
       {
          MFEM_FOREACH_THREAD(dy,y,D1D)
@@ -1791,6 +1968,7 @@ static void SmemPADiffusionApply3D(const int NE,
             }
          }
       }
+      MFEM_SYNC_THREAD;
    });
 }
 
@@ -1830,8 +2008,11 @@ static void PADiffusionApply(const int dim,
       switch (ID)
       {
          case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
+         case 0x23: return SmemPADiffusionApply2D<2,3,16>(NE,symm,B,G,D,X,Y);
          case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
+         case 0x34: return SmemPADiffusionApply2D<3,4,16>(NE,symm,B,G,D,X,Y);
          case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
+         case 0x45: return SmemPADiffusionApply2D<4,5,8>(NE,symm,B,G,D,X,Y);
          case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
          case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
          case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
@@ -1868,6 +2049,12 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
    {
       ceedOp->AddMult(x, y);
    }
+   else if (Device::FastKernelsEnabled())
+   {
+      NDK_PADiffusionApply(dim, dofs1D, quad1D, ne, CoG,
+                           fespace, maps,
+                           pa_data, x, y);
+   }
    else
    {
       PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,

fem/bilininteg_diffusion_pa_diag_fast.cpp

@@ -0,0 +1,186 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#define MFEM_NVTX_COLOR Pink
+#include "../general/nvtx.hpp"
+
+#define MFEM_DEBUG_COLOR 206
+#include "../general/debug.hpp"
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "ceed/diffusion.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Shared memory PA Diffusion Diagonal 3D kernel
+template<int D1D, int Q1D>
+static void NDK_SmemPADiffusionDiag3D(const int ndofs,
+                                      const int NE,
+                                      const bool symmetric,
+                                      const int *map_,
+                                      const double *b_,
+                                      const double *g_,
+                                      const double *d_,
+                                      double *y_)
+{
+   MFEM_NVTX;
+   constexpr int DIM = 3;
+   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
+   auto b = Reshape(b_, Q1D, D1D);
+   auto g = Reshape(g_, Q1D, D1D);
+   auto D = Reshape(d_, Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
+   auto Y = Reshape(y_, D1D, D1D, D1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      MFEM_SHARED double BG[2][Q1D*D1D];
+      double (*B)[D1D] = (double (*)[D1D]) (BG+0);
+      double (*G)[D1D] = (double (*)[D1D]) (BG+1);
+      MFEM_SHARED double QQD[Q1D][Q1D][D1D];
+      MFEM_SHARED double QDD[Q1D][D1D][D1D];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+               G[q][d] = g(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      for (int i = 0; i < DIM; ++i)
+      {
+         for (int j = 0; j < DIM; ++j)
+         {
+            // first tensor contraction, along z direction
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy,y,Q1D)
+               {
+                  MFEM_FOREACH_THREAD(dz,z,D1D)
+                  {
+                     QQD[qx][qy][dz] = 0.0;
+                     for (int qz = 0; qz < Q1D; ++qz)
+                     {
+                        const int q = qx + (qy + qz * Q1D) * Q1D;
+                        const int ksym = j >= i ?
+                                         3 - (3-i)*(2-i)/2 + j:
+                                         3 - (3-j)*(2-j)/2 + i;
+                        const int k = symmetric ? ksym : (i*DIM) + j;
+                        const double O = D(q,k,e);
+                        const double Bz = B[qz][dz];
+                        const double Gz = G[qz][dz];
+                        const double L = i==2 ? Gz : Bz;
+                        const double R = j==2 ? Gz : Bz;
+                        QQD[qx][qy][dz] += L * O * R;
+                     }
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            // second tensor contraction, along y direction
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               MFEM_FOREACH_THREAD(dz,z,D1D)
+               {
+                  MFEM_FOREACH_THREAD(dy,y,D1D)
+                  {
+                     QDD[qx][dy][dz] = 0.0;
+                     for (int qy = 0; qy < Q1D; ++qy)
+                     {
+                        const double By = B[qy][dy];
+                        const double Gy = G[qy][dy];
+                        const double L = i==1 ? Gy : By;
+                        const double R = j==1 ? Gy : By;
+                        QDD[qx][dy][dz] += L * QQD[qx][qy][dz] * R;
+                     }
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            // third tensor contraction, along x direction
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1D)
+               {
+                  MFEM_FOREACH_THREAD(dx,x,D1D)
+                  {
+                     for (int qx = 0; qx < Q1D; ++qx)
+                     {
+                        const double Bx = B[qx][dx];
+                        const double Gx = G[qx][dx];
+                        const double L = i==0 ? Gx : Bx;
+                        const double R = j==0 ? Gx : Bx;
+                        const double lvr = L * QDD[qx][dy][dz] * R;
+                        const int gid = MAP(dx,dy,dz,e);
+                        const int idx = gid >= 0 ? gid : -1 - gid;
+                        AtomicAdd(Y[idx], lvr);
+                     }
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   });
+}
+
+void NDK_PADiffusionAssembleDiagonal(const int dim,
+                                     const int D1D,
+                                     const int Q1D,
+                                     const int NE,
+                                     const bool symm,
+                                     const FiniteElementSpace *fes,
+                                     const DofToQuad *maps,
+                                     const Vector &D,
+                                     Vector &Y)
+{
+   MFEM_NVTX;
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   assert(ER);
+   const int *m = ER->GatherMap().Read();
+   const double *b = maps->B.Read();
+   const double *g = maps->G.Read();
+   const double *d = D.Read();
+   double *y = Y.ReadWrite();
+
+   if (dim == 2) {  assert(false); }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return NDK_SmemPADiffusionDiag3D<2,2>(ND,NE,symm,m,b,g,d,y);
+         case 0x23: return NDK_SmemPADiffusionDiag3D<2,3>(ND,NE,symm,m,b,g,d,y);
+         case 0x34: return NDK_SmemPADiffusionDiag3D<3,4>(ND,NE,symm,m,b,g,d,y);
+         case 0x45: return NDK_SmemPADiffusionDiag3D<4,5>(ND,NE,symm,m,b,g,d,y);
+         case 0x46: return NDK_SmemPADiffusionDiag3D<4,6>(ND,NE,symm,m,b,g,d,y);
+         case 0x56: return NDK_SmemPADiffusionDiag3D<5,6>(ND,NE,symm,m,b,g,d,y);
+         case 0x67: return NDK_SmemPADiffusionDiag3D<6,7>(ND,NE,symm,m,b,g,d,y);
+         case 0x78: return NDK_SmemPADiffusionDiag3D<7,8>(ND,NE,symm,m,b,g,d,y);
+         //case 0x89: return NDK_SmemPADiffusionDiag3D<8,9>(ND,NE,symm,m,b,g,d,y);
+         //case 0x9A: return NDK_SmemPADiffusionDiag3D<9,10>(ND,NE,symm,m,b,g,d,y);
+         default: MFEM_ABORT("Not implemented!");
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+} // namespace mfem

fem/bilininteg_diffusion_pa_fast.cpp

@@ -0,0 +1,332 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#define MFEM_NVTX_COLOR Pink
+#include "../general/nvtx.hpp"
+
+#define MFEM_DEBUG_COLOR 206
+#include "../general/debug.hpp"
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "ceed/diffusion.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+////////////////////////////////////////////////////////////////////////////////
+template<int D1D, int Q1D, int NBZ, int NBK> static
+MFEM_GLOBAL MFEM_LAUNCH_BOUNDS(Q1D*Q1D*NBZ,NBK)
+void NDK_PADiffApply(const int NE,
+                     const int *MAP,
+                     const double *B,
+                     const double *G,
+                     const double *D,
+                     const double *X,
+                     double *Y)
+{
+   double r_qt, r_q[Q1D];
+   MFEM_SHARED double s_B[D1D][Q1D];
+   MFEM_SHARED double s_G[Q1D][Q1D];
+   MFEM_SHARED double s_Iq[NBZ][Q1D][Q1D][Q1D];
+   MFEM_SHARED double s_Gqr[NBZ][Q1D][Q1D];
+   MFEM_SHARED double s_Gqs[NBZ][Q1D][Q1D];
+
+   for (int be = MFEM_BLOCK_ID(x); be < (NE+NBZ-1)/NBZ; be += MFEM_GRID_DIM(x))
+   {
+      const int tz = MFEM_THREAD_ID(z);
+      const int e = be * NBZ + tz;
+      if (e>=NE) { return; }
+
+      // Scatter X
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            if (tz == 0) { s_G[j][i] = G[i+Q1D*j]; } // ok with init
+            if (tz == 0 && j<D1D) { s_B[j][i] = B[i+Q1D*j]; } // ok
+            if (j<D1D && i<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int k = 0; k < D1D; k++)
+               {
+                  const int gid = MAP[e*D1D*D1D*D1D + k*D1D*D1D + j*D1D + i];
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  r_q[k] = X[idx];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Grad1X
+      MFEM_FOREACH_THREAD(b,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(D1D)
+               for (int c=0; c<D1D; ++c) { u += s_B[c][k] * r_q[c]; }
+               s_Iq[tz][k][b][a] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Grad1Y
+      MFEM_FOREACH_THREAD(k,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b) { r_q[b] = s_Iq[tz][k][b][a]; }
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(D1D)
+               for (int b=0; b<D1D; ++b) { u += s_B[b][j] * r_q[b]; }
+               s_Iq[tz][k][j][a] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Grad1Z
+      MFEM_FOREACH_THREAD(k,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(j,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a) { r_q[a] = s_Iq[tz][k][j][a]; }
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(D1D)
+               for (int a=0; a<D1D; ++a) { u += s_B[a][i] * r_q[a]; }
+               s_Iq[tz][k][j][i] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Flush
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int k = 0; k < Q1D; ++k) { r_q[k] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Q-Function
+      MFEM_UNROLL(Q1D)
+      for (int k = 0; k < Q1D; ++k)
+      {
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(j,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(i,x,Q1D)
+            {
+               double qr = 0.0, qs = 0.0, qt = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int m = 0; m < Q1D; ++m)
+               {
+                  const double Dim = s_G[i][m];
+                  const double Djm = s_G[j][m];
+                  const double Dkm = s_G[k][m];
+                  qr += Dim * s_Iq[tz][k][j][m];
+                  qs += Djm * s_Iq[tz][k][m][i];
+                  qt += Dkm * s_Iq[tz][m][j][i];
+               }
+               //(d, Q1D,Q1D,Q1D, 6, NE);
+               const int ebase = e * 6*Q1D*Q1D*Q1D;
+               const int gbase = i + j*Q1D + k*Q1D*Q1D;
+               const double D00 = D[0*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D01 = D[1*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D02 = D[2*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D11 = D[3*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D12 = D[4*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D22 = D[5*Q1D*Q1D*Q1D + gbase + ebase];
+
+               s_Gqr[tz][j][i] = D00*qr + D01*qs + D02*qt;
+               s_Gqs[tz][j][i] = D01*qr + D11*qs + D12*qt;
+               r_qt = D02*qr + D12*qs + D22*qt;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(j,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(i,x,Q1D)
+            {
+               double Aqtmp = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int m = 0; m < Q1D; ++m)
+               {
+                  const double Dmi = s_G[m][i];
+                  const double Dmj = s_G[m][j];
+                  const double Dkm = s_G[k][m];
+                  Aqtmp += Dmi * s_Gqr[tz][j][m];
+                  Aqtmp += Dmj * s_Gqs[tz][m][i];
+                  r_q[m] += Dkm * r_qt;
+               }
+               r_q[k] += Aqtmp;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+
+      // GradZT
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { u += s_B[c][k] * r_q[k]; }
+               s_Iq[tz][c][j][i] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // GradYT
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_q[j] = s_Iq[tz][c][j][i]; }
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { u += s_B[b][j] * r_q[j]; }
+               s_Iq[tz][c][b][i] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // GradXT
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,D1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_q[i] = s_Iq[tz][c][b][i]; }
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { u += s_B[a][i] * r_q[i]; }
+               s_Iq[tz][c][b][a] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Gather
+      MFEM_FOREACH_THREAD(j,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int k = 0; k < D1D; k++)
+            {
+               const int gid = MAP[e*D1D*D1D*D1D + k*D1D*D1D + j*D1D + i];
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               const double output = s_Iq[tz][k][j][i];
+               AtomicAdd(Y[idx], output);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+}
+
+void NDK_PADiffusionApply(const int dim,
+                          const int D1D,
+                          const int Q1D,
+                          const int NE,
+                          const Vector &CoG,
+                          const FiniteElementSpace *fes,
+                          const DofToQuad *maps,
+                          const Vector &D,
+                          const Vector &X,
+                          Vector &Y)
+{
+   //dbg();
+   MFEM_NVTX;
+   assert(dim == 3);
+
+   const double *b = maps->B.Read();
+   const double *d = D.Read();
+   const double *x = X.Read();
+   double *y = Y.ReadWrite();
+
+   const int id = (D1D << 4) | Q1D;
+
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction *ER = dynamic_cast<const ElementRestriction*>(ERop);
+   assert(ER);
+   const int *map = ER->GatherMap().Read();
+
+   const auto dM = Reshape(map, D1D,D1D,D1D, NE);
+   const auto dB = Reshape(b, Q1D,D1D);
+   const auto dG = Reshape(CoG.Read(), Q1D,Q1D);
+   const auto dD = Reshape(d, Q1D,Q1D,Q1D, 6, NE);
+   const auto dX = Reshape(x, ND);
+   auto dY = Reshape(y, ND);
+
+   void (*Ker)(const int NE,
+               const int *MAP,
+               const double *B,
+               const double *G,
+               const double *D,
+               const double *X,
+               double *Y) = nullptr;
+
+   int NBZ = 1;
+
+   switch (id) // orders 1~8
+   {
+      case 0x23: Ker=NDK_PADiffApply<2,3,16,5>; NBZ=16; break; // 1
+      case 0x34: Ker=NDK_PADiffApply<3,4,8,6>; NBZ=8; break; // 2
+      case 0x45: Ker=NDK_PADiffApply<4,5,4,4>; NBZ=4; break; // 3
+      case 0x56: Ker=NDK_PADiffApply<5,6,4,3>; NBZ=4; break; // 4
+      case 0x67: Ker=NDK_PADiffApply<6,7,1,3>; break; // 5
+      case 0x78: Ker=NDK_PADiffApply<7,8,1,0>; break; // 6
+      //case 0x89: Ker=NDK_PADiffApply<8,9,1,0>; break; // 7
+      //case 0x9A: Ker=NDK_PADiffApply<9,10,1,1>; break; // 8
+      default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
+   }
+   MFEM_CONTRACT_VAR(NBZ);
+   MFEM_LAUNCH_KERNEL(Ker,(NE+NBZ-1)/NBZ,dim3(Q1D,Q1D,NBZ),0,NE,dM,dB,dG,dD,dX,dY);
+}
+
+} // namespace mfem

fem/bilininteg_mass_pa.cpp

@@ -19,14 +19,49 @@ using namespace std;
 namespace mfem
 {
 
+// Forward declaration of non-deterministic 'fast' kernels
+void NDK_PAMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const FiniteElementSpace *fes,
+                     const DofToQuad *maps,
+                     const Vector &D,
+                     const Vector &X,
+                     Vector &Y);
+
+void NDK_AMD_PAMassApply(const int dim,
+                         const int D1D,
+                         const int Q1D,
+                         const int NE,
+                         const FiniteElementSpace *fes,
+                         const DofToQuad *maps,
+                         const Vector &D,
+                         const Vector &X,
+                         Vector &Y);
+
+void NDK_HIP_PAMassApply(const int dim,
+                         const int D1D,
+                         const int Q1D,
+                         const int NE,
+                         const FiniteElementSpace *fes,
+                         const DofToQuad *maps,
+                         const Vector &D,
+                         const Vector &X,
+                         Vector &Y);
+
 // PA Mass Integrator
 
 // PA Mass Assemble kernel
 
 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : memory_type;
+
+   // If device options allow fast kernels, set the action type to L2L
+   action_type =
+      Device::FastKernelsEnabled() ? ActionType::L2L : ActionType::E2E;
 
    // Assuming the same element type
    fespace = &fes;
@@ -38,7 +73,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q, action_type);
       return;
    }
    dim = mesh->Dimension();
@@ -154,328 +189,6 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    }
 }
 
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal2D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QD[MQ1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassAssembleDiagonal2D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
-      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               // might need absolute values on next line
-               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal3D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QQD[MQ1][MQ1][MD1];
-      double QDD[MQ1][MD1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassAssembleDiagonal3D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
-      MFEM_SHARED double QDD[MQ1][MD1][MD1];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-static void PAMassAssembleDiagonal(const int dim, const int D1D,
-                                   const int Q1D, const int NE,
-                                   const Array<double> &B,
-                                   const Vector &D,
-                                   Vector &Y)
-{
-   if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
-         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
-         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
-         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
-         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
-         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
-         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
-         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
-         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
-         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
-         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
-         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
-         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
-         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
-         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
-         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
-         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
-         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else
-   {
-      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
-}
-
-
 #ifdef MFEM_USE_OCCA
 // OCCA PA Mass Apply 2D kernel
 static void OccaPAMassApply2D(const int D1D,
@@ -1151,6 +864,159 @@ static void SmemPAMassApply3D(const int NE,
    });
 }
 
+template<int D1D, int Q1D>
+void SmemPAMassApply3D_v1(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(bt_);
+   MFEM_CONTRACT_VAR(d1d);
+   MFEM_CONTRACT_VAR(q1d);
+   const auto B = Reshape(b_.Read(), Q1D,D1D);
+   const auto D = Reshape(d_.Read(), Q1D,Q1D,Q1D, NE);
+   const auto X = Reshape(x_.Read(), D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D,D1D,D1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      double u[Q1D];
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
+
+      // Load input, B & X interpolation
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            s_B[qx][dy] = B(qx,dy);
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double Bx = B(qx,dx);
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += X(dx,dy,dz,e) * Bx;
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[dz][dy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Y interpolation
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double zyX = s_q[dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { s_q[dz][qy][qx] = u[qy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Z interpolation, Q-function & Zt projection
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Z interpolation
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double zYX = s_q[dz][qy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
+            }
+
+            // Q-function
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               s_q[qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
+            }
+
+            // Zt projection
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double ZYX = s_q[qz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[dz][qy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Yt projection
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double zYX = s_q[dz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { s_q[dz][dy][qx] = u[dy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Xt projection & save output
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double zyX = s_q[dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double output = u[dx];
+               Y(dx,dy,dz,e) += output;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 static void PAMassApply(const int dim,
                         const int D1D,
                         const int Q1D,
@@ -1201,30 +1067,47 @@ static void PAMassApply(const int dim,
    }
    else if (dim == 3)
    {
+      const int ver = Device::KernelsVersion();
+      const int id = (ver << 8) | (D1D << 4) | Q1D;
+
+      static int ini = 0;
+      if (!ini++) { printf("\033[33mkernel #0x%x\033[m\n",id); }
+
       switch (id)
       {
-         case 0x22: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
-         case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
-         case 0x26: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
-         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
-         case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
-         case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
-         case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
-         case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
-         case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
-         default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
+         case 0x123: return SmemPAMassApply3D_v1<2,3>(NE,B,Bt,D,X,Y);
+         case 0x124: return SmemPAMassApply3D_v1<2,4>(NE,B,Bt,D,X,Y);
+         case 0x134: return SmemPAMassApply3D_v1<3,4>(NE,B,Bt,D,X,Y);
+         case 0x136: return SmemPAMassApply3D_v1<3,6>(NE,B,Bt,D,X,Y);
+         case 0x145: return SmemPAMassApply3D_v1<4,5>(NE,B,Bt,D,X,Y);
+         case 0x148: return SmemPAMassApply3D_v1<4,8>(NE,B,Bt,D,X,Y);
+         case 0x156: return SmemPAMassApply3D_v1<5,6>(NE,B,Bt,D,X,Y);
+         case 0x158: return SmemPAMassApply3D_v1<5,8>(NE,B,Bt,D,X,Y);
+         case 0x167: return SmemPAMassApply3D_v1<6,7>(NE,B,Bt,D,X,Y);
+         case 0x178: return SmemPAMassApply3D_v1<7,8>(NE,B,Bt,D,X,Y);
+
+         case 0x022: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
+         case 0x023: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
+         case 0x024: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
+         case 0x026: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
+         case 0x034: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
+         case 0x035: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
+         case 0x036: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
+         case 0x037: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
+         case 0x045: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
+         case 0x046: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
+         case 0x048: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
+         case 0x056: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
+         case 0x058: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
+         case 0x067: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
+         case 0x078: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
+         case 0x089: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
+         case 0x09A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
+
+         default:   break; //return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
    }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Unknown kernel.");
+   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
 }
 
 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -1233,6 +1116,35 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    {
       ceedOp->AddMult(x, y);
    }
+   else if (Device::FastKernelsEnabled())
+   {
+      const int version = Device::KernelsVersion();
+      MFEM_VERIFY(version < 4 || version==7, "Unsupported version!");
+      if (version == 3) // AMD
+      {
+         NDK_AMD_PAMassApply(dim, dofs1D, quad1D, ne,
+                             fespace, maps,
+                             pa_data, x, y);
+      }
+      // 4 E-vector
+      // 5 fused
+      // 6 MMA
+      else if (version == 7) // HIP
+      {
+         NDK_HIP_PAMassApply(dim, dofs1D, quad1D, ne,
+                             fespace, maps,
+                             pa_data, x, y);
+      }
+      // 0 legacy
+      // 1 fast
+      // 2 libP
+      else
+      {
+         NDK_PAMassApply(dim, dofs1D, quad1D, ne,
+                         fespace, maps,
+                         pa_data, x, y);
+      }
+   }
    else
    {
       PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);

fem/bilininteg_mass_pa_diag.cpp

@@ -0,0 +1,358 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "ceed/mass.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+void NDK_PAMassAssembleDiagonal(const int dim,
+                                const int D1D,
+                                const int Q1D,
+                                const int NE,
+                                const FiniteElementSpace *fes,
+                                const DofToQuad *maps,
+                                const Vector &D,
+                                Vector &Y);
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAMassAssembleDiagonal2D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QD[MQ1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+static void SmemPAMassAssembleDiagonal2D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
+      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               // might need absolute values on next line
+               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAMassAssembleDiagonal3D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QQD[MQ1][MQ1][MD1];
+      double QDD[MQ1][MD1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void SmemPAMassAssembleDiagonal3D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
+      MFEM_SHARED double QDD[MQ1][MD1][MD1];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+static void PAMassAssembleDiagonal(const int dim, const int D1D,
+                                   const int Q1D, const int NE,
+                                   const Array<double> &B,
+                                   const Vector &D,
+                                   Vector &Y)
+{
+   if (dim == 2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
+         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
+         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
+         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
+         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
+         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
+         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
+         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
+         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
+         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
+         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
+         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
+         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
+         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
+         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
+         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
+         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
+         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void MassIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->GetDiagonal(diag);
+   }
+   else if (Device::FastKernelsEnabled())
+   {
+      NDK_PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne,
+                                 fespace, maps,
+                                 pa_data, diag);
+   }
+   else
+   {
+      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
+}
+
+} // namespace mfem

fem/bilininteg_mass_pa_fast.cpp

@@ -0,0 +1,684 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "gridfunc.hpp"
+#include "restriction.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Fast '0' non-deterministic 3D mass kernel
+template<int D1D, int Q1D>
+void NDK_SmemPAMassApply3D(const int ndofs,
+                           const int NE,
+                           const int *map,
+                           const double *b_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_)
+{
+   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x_, ndofs);
+   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      MFEM_SHARED double sDQ[Q1D*Q1D];
+      double (*B)[D1D] = (double (*)[D1D]) sDQ;
+      double (*Bt)[Q1D] = (double (*)[Q1D]) sDQ;
+      MFEM_SHARED double sm0[Q1D*Q1D*Q1D];
+      MFEM_SHARED double sm1[Q1D*Q1D*Q1D];
+      double (*DDQ)[D1D][Q1D] = (double (*)[D1D][Q1D]) sm1;
+      double (*DQQ)[Q1D][Q1D] = (double (*)[Q1D][Q1D]) sm0;
+      double (*QQQ)[Q1D][Q1D] = (double (*)[Q1D][Q1D]) sm1;
+      double (*QQD)[Q1D][D1D] = (double (*)[Q1D][D1D]) sm0;
+      double (*QDD)[D1D][D1D] = (double (*)[D1D][D1D]) sm1;
+
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            B[qx][dy] = b(qx,dy);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  if (map)
+                  {
+                     const int gid = MAP(dx, dy, dz, e);
+                     const int idx = gid >= 0 ? gid : -1 - gid;
+                     u[dz] += X(idx) * B[qx][dx];
+                  }
+                  else
+                  {
+                     u[dz] += X1(dx,dy,dz,e) * B[qx][dx];
+                  }
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               DDQ[dz][dy][qx] = u[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; dz++)
+               {
+                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               DQQ[dz][qy][qx] = u[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; qz++)
+               {
+                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+               }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               QQQ[qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(d,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt[d][q] = b(q,d);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+               }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QQD[qz][qy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+               }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QDD[qz][dy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               if (map)
+               {
+                  const int gid = MAP(dx, dy, dz, e);
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  AtomicAdd(Y(idx), u[dz]);
+               }
+               else
+               {
+                  Y1(dx,dy,dz,e) += u[dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+// Fast '1' non-deterministic 3D mass kernel
+// Smem version melded toward registers + BZ-batch
+template<int D1D, int Q1D, int NBZ>
+void NDK_SmRgPAMassApply3D(const int ndofs,
+                           const int NE,
+                           const int *map,
+                           const double *b_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_)
+{
+   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
+   const auto B = Reshape(b_, Q1D, D1D);
+   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x_, ndofs);
+   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
+
+   MFEM_FORALL_3D(be, (NE+NBZ-1)/NBZ, Q1D, Q1D, NBZ,
+   {
+      double u[Q1D];
+      const int tz = MFEM_THREAD_ID(z);
+      const int e = be * MFEM_THREAD_SIZE(z) + tz;
+
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
+
+      // Load input, B & X interpolation
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            if (tz == 0) { s_B[qx][dy] = B(qx,dy); }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double Bx = B(qx,dx);
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  const int gid = map ? MAP(dx,dy,dz,e) : 0;
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  u[dz] += (map ? X(idx) : X1(dx,dy,dz,e)) * Bx;
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Y interpolation
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Z interpolation, Q-function & Zt projection
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Z interpolation
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
+            }
+
+            // Q-function
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               s_q[tz][qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
+            }
+
+            // Zt projection
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double ZYX = s_q[tz][qz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Yt projection
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Xt projection & save output
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double output = u[dx];
+               if (map)
+               {
+                  const int gid = MAP(dx,dy,dz,e);
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  AtomicAdd(Y(idx), output);
+               }
+               else
+               {
+                  Y1(dx,dy,dz,e) += output;
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+// Fast '2' non-deterministic 3D mass kernel
+// Smem version with registers
+template<int D1D, int Q1D>
+void NDK_RegsPAMassApply3D(const int ndofs,
+                           const int NE,
+                           const int *map,
+                           const double *b_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_)
+{
+   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
+   const auto B = Reshape(b_, Q1D,D1D);
+   const auto D = Reshape(d_, Q1D,Q1D,Q1D, NE);
+   const auto X = Reshape(x_, ndofs);
+   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      double r_wk[Q1D];
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
+
+      // Load s_B, load X in shared memory
+      MFEM_FOREACH_THREAD(b,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,Q1D)
+         {
+            if (a<D1D) { s_B[b][a] = B(b,a); }
+
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = 0.0; }
+
+            if (a<D1D && b<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int c=0; c<D1D; ++c)
+               {
+                  const int gid = map ? MAP(a,b,c,e) : 0;
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  s_q[c][b][a] = map ? X(idx) : X1(a,b,c,e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Interpolate in X direction
+      MFEM_FOREACH_THREAD(c,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,Q1D)
+         {
+            if (b<D1D && c<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int a=0; a<D1D; ++a)
+               {
+                  const double q_cba = s_q[c][b][a];
+                  MFEM_UNROLL(Q1D)
+                  for (int i=0; i<Q1D; ++i) { r_wk[i] += s_B[i][a]*q_cba; }
+               }
+               // reg => s_mem
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { s_q[c][b][i] = r_wk[i]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Interpolate in Y direction
+      MFEM_FOREACH_THREAD(c,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            if (c<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int b=0; b<D1D; ++b)
+               {
+                  const double q_cbi = s_q[c][b][i];
+                  MFEM_UNROLL(Q1D)
+                  for (int j=0; j<Q1D; ++j) { r_wk[j] += s_B[j][b]*q_cbi; }
+               }
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { s_q[c][j][i] = r_wk[j]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            // Interpolate in Z direction
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cji = s_q[c][j][i];
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { r_wk[k] += s_B[k][c]*q_cji; }
+            }
+
+            // Scale by Jacobian and integration weights
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] *= D(i,j,k,e); }
+
+            // Project back in Z direction
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               double q_cji = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { q_cji += s_B[k][c] * r_wk[k]; }
+               s_q[c][j][i] = q_cji;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Project back in Y direction
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b)
+            {
+               double q_cbi = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { q_cbi += s_B[j][b] * r_wk[j]; }
+               s_q[c][b][i] = q_cbi;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Project back in X direction
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,D1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a)
+            {
+               double q_cba = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { q_cba += s_B[i][a] * r_wk[i]; }
+               s_q[c][b][a] = q_cba;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Save back to memory
+      MFEM_FOREACH_THREAD(b,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cba = s_q[c][b][a];
+               const int gid = map ? MAP(a,b,c,e) : 0;
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(map?Y(idx):Y1(a,b,c,e), q_cba);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+void NDK_PAMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const FiniteElementSpace *fes,
+                     const DofToQuad *maps,
+                     const Vector &D,
+                     const Vector &X,
+                     Vector &Y)
+{
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   const int *map = ER ? ER->GatherMap().Read() : nullptr;
+   const double *b = maps->B.Read();
+   const double *d = D.Read();
+   const double *x = X.Read();
+   double *y = Y.ReadWrite();
+
+   assert(dim == 3);
+   const int ver = Device::KernelsVersion();
+   const int id = (ver << 8) | (D1D << 4) | Q1D;
+
+   //printf("\033[32mkernel #0x%x\033[m\n",id); fflush(0);
+
+   switch (id) // orders 1~6
+   {
+      // Fast '0': Legacy non-deterministic 3D mass kernel
+      case 0x023: return NDK_SmemPAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
+      case 0x024: return NDK_SmemPAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
+      case 0x034: return NDK_SmemPAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
+      case 0x036: return NDK_SmemPAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
+      case 0x045: return NDK_SmemPAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
+      case 0x046: return NDK_SmemPAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
+      case 0x048: return NDK_SmemPAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
+      case 0x056: return NDK_SmemPAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
+      case 0x058: return NDK_SmemPAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
+      case 0x067: return NDK_SmemPAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
+      case 0x078: return NDK_SmemPAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
+
+      // Fast '1': Legacy & half smem non-deterministic 3D mass kernel + Z-batch
+      case 0x123: return NDK_SmRgPAMassApply3D<2,3,32>(ND,NE,map,b,d,x,y);//1
+      case 0x124: return NDK_SmRgPAMassApply3D<2,4,16>(ND,NE,map,b,d,x,y);
+      case 0x134: return NDK_SmRgPAMassApply3D<3,4,16>(ND,NE,map,b,d,x,y);//2
+      case 0x136: return NDK_SmRgPAMassApply3D<3,6,8>(ND,NE,map,b,d,x,y);
+      case 0x145: return NDK_SmRgPAMassApply3D<4,5,4>(ND,NE,map,b,d,x,y);//3
+      case 0x146: return NDK_SmRgPAMassApply3D<4,6,4>(ND,NE,map,b,d,x,y);
+      case 0x148: return NDK_SmRgPAMassApply3D<4,8,4>(ND,NE,map,b,d,x,y);
+      case 0x156: return NDK_SmRgPAMassApply3D<5,6,4>(ND,NE,map,b,d,x,y);//4
+      case 0x158: return NDK_SmRgPAMassApply3D<5,8,1>(ND,NE,map,b,d,x,y);
+      case 0x167: return NDK_SmRgPAMassApply3D<6,7,1>(ND,NE,map,b,d,x,y);//5
+      case 0x178: return NDK_SmRgPAMassApply3D<7,8,1>(ND,NE,map,b,d,x,y);//6
+
+      // Fast '2': libP non-deterministic 3D mass kernel
+      case 0x223: return NDK_RegsPAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
+      case 0x224: return NDK_RegsPAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
+      case 0x234: return NDK_RegsPAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
+      case 0x236: return NDK_RegsPAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
+      case 0x245: return NDK_RegsPAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
+      case 0x246: return NDK_RegsPAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
+      case 0x248: return NDK_RegsPAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
+      case 0x256: return NDK_RegsPAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
+      case 0x258: return NDK_RegsPAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
+      case 0x267: return NDK_RegsPAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
+      case 0x278: return NDK_RegsPAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
+
+      // Fast '3': libP + AMD specific non-deterministic 3D mass kernel
+      // in bilininteg_mass_pa_fast_amd.cpp
+
+      default: break;
+   }
+
+   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
+}
+
+} // namespace mfem

fem/bilininteg_mass_pa_fast_amd.cpp

@@ -0,0 +1,250 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "gridfunc.hpp"
+#include "restriction.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Fast '3' non-deterministic 3D mass kernel
+template<int D1D, int Q1D>
+void NDK_AMD_PAMassApply3D(const int ndofs,
+                           const int NE,
+                           const int *map,
+                           const double *b_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_)
+{
+   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
+   const auto B = Reshape(b_, Q1D,D1D);
+   const auto D = Reshape(d_, Q1D,Q1D,Q1D, NE);
+   const auto X = Reshape(x_, ndofs);
+   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      double r_wk[Q1D];
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
+
+      // Load s_B, load X in shared memory
+      MFEM_FOREACH_THREAD(b,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,Q1D)
+         {
+            if (a<D1D) { s_B[b][a] = B(b,a); }
+
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = 0.0; }
+
+            if (a<D1D && b<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int c=0; c<D1D; ++c)
+               {
+                  const int gid = map ? MAP(a,b,c,e) : 0;
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  s_q[c][b][a] = map ? X(idx) : X1(a,b,c,e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Interpolate in X direction
+      MFEM_FOREACH_THREAD(c,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,Q1D)
+         {
+            if (b<D1D && c<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int a=0; a<D1D; ++a)
+               {
+                  const double q_cba = s_q[c][b][a];
+                  MFEM_UNROLL(Q1D)
+                  for (int i=0; i<Q1D; ++i) { r_wk[i] += s_B[i][a]*q_cba; }
+               }
+               // reg => s_mem
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { s_q[c][b][i] = r_wk[i]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Interpolate in Y direction
+      MFEM_FOREACH_THREAD(c,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            if (c<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int b=0; b<D1D; ++b)
+               {
+                  const double q_cbi = s_q[c][b][i];
+                  MFEM_UNROLL(Q1D)
+                  for (int j=0; j<Q1D; ++j) { r_wk[j] += s_B[j][b]*q_cbi; }
+               }
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { s_q[c][j][i] = r_wk[j]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            // Interpolate in Z direction
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cji = s_q[c][j][i];
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { r_wk[k] += s_B[k][c]*q_cji; }
+            }
+
+            // Scale by Jacobian and integration weights
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] *= D(i,j,k,e); }
+
+            // Project back in Z direction
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               double q_cji = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { q_cji += s_B[k][c] * r_wk[k]; }
+               s_q[c][j][i] = q_cji;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Project back in Y direction
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b)
+            {
+               double q_cbi = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { q_cbi += s_B[j][b] * r_wk[j]; }
+               s_q[c][b][i] = q_cbi;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Project back in X direction
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,D1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a)
+            {
+               double q_cba = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { q_cba += s_B[i][a] * r_wk[i]; }
+               s_q[c][b][a] = q_cba;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Save back to memory
+      MFEM_FOREACH_THREAD(b,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cba = s_q[c][b][a];
+               const int gid = map ? MAP(a,b,c,e) : 0;
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(map?Y(idx):Y1(a,b,c,e), q_cba);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+void NDK_AMD_PAMassApply(const int dim,
+                         const int D1D,
+                         const int Q1D,
+                         const int NE,
+                         const FiniteElementSpace *fes,
+                         const DofToQuad *maps,
+                         const Vector &D,
+                         const Vector &X,
+                         Vector &Y)
+{
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   const int *map = ER ? ER->GatherMap().Read() : nullptr;
+   const double *b = maps->B.Read();
+   const double *d = D.Read();
+   const double *x = X.Read();
+   double *y = Y.ReadWrite();
+
+   assert(dim == 3);
+   const int ver = Device::KernelsVersion();
+   const int id = (ver << 8) | (D1D << 4) | Q1D;
+
+   //printf("\033[32mkernel #0x%x\033[m\n",id); fflush(0);
+
+   switch (id) // orders 1~6
+   {
+      // Fast '3': libP + AMD specific non-deterministic 3D mass kernel
+      case 0x323: return NDK_AMD_PAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
+      case 0x324: return NDK_AMD_PAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
+      case 0x334: return NDK_AMD_PAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
+      case 0x336: return NDK_AMD_PAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
+      case 0x345: return NDK_AMD_PAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
+      case 0x346: return NDK_AMD_PAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
+      case 0x348: return NDK_AMD_PAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
+      case 0x356: return NDK_AMD_PAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
+      case 0x358: return NDK_AMD_PAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
+      case 0x367: return NDK_AMD_PAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
+      case 0x378: return NDK_AMD_PAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
+
+      default: break;
+   }
+
+   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
+}
+
+} // namespace mfem

fem/bilininteg_mass_pa_fast_diag.cpp

@@ -0,0 +1,269 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "gridfunc.hpp"
+#include "restriction.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Fast '0' non-deterministic 3D mass diag kernel
+template<int D1D, int Q1D>
+static void NDK_SmemPAMassDiag3D(const int ndofs,
+                                 const int NE,
+                                 const int *map_,
+                                 const double *b_,
+                                 const double *d_,
+                                 double *y_)
+{
+   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      MFEM_SHARED double B[Q1D][D1D];
+      MFEM_SHARED double QQD[Q1D][Q1D][D1D];
+      MFEM_SHARED double QDD[Q1D][D1D][D1D];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
+               }
+               const int gid = MAP(dx, dy, dz, e);
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(Y(idx), t);
+            }
+         }
+      }
+   });
+}
+
+// Fast non-deterministic 3D mass diag kernel
+// Less smem version with registers
+template<int D1D, int Q1D>
+static void NDK_RegsPAMassDiag3D(const int ndofs,
+                                 const int NE,
+                                 const int *map_,
+                                 const double *b_,
+                                 const double *d_,
+                                 double *y_)
+{
+   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
+   const auto B = Reshape(b_, Q1D, D1D);
+   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_, ndofs);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      double r_wk[Q1D];
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
+
+
+      MFEM_FOREACH_THREAD(d,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            if (d<D1D) { s_B[q][d] = B(q,d); }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] = D(i,j,k,e); }
+            for (int c=0; c<D1D; ++c)
+            {
+               double q_cji = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k)
+               {
+                  const double Bkc = s_B[k][c];
+                  q_cji += Bkc * Bkc * r_wk[k];
+               }
+               s_q[c][j][i] = q_cji;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b)
+            {
+               double q_cbi = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j)
+               {
+                  const double Bjb = s_B[j][b];
+                  q_cbi += Bjb * Bjb * r_wk[j];
+               }
+               s_q[c][b][i] = q_cbi;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,D1D)
+         {
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a)
+            {
+               double q_cba = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i)
+               {
+                  const double Bia = s_B[i][a];
+                  q_cba += Bia * Bia * r_wk[i];
+               }
+               s_q[c][b][a] = q_cba;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(b,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cba = s_q[c][b][a];
+               const int gid = MAP(a,b,c,e);
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(Y(idx), q_cba);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+void NDK_PAMassAssembleDiagonal(const int dim,
+                                const int D1D,
+                                const int Q1D,
+                                const int NE,
+                                const FiniteElementSpace *fes,
+                                const DofToQuad *maps,
+                                const Vector &D,
+                                Vector &Y)
+{
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   assert(ER);
+   const int *map = ER->GatherMap().Read();
+   const double *b = maps->B.Read();
+   const double *d = D.Read();
+   double *y = Y.ReadWrite();
+
+   assert(dim == 3);
+   const int ver = Device::KernelsVersion();
+   const int vid = (ver << 8) | (D1D << 4) | Q1D;
+
+   // Fast '0' non-deterministic 3D mass diag kernel
+   switch (vid) // orders 1~6
+   {
+      case 0x023: return NDK_SmemPAMassDiag3D<2,3>(ND,NE,map,b,d,y);
+      case 0x024: return NDK_SmemPAMassDiag3D<2,4>(ND,NE,map,b,d,y);
+      case 0x034: return NDK_SmemPAMassDiag3D<3,4>(ND,NE,map,b,d,y);
+      case 0x045: return NDK_SmemPAMassDiag3D<4,5>(ND,NE,map,b,d,y);
+      case 0x046: return NDK_SmemPAMassDiag3D<4,6>(ND,NE,map,b,d,y);
+      case 0x056: return NDK_SmemPAMassDiag3D<5,6>(ND,NE,map,b,d,y);
+      case 0x058: return NDK_SmemPAMassDiag3D<5,8>(ND,NE,map,b,d,y);
+      case 0x067: return NDK_SmemPAMassDiag3D<6,7>(ND,NE,map,b,d,y);
+      case 0x078: return NDK_SmemPAMassDiag3D<7,8>(ND,NE,map,b,d,y);
+      default: break;
+   }
+
+   // all other versions should fold back through these kernels
+   switch ((D1D << 4) | Q1D)
+   {
+      case 0x23: return NDK_RegsPAMassDiag3D<2,3>(ND,NE,map,b,d,y);
+      case 0x24: return NDK_RegsPAMassDiag3D<2,4>(ND,NE,map,b,d,y);
+      case 0x34: return NDK_RegsPAMassDiag3D<3,4>(ND,NE,map,b,d,y);
+      case 0x45: return NDK_RegsPAMassDiag3D<4,5>(ND,NE,map,b,d,y);
+      case 0x46: return NDK_RegsPAMassDiag3D<4,6>(ND,NE,map,b,d,y);
+      case 0x56: return NDK_RegsPAMassDiag3D<5,6>(ND,NE,map,b,d,y);
+      case 0x58: return NDK_RegsPAMassDiag3D<5,8>(ND,NE,map,b,d,y);
+      case 0x67: return NDK_RegsPAMassDiag3D<6,7>(ND,NE,map,b,d,y);
+      case 0x78: return NDK_RegsPAMassDiag3D<7,8>(ND,NE,map,b,d,y);
+      default: break;
+   }
+
+   MFEM_ABORT("Unknown kernel 0x" << std::hex << vid);
+}
+
+} // namespace mfem

fem/bilininteg_mass_pa_fast_hip.cpp

@@ -0,0 +1,224 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "gridfunc.hpp"
+#include "restriction.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Fast '7' HIP
+template<int D1D, int Q1D, int NBZ=1, int NBK=1> MFEM_GLOBAL static
+//MFEM_LAUNCH_BOUNDS(Q1D*Q1D*NBZ,NBK)
+void HIP_PAMassApply(const int NE,
+                     const int* MAP,
+                     const double* B,
+                     const double* D,
+                     const double* X,
+                     double* Y)
+{
+   double u[Q1D];
+   const int tz = MFEM_THREAD_ID(z);
+   MFEM_SHARED double s_B[Q1D][D1D];
+   MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
+
+   for (int be = MFEM_BLOCK_ID(x); be < (NE+NBZ-1)/NBZ; be += MFEM_GRID_DIM(x))
+   {
+      const int e = be * NBZ + tz;
+      if (e>=NE) { return; }
+
+      // Load input, B & X interpolation
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            if (tz == 0) { s_B[qx][dy] = B[qx+Q1D*dy]; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double Bx = B[qx+Q1D*dx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  const int gid = MAP[e*D1D*D1D*D1D+dz*D1D*D1D+dy*D1D+dx];
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  u[dz] = fma(X[idx], Bx, u[dz]);
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Y interpolation
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qy = 0; qy < Q1D; ++qy) { u[qy] = fma(zyX,s_B[qy][dy],u[qy]); }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Z interpolation, Q-function & Zt projection
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Z interpolation
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = fma(zYX,s_B[qz][dz],u[qz]); }
+            }
+
+            // Q-function
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const int idx = e*Q1D*Q1D*Q1D + qx + qy*Q1D + qz*Q1D*Q1D;
+               s_q[tz][qz][qy][qx] = u[qz] * D[idx];
+            }
+
+            // Zt projection
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double ZYX = s_q[tz][qz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz) { u[dz] = fma(ZYX,s_B[qz][dz],u[dz]); }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Yt projection
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dy = 0; dy < D1D; ++dy) { u[dy] = fma(zYX,s_B[qy][dy],u[dy]); }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Xt projection & save output
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dx = 0; dx < D1D; ++dx) { u[dx] = fma(zyX,s_B[qx][dx],u[dx]); }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double output = u[dx];
+               const int gid = MAP[e*D1D*D1D*D1D+dz*D1D*D1D+dy*D1D+dx];
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(Y[idx], output);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+}
+
+
+void NDK_HIP_PAMassApply(const int dim,
+                         const int D1D,
+                         const int Q1D,
+                         const int NE,
+                         const FiniteElementSpace *fes,
+                         const DofToQuad *maps,
+                         const Vector &d,
+                         const Vector &x,
+                         Vector &y)
+{
+   assert(dim == 3);
+
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction *ER = dynamic_cast<const ElementRestriction*>(ERop);
+   assert(ER);
+
+   const auto M = ER->GatherMap().Read();
+   const double *B = maps->B.Read();
+   const double *D = d.Read();
+   const double *X = x.Read();
+   double *Y = y.ReadWrite();
+
+   void (*Ker)(const int NE,
+               const int *M,
+               const double *B,
+               const double *D,
+               const double *X,
+               double *Y) = nullptr;
+
+   const int ver = Device::KernelsVersion();
+   const int id = (ver << 8) | (D1D << 4) | Q1D;
+
+   switch (id) // orders 1~8
+   {
+      case 0x723: Ker=HIP_PAMassApply<2,3>; break; // 1
+      case 0x734: Ker=HIP_PAMassApply<3,4>; break; // 2
+      case 0x745: Ker=HIP_PAMassApply<4,5>; break; // 3
+      case 0x756: Ker=HIP_PAMassApply<5,6>; break; // 4
+      case 0x767: Ker=HIP_PAMassApply<6,7>; break; // 5
+      case 0x778: Ker=HIP_PAMassApply<7,8>; break; // 6
+      case 0x789: Ker=HIP_PAMassApply<8,9>; break; // 7
+      case 0x79A: Ker=HIP_PAMassApply<9,10>; break; // 8
+      default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
+   }
+   MFEM_LAUNCH_KERNEL(Ker,NE,dim3(Q1D,Q1D,1),0,NE,M,B,D,X,Y);
+}
+
+} // namespace mfem

fem/bilininteg_vecmass.cpp

@@ -34,7 +34,7 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q, action_type);
       return;
    }
    dim = mesh->Dimension();

fem/ceed/mass.cpp

@@ -48,7 +48,8 @@ struct MassOperatorInfo : public OperatorInfo
 
 PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
                                    const mfem::IntegrationRule &irm,
-                                   mfem::Coefficient *Q)
+                                   mfem::Coefficient *Q,
+                                   mfem::ActionType &action_type)
    : PAIntegrator()
 {
 #ifdef MFEM_USE_CEED
@@ -57,6 +58,7 @@ PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
 #else
    MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
 #endif
+   action_type = mfem::ActionType::L2L;
 }
 
 MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,

fem/ceed/mass.hpp

@@ -27,7 +27,8 @@ class PAMassIntegrator : public PAIntegrator
 public:
    PAMassIntegrator(const mfem::FiniteElementSpace &fes,
                     const mfem::IntegrationRule &irm,
-                    mfem::Coefficient *Q);
+                    mfem::Coefficient *Q,
+                    mfem::ActionType &action_type);
 };
 
 /// Represent a MassIntegrator with AssemblyLevel::None using libCEED.

fem/ceed/util.cpp

@@ -65,6 +65,8 @@ void RemoveBasisAndRestriction(const mfem::FiniteElementSpace *fes)
          itr++;
       }
    }
+#else
+   MFEM_CONTRACT_VAR(fes);
 #endif
 }
 

fem/coefficient.cpp

@@ -21,6 +21,33 @@ namespace mfem
 
 using namespace std;
 
+// Given an ElementTransformation and IntegrationPoint in a refined mesh,
+// return the ElementTransformation of the parent coarse element, and set
+// coarse_ip to the location of the original ip within the coarse element.
+ElementTransformation *RefinedToCoarse(
+   Mesh &coarse_mesh, const ElementTransformation &T,
+   const IntegrationPoint &ip, IntegrationPoint &coarse_ip)
+{
+   Mesh &fine_mesh = *T.mesh;
+   // Get the element transformation of the coarse element containing the
+   // fine element.
+   int fine_element = T.ElementNo;
+   const CoarseFineTransformations &cf = fine_mesh.GetRefinementTransforms();
+   int coarse_element = cf.embeddings[fine_element].parent;
+   ElementTransformation *coarse_T = coarse_mesh.GetElementTransformation(
+                                        coarse_element);
+   // Transform the integration point from fine element coordinates to coarse
+   // element coordinates.
+   Geometry::Type geom = T.GetGeometryType();
+   IntegrationPointTransformation fine_to_coarse;
+   IsoparametricTransformation &emb_tr = fine_to_coarse.Transf;
+   emb_tr.SetIdentityTransformation(geom);
+   emb_tr.SetPointMat(cf.point_matrices[geom](cf.embeddings[fine_element].matrix));
+   fine_to_coarse.Transform(ip, coarse_ip);
+   coarse_T->SetIntPoint(&coarse_ip);
+   return coarse_T;
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -95,7 +122,17 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
 double GridFunctionCoefficient::Eval (ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
-   return GridF -> GetValue (T, ip, Component);
+   Mesh *gf_mesh = GridF->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      return GridF->GetValue(T, ip, Component);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      return GridF->GetValue(*coarse_T, coarse_ip, Component);
+   }
 }
 
 void TransformedCoefficient::SetTime(double t)
@@ -305,13 +342,30 @@ void VectorGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
-   GridFunc->GetVectorValue(T, ip, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetVectorValue(T, ip, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetVectorValue(*coarse_T, coarse_ip, V);
+   }
 }
 
 void VectorGridFunctionCoefficient::Eval(
    DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir)
 {
-   GridFunc->GetVectorValues(T, ir, M);
+   if (T.mesh == GridFunc->FESpace()->GetMesh())
+   {
+      GridFunc->GetVectorValues(T, ir, M);
+   }
+   else
+   {
+      VectorCoefficient::Eval(M, T, ir);
+   }
 }
 
 GradientGridFunctionCoefficient::GradientGridFunctionCoefficient (
@@ -331,13 +385,30 @@ void GradientGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void GradientGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                            const IntegrationPoint &ip)
 {
-   GridFunc->GetGradient(T, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetGradient(T, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetGradient(*coarse_T, V);
+   }
 }
 
 void GradientGridFunctionCoefficient::Eval(
    DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir)
 {
-   GridFunc->GetGradients(T, ir, M);
+   if (T.mesh == GridFunc->FESpace()->GetMesh())
+   {
+      GridFunc->GetGradients(T, ir, M);
+   }
+   else
+   {
+      VectorCoefficient::Eval(M, T, ir);
+   }
 }
 
 CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
@@ -363,7 +434,17 @@ void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                        const IntegrationPoint &ip)
 {
-   GridFunc->GetCurl(T, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetCurl(T, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetCurl(*coarse_T, V);
+   }
 }
 
 DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
@@ -375,7 +456,17 @@ DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
 double DivergenceGridFunctionCoefficient::Eval(ElementTransformation &T,
                                                const IntegrationPoint &ip)
 {
-   return GridFunc->GetDivergence(T);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      return GridFunc->GetDivergence(T);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      return GridFunc->GetDivergence(*coarse_T);
+   }
 }
 
 void VectorDeltaCoefficient::SetTime(double t)

fem/coefficient.hpp

@@ -574,6 +574,7 @@ public:
    /// Evaluate the coefficient.
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
+   using VectorCoefficient::Eval;
 };
 
 /// A general vector function coefficient

fem/conduitdatacollection.cpp

@@ -330,7 +330,7 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
          }
          else
          {
-            Node &(n_bndry_conn_conv) =
+            Node &n_bndry_conn_conv =
                n_conv["topologies"][bndry_topo_name]["elements/connectivity"];
             n_bndry_conn.to_int_array(n_bndry_conn_conv);
             bndry_indices = (n_bndry_conn_conv).value();

fem/doftrans.hpp

@@ -243,6 +243,7 @@ public:
    void TransformDual(double *v) const;
 
    void InvTransformDual(double *v) const;
+   using DofTransformation::InvTransformDual;
 };
 
 /// DoF transformation implementation for the Nedelec basis on tetrahedra

fem/eltrans.cpp

@@ -21,7 +21,8 @@ ElementTransformation::ElementTransformation()
      EvalState(0),
      geom(Geometry::INVALID),
      Attribute(-1),
-     ElementNo(-1)
+     ElementNo(-1),
+     mesh(nullptr)
 { }
 
 double ElementTransformation::EvalWeight()

fem/eltrans.hpp

@@ -75,6 +75,12 @@ public:
 
    int Attribute, ElementNo, ElementType;
 
+   /// The Mesh object containing the element.
+   /** If the element transformation belongs to a mesh, this will point to the
+       containing Mesh object. ElementNo will be the number of the element in
+       this Mesh. This will be NULL if the element does not belong to a mesh. */
+   class Mesh *mesh;
+
    ElementTransformation();
 
    /** @brief Force the reevaluation of the Jacobian in the next call. */

fem/fe/fe_base.hpp

@@ -749,11 +749,12 @@ public:
                            DenseMatrix &div) const;
 
    /** @brief Get an Array<int> that maps lexicographically ordered indices to
-       the indices of the respective nodes/dofs/basis functions. Lexicographic
-       ordering of nodes is defined in terms of reference-space coordinates
-       (x,y,z). Lexicographically ordered nodes are listed first in order of
-       increasing x-coordinate, and then in order of increasing y-coordinate,
-       and finally in order of increasing z-coordinate.
+       the indices of the respective nodes/dofs/basis functions.
+
+       Lexicographic ordering of nodes is defined in terms of reference-space
+       coordinates (x,y,z). Lexicographically ordered nodes are listed first in
+       order of increasing x-coordinate, and then in order of increasing
+       y-coordinate, and finally in order of increasing z-coordinate.
 
        For example, the six nodes of a quadratic triangle are lexicographically
        ordered as follows:

fem/fespace.cpp

@@ -561,155 +561,6 @@ void FiniteElementSpace::GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
    }
 }
 
-void FiniteElementSpace::GetEssentialVDofs(EntitySets::EntityType type,
-                                           int set_index,
-                                           Array<int> &ess_vdofs,
-                                           int component) const
-{
-   Array<int> vdofs, dofs;
-
-   ess_vdofs.SetSize(GetVSize());
-   ess_vdofs = 0;
-
-   MFEM_VERIFY(mesh->ent_sets != NULL, "Mesh object contains no "
-               "entity set information");
-   if (!mesh->ent_sets->SetExists(type, set_index))
-   {
-      ostringstream oss; oss << "Entity set of type \""
-                             << EntitySets::GetTypeName(type)
-                             << "\" and index " << set_index
-                             << " was not found.";
-
-      MFEM_VERIFY(false, oss.str().c_str());
-   }
-
-   set<int>::iterator it;
-   for (it=(*mesh->ent_sets)(type, set_index).begin();
-        it!=(*mesh->ent_sets)(type, set_index).end(); it++)
-   {
-      int ent_index = *it;
-      cout << "collecting vdofs for entity " << ent_index << "->";
-      if (component < 0)
-      {
-         switch (type)
-         {
-            case EntitySets::VERTEX:
-               GetVertexVDofs(ent_index, vdofs);
-               break;
-            case EntitySets::EDGE:
-               GetEdgeVDofs(ent_index, vdofs);
-               break;
-            case EntitySets::FACE:
-               GetFaceVDofs(ent_index, vdofs);
-               break;
-            case EntitySets::ELEMENT:
-               GetElementVDofs(ent_index, vdofs);
-               break;
-            default:
-               mfem_error("GetEssentialVDofs: Invalid entity type");
-         }
-         vdofs.Print(cout);
-         mark_dofs(vdofs, ess_vdofs);
-      }
-      else
-      {
-         switch (type)
-         {
-            case EntitySets::VERTEX:
-               GetVertexDofs(ent_index, dofs);
-               break;
-            case EntitySets::EDGE:
-               GetEdgeDofs(ent_index, dofs);
-               break;
-            case EntitySets::FACE:
-               GetFaceDofs(ent_index, dofs);
-               break;
-            case EntitySets::ELEMENT:
-               GetElementDofs(ent_index, dofs);
-               break;
-            default:
-               mfem_error("GetEssentialDofs: Invalid entity type");
-         }
-         for (int d = 0; d < dofs.Size(); d++)
-         { dofs[d] = DofToVDof(dofs[d], component); }
-         mark_dofs(dofs, ess_vdofs);
-      }
-   }
-
-   if (mesh->ncmesh)
-   {
-      Array<int> es_verts, es_edges, es_faces;
-      mesh->ncmesh->GetEntitySetClosure(type, set_index,
-                                        es_verts, es_edges, es_faces);
-      cout << "returned from get closure" << endl;
-      for (int i = 0; i < es_verts.Size(); i++)
-      {
-         if (es_verts[i] < GetNV())
-         {
-            if (component < 0)
-            {
-               GetVertexVDofs(es_verts[i], vdofs);
-               mark_dofs(vdofs, ess_vdofs);
-            }
-            else
-            {
-               GetVertexDofs(es_verts[i], dofs);
-               for (int d = 0; d < dofs.Size(); d++)
-               { dofs[d] = DofToVDof(dofs[d], component); }
-               mark_dofs(dofs, ess_vdofs);
-            }
-         }
-      }
-      for (int i = 0; i < es_edges.Size(); i++)
-      {
-         if (es_edges[i] < GetMesh()->GetNEdges())
-         {
-            if (component < 0)
-            {
-               GetEdgeVDofs(es_edges[i], vdofs);
-               mark_dofs(vdofs, ess_vdofs);
-            }
-            else
-            {
-               GetEdgeDofs(es_edges[i], dofs);
-               for (int d = 0; d < dofs.Size(); d++)
-               { dofs[d] = DofToVDof(dofs[d], component); }
-               mark_dofs(dofs, ess_vdofs);
-            }
-         }
-      }
-      for (int i = 0; i < es_faces.Size(); i++)
-      {
-         if (es_faces[i] < GetMesh()->GetNFaces())
-         {
-            if (component < 0)
-            {
-               GetFaceVDofs(es_faces[i], vdofs);
-               mark_dofs(vdofs, ess_vdofs);
-            }
-            else
-            {
-               GetFaceDofs(es_faces[i], dofs);
-               for (int d = 0; d < dofs.Size(); d++)
-               { dofs[d] = DofToVDof(dofs[d], component); }
-               mark_dofs(dofs, ess_vdofs);
-            }
-         }
-      }
-   }
-}
-
-void FiniteElementSpace::GetEssentialVDofs(EntitySets::EntityType type,
-                                           const string & set_name,
-                                           Array<int> &ess_vdofs,
-                                           int component) const
-{
-   MFEM_VERIFY(mesh->ent_sets != NULL, "Mesh object contains no "
-               "entity set information");
-   GetEssentialVDofs(type, mesh->ent_sets->GetSetIndex(type, set_name),
-                     ess_vdofs, component);
-}
-
 void FiniteElementSpace::GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
                                               Array<int> &ess_tdof_list,
                                               int component)
@@ -728,36 +579,6 @@ void FiniteElementSpace::GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
    MarkerToList(ess_tdofs, ess_tdof_list);
 }
 
-void FiniteElementSpace::GetEssentialTrueDofs(EntitySets::EntityType type,
-                                              int set_index,
-                                              Array<int> &ess_tdof_list,
-                                              int component)
-{
-   Array<int> ess_vdofs, ess_tdofs;
-   GetEssentialVDofs(type, set_index, ess_vdofs, component);
-   const SparseMatrix *R = GetConformingRestriction();
-   if (!R)
-   {
-      ess_tdofs.MakeRef(ess_vdofs);
-   }
-   else
-   {
-      R->BooleanMult(ess_vdofs, ess_tdofs);
-   }
-   MarkerToList(ess_tdofs, ess_tdof_list);
-}
-
-void FiniteElementSpace::GetEssentialTrueDofs(EntitySets::EntityType type,
-                                              const string & set_name,
-                                              Array<int> &ess_tdof_list,
-                                              int component)
-{
-   MFEM_VERIFY(mesh->ent_sets != NULL, "Mesh object contains no "
-               "entity set information");
-   GetEssentialTrueDofs(type, mesh->ent_sets->GetSetIndex(type, set_name),
-                        ess_tdof_list, component);
-}
-
 void FiniteElementSpace::GetBoundaryTrueDofs(Array<int> &boundary_dofs,
                                              int component)
 {

fem/fespace.hpp

@@ -778,19 +778,6 @@ public:
                                   Array<int> &ess_vdofs,
                                   int component = -1) const;
 
-   /** Mark degrees of freedom associated with the entity set with the
-       specified entity type and set index. */
-   virtual void GetEssentialVDofs(EntitySets::EntityType type, int set_index,
-                                  Array<int> &ess_vdofs,
-                                  int component = -1) const;
-
-   /** Mark degrees of freedom associated with the entity set with the
-       specified entity type and set index. */
-   virtual void GetEssentialVDofs(EntitySets::EntityType type,
-                                  const std::string & set_name,
-                                  Array<int> &ess_vdofs,
-                                  int component = -1) const;
-
    /** @brief Get a list of essential true dofs, ess_tdof_list, corresponding to the
        boundary attributes marked in the array bdr_attr_is_ess.
        For spaces with 'vdim' > 1, the 'component' parameter can be used
@@ -799,19 +786,6 @@ public:
                                      Array<int> &ess_tdof_list,
                                      int component = -1);
 
-   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
-       entity set specified by the given entity type and set index. */
-   virtual void GetEssentialTrueDofs(EntitySets::EntityType type, int set_index,
-                                     Array<int> &ess_tdof_list,
-                                     int component = -1);
-
-   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
-       entity set specified by the given entity type and set name. */
-   virtual void GetEssentialTrueDofs(EntitySets::EntityType type,
-                                     const std::string & set_name,
-                                     Array<int> &ess_tdof_list,
-                                     int component = -1);
-
    /** @brief Get a list of all boundary true dofs, @a boundary_dofs. For spaces
        with 'vdim' > 1, the 'component' parameter can be used to restricts the
        marked tDOFs to the specified component. Equivalent to

fem/fmsconvert.cpp

@@ -112,7 +112,7 @@ FmsFieldToGridFunction(FmsMesh fms_mesh, FmsField f, Mesh *mesh,
    // NOTE: transplanted from the FmsMeshToMesh function
    //       We should do this work once and save it.
    //--------------------------------------------------
-   FmsInt dim, n_vert, n_elem, space_dim;
+   FmsInt dim, n_elem, space_dim;
 
    // Find the first component that has coordinates - that will be the new mfem
    // mesh.
@@ -144,7 +144,6 @@ FmsFieldToGridFunction(FmsMesh fms_mesh, FmsField f, Mesh *mesh,
          n_ents[et] += num_ents;
       }
    }
-   n_vert = n_ents[FMS_VERTEX];
    //--------------------------------------------------
 
    // Interrogate the field.

fem/kernels.hpp

@@ -48,6 +48,26 @@ MFEM_HOST_DEVICE inline void LoadB(const int D1D, const int Q1D,
    MFEM_SYNC_THREAD;
 }
 
+/// Load B1d matrice into shared memory
+MFEM_HOST_DEVICE inline void LoadB(const int D1D, const int Q1D,
+                                   const ConstDeviceMatrix &b,
+                                   const DeviceMatrix &B)
+{
+   const int tidz = MFEM_THREAD_ID(z);
+
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(d,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            B(q,d) = b(q,d);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
 /// Load Bt1d matrices into shared memory
 template<int MD1, int MQ1>
 MFEM_HOST_DEVICE inline void LoadBt(const int D1D, const int Q1D,
@@ -120,6 +140,29 @@ MFEM_HOST_DEVICE inline void LoadBGt(const int D1D, const int Q1D,
    MFEM_SYNC_THREAD;
 }
 
+/// Load Bt1d & Gt1d matrices into shared memory w/o MAX
+MFEM_HOST_DEVICE inline void LoadBGt(const int D1D, const int Q1D,
+                                     const ConstDeviceMatrix &b,
+                                     const ConstDeviceMatrix &g,
+                                     const DeviceMatrix &Bt,
+                                     const DeviceMatrix &Gt)
+{
+   const int tidz = MFEM_THREAD_ID(z);
+
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(d,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt(d,q) = b(q,d);
+            Gt(d,q) = g(q,d);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
 /// Load 2D input scalar into shared memory
 template<int MD1, int NBZ>
 MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
@@ -628,6 +671,141 @@ MFEM_HOST_DEVICE inline void GradXt(const int D1D, const int Q1D,
    MFEM_SYNC_THREAD;
 }
 
+/// Atomic 2D Transposed Evaluation, 1/2
+MFEM_HOST_DEVICE inline void Atomic2DEvalYt(const int D1D, const int Q1D,
+                                            const DeviceMatrix &B,
+                                            const DeviceMatrix &QQ,
+                                            const DeviceMatrix &QD)
+{
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u = 0.0;
+         for (int qx = 0; qx < Q1D; ++qx) { u += QQ(qy,qx) * B(qx,dx); }
+         QD(qy,dx) = u;
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 2D Transposed Evaluation, 2/2
+MFEM_HOST_DEVICE inline void Atomic2DEvalXt(const int D1D, const int Q1D,
+                                            const DeviceMatrix &B,
+                                            const DeviceMatrix &Q,
+                                            const DeviceTensor<3,const int> &I,
+                                            const DeviceMatrix &Y,
+                                            const int c,
+                                            const int e,
+                                            const bool byVDIM)
+{
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u = 0.0;
+         for (int qy = 0; qy < Q1D; ++qy) { u += Q(qy,dx) * B(qy,dy); }
+         const int gid = I(dx,dy,e);
+         const int idx = gid >= 0 ? gid : -1 - gid;
+         if (byVDIM) { AtomicAdd(Y(c,idx), u); }
+         else { AtomicAdd(Y(idx,c), u); }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline
+void Atomic2DEvalTranspose(const int D1D,
+                           const int Q1D,
+                           const DeviceMatrix &B,
+                           const DeviceMatrix &QQ,
+                           const DeviceMatrix &QD,
+                           const DeviceTensor<3,const int> &I,
+                           const DeviceMatrix &Y,
+                           const int c,
+                           const int e,
+                           const bool byVDIM)
+{
+   kernels::internal::Atomic2DEvalYt(D1D,Q1D,B,QQ,QD);
+   kernels::internal::Atomic2DEvalXt(D1D,Q1D,B,QD,I,Y,c,e,byVDIM);
+}
+
+/// Atomic 2D Transposed Gradient, 1/2
+MFEM_HOST_DEVICE inline void AtomicGradYt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceMatrix &QQ0,
+                                          const DeviceMatrix &QQ1,
+                                          const DeviceMatrix &DQ0,
+                                          const DeviceMatrix &DQ1)
+{
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u = 0.0, v = 0.0;
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            u += Gt(dx,qx) * QQ0(qy,qx);
+            v += Bt(dx,qx) * QQ1(qy,qx);
+         }
+         DQ0(dx,qy) = u;
+         DQ1(dx,qy) = v;
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 2D Transposed Gradient, 2/2
+MFEM_HOST_DEVICE inline void AtomicGradXt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceMatrix &DQ0,
+                                          const DeviceMatrix &DQ1,
+                                          const DeviceTensor<3,const int> &I,
+                                          const DeviceMatrix &Y,
+                                          const int c,
+                                          const int e,
+                                          const bool byVDIM)
+{
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u = 0.0, v = 0.0;
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            u += DQ0(dx,qy) * Bt(dy,qy);
+            v += DQ1(dx,qy) * Gt(dy,qy);
+         }
+         const double sum = u + v;
+         const int gid = I(dx,dy,e);
+         const int idx = gid >= 0 ? gid : -1-gid;
+         if (byVDIM) { AtomicAdd(Y(c,idx), sum); }
+         else { AtomicAdd(Y(idx,c), sum); }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline
+void Atomic2DGradTranspose(const int D1D, const int Q1D,
+                           const DeviceMatrix &Bt,
+                           const DeviceMatrix &Gt,
+                           const DeviceMatrix &QQ0,
+                           const DeviceMatrix &QQ1,
+                           const DeviceMatrix &DQ0,
+                           const DeviceMatrix &DQ1,
+                           const DeviceTensor<3,const int> &I,
+                           const DeviceMatrix &Y,
+                           const int c,
+                           const int e,
+                           const bool byVDIM)
+{
+   kernels::internal::AtomicGradYt(D1D,Q1D,Bt,Gt,QQ0,QQ1,DQ0,DQ1);
+   kernels::internal::AtomicGradXt(D1D,Q1D,Bt,Gt,DQ0,DQ1,I,Y,c,e,byVDIM);
+}
+
 /// Load 3D scalar input vector into shared memory
 MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
                                    const DeviceTensor<4, const double> &x,
@@ -1543,6 +1721,269 @@ MFEM_HOST_DEVICE inline void GradXt(const int D1D, const int Q1D,
    }
 }
 
+/// Atomic 3D Transposed Evaluation, 1/3
+MFEM_HOST_DEVICE inline void Atomic3DEvalZt(const int D1D, const int Q1D,
+                                            double *u,
+                                            const DeviceMatrix &B,
+                                            const DeviceCube &Q)
+{
+   MFEM_FOREACH_THREAD(qx,x,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const double ZYX = Q(qz,qy,qx);
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * B(qz,dz); }
+         }
+         for (int dz = 0; dz < D1D; ++dz) { Q(dz,qy,qx) = u[dz]; }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 3D Transposed Evaluation, 2/3
+MFEM_HOST_DEVICE inline void Atomic3DEvalYt(const int D1D, const int Q1D,
+                                            double *u,
+                                            const DeviceMatrix &B,
+                                            const DeviceCube &Q)
+{
+   MFEM_FOREACH_THREAD(dz,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double zYX = Q(dz,qy,qx);
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * B(qy,dy); }
+         }
+         for (int dy = 0; dy < D1D; ++dy) { Q(dz,dy,qx) = u[dy]; }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 3D Transposed Evaluation, 3/3
+MFEM_HOST_DEVICE inline void Atomic3DEvalXt(const int D1D, const int Q1D,
+                                            double *u,
+                                            const DeviceMatrix &B,
+                                            const DeviceCube &Q,
+                                            const DeviceTensor<4,const int> &I,
+                                            const DeviceMatrix &Y,
+                                            const int c,
+                                            const int e,
+                                            const bool byVDIM)
+{
+   MFEM_FOREACH_THREAD(dz,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dy,x,D1D)
+      {
+         for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double zyX = Q(dz,dy,qx);
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * B(qx,dx); }
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double val = u[dx];
+            const int gid = I(dx,dy,dz,e);
+            const int idx = gid >= 0 ? gid : -1 - gid;
+            if (byVDIM) { AtomicAdd(Y(c,idx), val); }
+            else { AtomicAdd(Y(idx,c), val); }
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline
+void Atomic3DEvalTranspose(const int D1D,
+                           const int Q1D,
+                           double *u,
+                           const DeviceMatrix &B,
+                           const DeviceCube &Q,
+                           const DeviceTensor<4,const int> &I,
+                           const DeviceMatrix &Y,
+                           const int c,
+                           const int e,
+                           const bool byVDIM)
+{
+   kernels::internal::Atomic3DEvalZt(D1D,Q1D,u,B,Q);
+   kernels::internal::Atomic3DEvalYt(D1D,Q1D,u,B,Q);
+   kernels::internal::Atomic3DEvalXt(D1D,Q1D,u,B,Q,I,Y,c,e,byVDIM);
+}
+
+// Half of B and G are stored in shared to get B, Bt, G and Gt.
+// Indices computation for SmemPADiffusionApply3D.
+static MFEM_HOST_DEVICE inline int qi(const int q, const int d, const int Q)
+{
+   return (q<=d) ? q : Q-1-q;
+}
+
+static MFEM_HOST_DEVICE inline int dj(const int q, const int d, const int D)
+{
+   return (q<=d) ? d : D-1-d;
+}
+
+static MFEM_HOST_DEVICE inline int qk(const int q, const int d, const int Q)
+{
+   return (q<=d) ? Q-1-q : q;
+}
+
+static MFEM_HOST_DEVICE inline int dl(const int q, const int d, const int D)
+{
+   return (q<=d) ? D-1-d : d;
+}
+
+static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
+{
+   return (q<=d) ? -1.0 : 1.0;
+}
+
+/// Atomic 3D Transposed Gradient, 1/3
+MFEM_HOST_DEVICE inline void AtomicGradZt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceCube &QQQ0,
+                                          const DeviceCube &QQQ1,
+                                          const DeviceCube &QQQ2,
+                                          const DeviceCube &QQD0,
+                                          const DeviceCube &QQD1,
+                                          const DeviceCube &QQD2)
+{
+   MFEM_FOREACH_THREAD(qz,z,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u = 0.0, v = 0.0, w = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int i = qi(qx,dx,Q1D);
+               const int j = dj(qx,dx,D1D);
+               const int k = qk(qx,dx,Q1D);
+               const int l = dl(qx,dx,D1D);
+               const double s = sign(qx,dx);
+               u += QQQ0(qz,qy,qx) * Gt(l,k) * s;
+               v += QQQ1(qz,qy,qx) * Bt(j,i);
+               w += QQQ2(qz,qy,qx) * Bt(j,i);
+            }
+            QQD0(qz,qy,dx) = u;
+            QQD1(qz,qy,dx) = v;
+            QQD2(qz,qy,dx) = w;
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 3D Transposed Gradient, 2/3
+MFEM_HOST_DEVICE inline void AtomicGradYt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceCube &QQD0,
+                                          const DeviceCube &QQD1,
+                                          const DeviceCube &QQD2,
+                                          const DeviceCube &QDD0,
+                                          const DeviceCube &QDD1,
+                                          const DeviceCube &QDD2)
+{
+   MFEM_FOREACH_THREAD(qz,z,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u = 0.0, v = 0.0, w = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const int i = qi(qy,dy,Q1D);
+               const int j = dj(qy,dy,D1D);
+               const int k = qk(qy,dy,Q1D);
+               const int l = dl(qy,dy,D1D);
+               const double s = sign(qy,dy);
+               u += QQD0(qz,qy,dx) * Bt(j,i);
+               v += QQD1(qz,qy,dx) * Gt(l,k) * s;
+               w += QQD2(qz,qy,dx) * Bt(j,i);
+            }
+            QDD0(qz,dy,dx) = u;
+            QDD1(qz,dy,dx) = v;
+            QDD2(qz,dy,dx) = w;
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 3D Transposed Gradient, 3/3
+MFEM_HOST_DEVICE inline void AtomicGradXt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceCube &QDD0,
+                                          const DeviceCube &QDD1,
+                                          const DeviceCube &QDD2,
+                                          const DeviceTensor<4,const int> &I,
+                                          const DeviceMatrix &Y,
+                                          const int c,
+                                          const int e,
+                                          const bool byVDIM)
+{
+   MFEM_FOREACH_THREAD(dz,z,D1D)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u = 0.0, v = 0.0, w = 0.0;
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const int i = qi(qz,dz,Q1D);
+               const int j = dj(qz,dz,D1D);
+               const int k = qk(qz,dz,Q1D);
+               const int l = dl(qz,dz,D1D);
+               const double s = sign(qz,dz);
+               u += QDD0(qz,dy,dx) * Bt(j,i);
+               v += QDD1(qz,dy,dx) * Bt(j,i);
+               w += QDD2(qz,dy,dx) * Gt(l,k) * s;
+            }
+            const double sum = u + v + w;
+            const int gid = I(dx,dy,dz,e);
+            const int idx = gid >= 0 ? gid : -1-gid;
+            if (byVDIM) { AtomicAdd(Y(c,idx), sum); }
+            else { AtomicAdd(Y(idx,c), sum); }
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline void Atomic3DGrad(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceCube &QQ0,
+                                          const DeviceCube &QQ1,
+                                          const DeviceCube &QQ2,
+                                          const DeviceCube &QD0,
+                                          const DeviceCube &QD1,
+                                          const DeviceCube &QD2,
+                                          const DeviceCube &DD0,
+                                          const DeviceCube &DD1,
+                                          const DeviceCube &DD2,
+                                          const DeviceTensor<4,const int> &I,
+                                          const DeviceMatrix &Y,
+                                          const int c,
+                                          const int e,
+                                          const bool byVDIM)
+{
+   kernels::internal::AtomicGradZt(D1D,Q1D,Bt,Gt,QQ0,QQ1,QQ2,QD0,QD1,QD2);
+   kernels::internal::AtomicGradYt(D1D,Q1D,Bt,Gt,QD0,QD1,QD2,DD0,DD1,DD2);
+   kernels::internal::AtomicGradXt(D1D,Q1D,Bt,Gt,DD0,DD1,DD2,I,Y,c,e,byVDIM);
+}
+
 } // namespace kernels::internal
 
 } // namespace kernels

fem/linearform.cpp

@@ -23,6 +23,7 @@ LinearForm::LinearForm(FiniteElementSpace *f, LinearForm *lf)
    UseDevice(true);
 
    fes = f;
+   ext = nullptr;
    extern_lfs = 1;
 
    // Copy the pointers to the integrators
@@ -69,14 +70,14 @@ void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi,
 
 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi)
 {
-   boundary_integs.Append (lfi);
+   boundary_integs.Append(lfi);
    boundary_integs_marker.Append(NULL); // NULL -> all attributes are active
 }
 
 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi,
                                         Array<int> &bdr_attr_marker)
 {
-   boundary_integs.Append (lfi);
+   boundary_integs.Append(lfi);
    boundary_integs_marker.Append(&bdr_attr_marker);
 }
 
@@ -99,8 +100,35 @@ void LinearForm::AddInteriorFaceIntegrator(LinearFormIntegrator *lfi)
    interior_face_integs.Append(lfi);
 }
 
+void LinearForm::SetAssemblyLevel(LinearAssemblyLevel assembly_level)
+{
+   if (ext)
+   {
+      MFEM_ABORT("the assembly level has already been set!");
+   }
+   assembly = assembly_level;
+   switch (assembly)
+   {
+      case LinearAssemblyLevel::LEGACY:
+         break;
+      case LinearAssemblyLevel::FULL:
+         ext = new FullLinearFormExtension(this);
+         break;
+      default:
+         mfem_error("Unknown assembly level");
+   }
+}
+
 void LinearForm::Assemble()
 {
+   Vector::operator=(0.0);
+
+   // The above operation is executed on device because of UseDevice().
+   // The first use of AddElementVector() below will move it back to host
+   // because both 'vdofs' and 'elemvect' are on host.
+
+   if (ext) { return ext->Assemble(); }
+
    Array<int> vdofs;
    ElementTransformation *eltrans;
    DofTransformation *doftrans;
@@ -108,12 +136,6 @@ void LinearForm::Assemble()
 
    int i;
 
-   Vector::operator=(0.0);
-
-   // The above operation is executed on device because of UseDevice().
-   // The first use of AddElementVector() below will move it back to host
-   // because both 'vdofs' and 'elemvect' are on host.
-
    if (domain_integs.Size())
    {
       for (int k = 0; k < domain_integs.Size(); k++)
@@ -143,7 +165,7 @@ void LinearForm::Assemble()
                {
                   doftrans->TransformDual(elemvect);
                }
-               AddElementVector (vdofs, elemvect);
+               AddElementVector(vdofs, elemvect);
             }
          }
       }
@@ -274,6 +296,18 @@ void LinearForm::Assemble()
    }
 }
 
+void LinearForm::Update()
+{
+   SetSize(fes->GetVSize()); ResetDeltaLocations();
+   if (ext) { ext->Update(); }
+}
+
+void LinearForm::Update(FiniteElementSpace *f)
+{
+   fes = f;
+   Update();
+}
+
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
    MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
@@ -281,6 +315,7 @@ void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
    v.UseDevice(true);
    this->Vector::MakeRef(v, v_offset, fes->GetVSize());
    ResetDeltaLocations();
+   if (ext) { ext->Update(); }
 }
 
 void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
@@ -292,6 +327,8 @@ void LinearForm::AssembleDelta()
 {
    if (domain_delta_integs.Size() == 0) { return; }
 
+   if (ext) { return ext->AssembleDelta(); }
+
    if (!HaveDeltaLocations())
    {
       int sdim = fes->GetMesh()->SpaceDimension();
@@ -356,6 +393,8 @@ LinearForm::~LinearForm()
       for (k=0; k < interior_face_integs.Size(); k++)
       { delete interior_face_integs[k]; }
    }
+
+   delete ext;
 }
 
 }

fem/linearform.hpp

@@ -14,18 +14,39 @@
 
 #include "../config/config.hpp"
 #include "lininteg.hpp"
+#include "linearform_ext.hpp"
 #include "gridfunc.hpp"
 
 namespace mfem
 {
 
+/** @brief Enumeration defining the assembly level for linear form classes
+ *  derived from Vector. */
+enum class LinearAssemblyLevel
+{
+   /// Legacy assembled form, i.e. a global vector in MFEM format.
+   /// This assembly level is ALWAYS performed on the host.
+   LEGACY = 0,
+   /// Fully assembled form, i.e. a global vector in MFEM format.
+   /// This assembly is compatible with device execution.
+   FULL
+};
+
 /// Vector with associated FE space and LinearFormIntegrators.
 class LinearForm : public Vector
 {
+   friend LinearFormExtension;
+
 protected:
    /// FE space on which the LinearForm lives. Not owned.
    FiniteElementSpace *fes;
 
+   /// The assembly level of the form (legacy or full)
+   LinearAssemblyLevel assembly;
+
+   /** @brief Extension for supporting different assembly levels. */
+   LinearFormExtension *ext;
+
    /** @brief Indicates the LinearFormIntegrator%s stored in #domain_integs,
        #domain_delta_integs, #boundary_integs, and #boundary_face_integs are
        owned by another LinearForm. */
@@ -75,7 +96,7 @@ public:
    /// Creates linear form associated with FE space @a *f.
    /** The pointer @a f is not owned by the newly constructed object. */
    LinearForm(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; extern_lfs = 0; UseDevice(true); }
+   { fes = f; ext = nullptr; extern_lfs = 0; UseDevice(true); }
 
    /** @brief Create a LinearForm on the FiniteElementSpace @a f, using the
        same integrators as the LinearForm @a lf.
@@ -90,7 +111,7 @@ public:
    /** The associated FiniteElementSpace can be set later using one of the
        methods: Update(FiniteElementSpace *) or
        Update(FiniteElementSpace *, Vector &, int). */
-   LinearForm() { fes = NULL; extern_lfs = 0; UseDevice(true); }
+   LinearForm() { fes = NULL; ext = nullptr; extern_lfs = 0; UseDevice(true); }
 
    /// Construct a LinearForm using previously allocated array @a data.
    /** The LinearForm does not assume ownership of @a data which is assumed to
@@ -98,7 +119,7 @@ public:
        for externally allocated array, the pointer @a data can be NULL. The data
        array can be replaced later using the method SetData(). */
    LinearForm(FiniteElementSpace *f, double *data) : Vector(data, f->GetVSize())
-   { fes = f; extern_lfs = 0; }
+   { fes = f; ext = nullptr; extern_lfs = 0; }
 
    /// Copy assignment. Only the data of the base class Vector is copied.
    /** It is assumed that this object and @a rhs use FiniteElementSpace%s that
@@ -155,6 +176,9 @@ public:
        coefficients. */
    Array<LinearFormIntegrator*> *GetDLFI() { return &domain_integs; }
 
+   /// Access the element attribute marker (of length mesh->attributes)
+   Array<Array<int>*> *GetDLFIM() { return &domain_integs_marker; }
+
    /** @brief Access all integrators added with AddDomainIntegrator() which are
        DeltaLFIntegrator%s with delta coefficients. */
    Array<DeltaLFIntegrator*> *GetDLFI_Delta() { return &domain_delta_integs; }
@@ -173,6 +197,10 @@ public:
        corresponding pointer (to Array<int>) will be NULL. */
    Array<Array<int>*> *GetFLFI_Marker() { return &boundary_face_integs_marker; }
 
+   /// Set the desired assembly level, default is LinearAssemblyLevel::LEGACY.
+   /** This method must be called before assembly. */
+   void SetAssemblyLevel(LinearAssemblyLevel);
+
    /// Assembles the linear form i.e. sums over all domain/bdr integrators.
    void Assemble();
 
@@ -184,11 +212,10 @@ public:
        updated, e.g. after its associated Mesh object has been refined.
 
        @note This method does not perform assembly. */
-   void Update() { SetSize(fes->GetVSize()); ResetDeltaLocations(); }
+   void Update();
 
    /// Associate a new FE space, @a *f, with this object and Update() it. */
-   void Update(FiniteElementSpace *f)
-   { fes = f; SetSize(f->GetVSize()); ResetDeltaLocations(); }
+   void Update(FiniteElementSpace *f);
 
    /** @brief Associate a new FE space, @a *f, with this object and use the data
        of @a v, offset by @a v_offset, to initialize this object's Vector::data.

fem/linearform_ext.cpp

@@ -0,0 +1,104 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+
+// Implementations of FullLinearFormExtension.
+
+namespace mfem
+{
+
+FullLinearFormExtension::FullLinearFormExtension(LinearForm *lf):
+   LinearFormExtension(lf)
+{
+   Update();
+}
+
+void FullLinearFormExtension::Assemble()
+{
+   MFEM_VERIFY(lf->Size() == lf->FESpace()->GetVSize(), "");
+
+   // Filter out the unsupported integrators
+   MFEM_VERIFY(lf->GetBLFI()->Size() == 0,
+               "Integrators added with AddBoundaryIntegrator() "
+               "are not supported!");
+
+   MFEM_VERIFY(lf->GetDLFI_Delta()->Size() == 0, ""
+               "Integrators added with AddDomainIntegrator() which are "
+               "DeltaLFIntegrators with delta coefficients "
+               "are not supported!");
+
+   MFEM_VERIFY(lf->GetIFLFI()->Size() == 0,
+               "Integrators added with AddInteriorFaceIntegrator() "
+               "are not supported!");
+
+   MFEM_VERIFY(lf->GetFLFI()->Size() == 0,
+               "Integrators added with AddBdrFaceIntegrator() "
+               " are not supported!");
+
+   const FiniteElementSpace &fes = *lf->FESpace();
+   const Array<Array<int>*> &domain_integs_marker = *lf->GetDLFIM();
+   const int mesh_attributes_size = fes.GetMesh()->attributes.Size();
+   const Array<LinearFormIntegrator*> &domain_integs = *lf->GetDLFI();
+
+   for (int k = 0; k < domain_integs.Size(); ++k)
+   {
+      // Get the markers for this integrator
+      const Array<int> *domain_integs_marker_k = domain_integs_marker[k];
+
+      // check if there are markers for this integrator
+      const bool has_markers_k = domain_integs_marker_k != nullptr;
+
+      if (has_markers_k)
+      {
+         // Element attribute marker should be of length mesh->attributes
+         MFEM_VERIFY(mesh_attributes_size == domain_integs_marker_k->Size(),
+                     "invalid element marker for domain linear form "
+                     "integrator #" << k << ", counting from zero");
+      }
+
+      const int NE = fes.GetNE();
+      auto markers_w = markers.Write();
+
+      // if there are no markers, just use the whole linear form (1)
+      if (!has_markers_k)
+      {
+         // done this way as operator= for array are still done on the host
+         MFEM_FORALL(e, NE, markers_w[e] = 1;);
+      }
+      else
+      {
+         // otherwise, scan the attributes to set the markers to 0 or 1
+         const auto attr = attributes.Read();
+         const auto dimk = domain_integs_marker_k->Read();
+         MFEM_FORALL(e, NE, markers_w[e] = dimk[attr[e]-1] == 1;);
+      }
+
+      domain_integs[k]->AssembleFull(fes, markers, *lf);
+   }
+}
+
+void FullLinearFormExtension::Update()
+{
+   MFEM_VERIFY(lf->Size() == lf->FESpace()->GetVSize(), "");
+
+   const int ne = lf->FESpace()->GetNE();
+   const Mesh &mesh = *lf->FESpace()->GetMesh();
+
+   markers.SetSize(ne);
+
+   // Gather the attributes on the host from all the elements
+   attributes.SetSize(ne);
+   for (int i = 0; i < ne; ++i) { attributes[i] = mesh.GetAttribute(i); }
+}
+
+} // namespace mfem

fem/linearform_ext.hpp

@@ -0,0 +1,71 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LINEARFORM_EXT
+#define MFEM_LINEARFORM_EXT
+
+#include "../general/array.hpp"
+
+namespace mfem
+{
+
+class LinearForm;
+class FullLinearFormExtension;
+
+/// Class extending the LinearForm class to support different AssemblyLevels.
+class LinearFormExtension
+{
+protected:
+   /// Linear form from which this extension depends. Not owned.
+   LinearForm *lf;
+
+public:
+   LinearFormExtension(LinearForm *lf): lf(lf) { }
+
+   virtual ~LinearFormExtension() { }
+
+   /// Updates the linear form extension
+   virtual void Update() = 0;
+
+   /// Assemble at the level given for the linear form extension
+   virtual void Assemble() = 0;
+
+   /// Assembles delta functions of the linear form extension
+   virtual void AssembleDelta() = 0;
+};
+
+/// Data and methods for fully-assembled linear forms
+class FullLinearFormExtension : public LinearFormExtension
+{
+private:
+   /// Attributes of all mesh elements.
+   Array<int> attributes;
+
+   /// Temporary markers for device kernels.
+   Array<int> markers;
+
+public:
+   FullLinearFormExtension(LinearForm *lf);
+
+   /// Fully assembles the linear form, compatible with device execution.
+   /// Only integrators added with AddDomainIntegrator are supported.
+   void Assemble() override;
+
+   /// Fully assembles the delta functions of the linear form.
+   /// Not yet supported.
+   void AssembleDelta() override { MFEM_ABORT("Not yet supported!"); }
+
+   void Update() override;
+};
+
+} // namespace mfem
+
+#endif // MFEM_LINEARFORM_EXT

fem/lininteg.cpp

@@ -9,13 +9,21 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-
 #include "fem.hpp"
 #include <cmath>
 
 namespace mfem
 {
 
+void LinearFormIntegrator::AssembleFull(const FiniteElementSpace&,
+                                        const Array<int> &markers,
+                                        Vector &b)
+{
+   MFEM_CONTRACT_VAR(b);
+   MFEM_CONTRACT_VAR(markers);
+   mfem_error("LinearFormIntegrator::AssembleFull(...)");
+}
+
 void LinearFormIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect)
 {
@@ -280,12 +288,11 @@ void VectorDomainLFIntegrator::AssembleRHSElementVect(
       val = Tr.Weight();
 
       el.CalcShape(ip, shape);
-      Q.Eval (Qvec, Tr, ip);
+      Q.Eval(Qvec, Tr, ip);
 
       for (int k = 0; k < vdim; k++)
       {
          cf = val * Qvec(k);
-
          for (int s = 0; s < dof; s++)
          {
             elvect(dof*k+s) += ip.weight * cf * shape(s);
@@ -311,6 +318,54 @@ void VectorDomainLFIntegrator::AssembleDeltaElementVect(
    MultVWt(shape, Qvec, elvec_as_mat);
 }
 
+void VectorDomainLFGradIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
+{
+   const int dim = el.GetDim();
+   const int dof = el.GetDof();
+   const int vdim = Q.GetVDim();
+   const int sdim = Tr.GetSpaceDim();
+
+   dshape.SetSize(dof,sdim);
+
+   elvect.SetSize(dof*(vdim/sdim));
+   elvect = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int intorder = 2 * el.GetOrder();
+      ir = &IntRules.Get(el.GetGeomType(), intorder);
+   }
+
+   Vector pelvect(dof);
+   Vector part_x(dim);
+
+   for (int q = 0; q < ir->GetNPoints(); q++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(q);
+
+      Tr.SetIntPoint(&ip);
+      el.CalcPhysDShape(Tr, dshape);
+
+      Q.Eval(Qvec, Tr, ip);
+      Qvec *= ip.weight * Tr.Weight();
+
+      for (int k = 0; k < vdim/sdim; k++)
+      {
+         for (int d=0; d < sdim; ++d) { part_x(d) = Qvec(k*sdim+d); }
+         dshape.Mult(part_x, pelvect);
+         for (int s = 0; s < dof; ++s) { elvect(s+k*dof) += pelvect(s); }
+      }
+   }
+}
+
+void VectorDomainLFGradIntegrator::AssembleDeltaElementVect(
+   const FiniteElement&, ElementTransformation&, Vector&)
+{
+   MFEM_ABORT("Not implemented!");
+}
+
 void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {

fem/lininteg.hpp

@@ -27,6 +27,11 @@ protected:
    LinearFormIntegrator(const IntegrationRule *ir = NULL) { IntRule = ir; }
 
 public:
+   /// Method defining full assembly
+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b);
+
    /** Given a particular Finite Element and a transformation (Tr)
        computes the element vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -110,6 +115,10 @@ public:
    DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir)
       : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1) { }
 
+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b);
+
    /** Given a particular Finite Element and a transformation (Tr)
        computes the element right hand side element vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -136,6 +145,10 @@ public:
    DomainLFGradIntegrator(VectorCoefficient &QF)
       : DeltaLFIntegrator(QF), Q(QF) { }
 
+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b);
+
    /** Given a particular Finite Element and a transformation (Tr)
        computes the element right hand side element vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -223,6 +236,10 @@ public:
    VectorDomainLFIntegrator(VectorCoefficient &QF)
       : DeltaLFIntegrator(QF), Q(QF) { }
 
+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b);
+
    /** Given a particular Finite Element and a transformation (Tr)
        computes the element right hand side element vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -236,6 +253,37 @@ public:
    using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
+/** Class for domain integrator L(v) := (f, grad v), where
+    f=(f1x,f1y,f1z,...,fnx,fny,fnz) and v=(v1,...,vn). */
+class VectorDomainLFGradIntegrator : public DeltaLFIntegrator
+{
+private:
+   Vector shape, Qvec;
+   VectorCoefficient &Q;
+   DenseMatrix dshape;
+
+public:
+   /// Constructs the domain integrator (Q, grad v)
+   VectorDomainLFGradIntegrator(VectorCoefficient &QF)
+      : DeltaLFIntegrator(QF), Q(QF) { }
+
+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b) override;
+
+   /** Given a particular Finite Element and a transformation (Tr)
+       computes the element right hand side element vector, elvect. */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override;
+
+   virtual void AssembleDeltaElementVect(const FiniteElement &fe,
+                                         ElementTransformation &Trans,
+                                         Vector &elvect) override;
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
+};
+
 /** Class for boundary integration of L(v) := (g, v), where
     f=(f1,...,fn) and v=(v1,...,vn). */
 class VectorBoundaryLFIntegrator : public LinearFormIntegrator

fem/lininteg_domain.cpp

@@ -0,0 +1,104 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "lininteg_domain.hpp"
+
+namespace mfem
+{
+
+using namespace internal::linearform_extension;
+
+void DomainLFIntegrator::AssembleFull(const FiniteElementSpace &fes,
+                                      const Array<int> &markers,
+                                      Vector &y)
+{
+   const int vdim = fes.GetVDim();
+   MFEM_VERIFY(vdim == 1, "vdim should be equal to 1!");
+   GetOrder_f qorder_fct = [&](int el_order) { return oa * el_order + ob; };
+   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, qorder_fct);
+
+   Vector coeff;
+   const int NQ = ir->GetNPoints();
+   const int NE = fes.GetMesh()->GetNE();
+
+   if (ConstantCoefficient *cQ =
+          dynamic_cast<ConstantCoefficient*>(&Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient *cQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qfun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qfun.Size() == vdim*NE*NQ,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different.\n");
+      qfun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
+   }
+   else
+   {
+      coeff.SetSize(NQ * NE);
+      auto C = Reshape(coeff.HostWrite(), NQ, NE);
+      for (int e = 0; e < NE; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < NQ; ++q)
+         {
+            C(q,e) = Q.Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
+
+   const int id = GetKernelId(fes,ir);
+   const int dim = fes.GetMesh()->Dimension();
+
+   LinearFormExtensionKernel_f ker = nullptr;
+   if (dim==2) { ker=VectorDomainLFIntegratorAssemble2D; }
+   if (dim==3) { ker=VectorDomainLFIntegratorAssemble3D; }
+
+   switch (id)
+   {
+      // 2D kernels, q=p+1
+      case 0x222: ker=VectorDomainLFIntegratorAssemble2D<2,2>; break;
+      case 0x233: ker=VectorDomainLFIntegratorAssemble2D<3,3>; break;
+      case 0x244: ker=VectorDomainLFIntegratorAssemble2D<4,4>; break;
+      case 0x255: ker=VectorDomainLFIntegratorAssemble2D<5,5>; break;
+
+      // 2D kernels, q=p+2
+      case 0x223: ker=VectorDomainLFIntegratorAssemble2D<2,3>; break;
+      case 0x234: ker=VectorDomainLFIntegratorAssemble2D<3,4>; break;
+      case 0x245: ker=VectorDomainLFIntegratorAssemble2D<4,5>; break;
+      case 0x256: ker=VectorDomainLFIntegratorAssemble2D<5,6>; break;
+
+      // 3D kernels, q=p+1, BENCH_FULL_DomainLF_3D_GLL
+      case 0x322: ker=VectorDomainLFIntegratorAssemble3D<2,2>; break;
+      case 0x333: ker=VectorDomainLFIntegratorAssemble3D<3,3>; break;
+      case 0x344: ker=VectorDomainLFIntegratorAssemble3D<4,4>; break;
+      case 0x355: ker=VectorDomainLFIntegratorAssemble3D<5,5>; break;
+      case 0x366: ker=VectorDomainLFIntegratorAssemble3D<6,6>; break;
+
+      // 3D kernels, q=p+2, BENCH_FULL_DomainLF_3D_GL
+      case 0x323: ker=VectorDomainLFIntegratorAssemble3D<2,3>; break;
+      case 0x334: ker=VectorDomainLFIntegratorAssemble3D<3,4>; break;
+      case 0x345: ker=VectorDomainLFIntegratorAssemble3D<4,5>; break;
+      case 0x356: ker=VectorDomainLFIntegratorAssemble3D<5,6>; break;
+      case 0x367: ker=VectorDomainLFIntegratorAssemble3D<6,7>; break;
+   }
+   MFEM_VERIFY(ker, "Unexpected kernel error!");
+   Launch(ker,fes,ir,coeff,markers,y);
+}
+
+} // namespace mfem

fem/lininteg_domain.hpp

@@ -0,0 +1,263 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Internal header, included only by .cpp files
+
+#include "fem.hpp"
+
+#include "../fem/kernels.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+namespace linearform_extension
+{
+
+/// Signature of the kernels used for linear form extension
+using LinearFormExtensionKernel_f = void (*)(const int vdim,
+                                             const bool byVDIM,
+                                             const int ND,
+                                             const int NE,
+                                             const int d,
+                                             const int q,
+                                             const int *markers,
+                                             const double *b,
+                                             const double *g,
+                                             const int *idx,
+                                             const double *J,
+                                             const double *detJ,
+                                             const double *weights,
+                                             const Vector &coeff,
+                                             double *output);
+
+/// Signature of the function used to compute the quadrature order
+using GetOrder_f = std::function<int(int)>;
+
+/// Internal helper function to get the integration rule
+inline const IntegrationRule *GetIntRuleFromOrder(const FiniteElementSpace &fes,
+                                                  const IntegrationRule *IntRule,
+                                                  const GetOrder_f &qorder_fct)
+{
+   const FiniteElement &fe = *fes.GetFE(0);
+   const int qorder = qorder_fct(fe.GetOrder());
+   const Geometry::Type geom_type = fe.GetGeomType();
+   return IntRule ? IntRule : &IntRules.Get(geom_type, qorder);
+}
+
+/// Internal helper function to encode the ID of a LinearFormExtension kernel
+/// The ID is equal to: (dim << 8) |
+///                     (1D number of degrees of freedom << 4) |
+///                     (1D number of quadrature points)
+inline int GetKernelId(const FiniteElementSpace &fes,
+                       const IntegrationRule *ir)
+{
+   Mesh *mesh = fes.GetMesh();
+   const int dim = mesh->Dimension();
+   const FiniteElement &el = *fes.GetFE(0);
+   const DofToQuad &maps = el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int d = maps.ndof;
+   const int q = maps.nqpt;
+   return (dim << 8) | (d << 4) | q;
+}
+
+/// Internal helper function to launch the LinearFormExtension kernel
+inline void Launch(const LinearFormExtensionKernel_f &kernel,
+                   const FiniteElementSpace &fes,
+                   const IntegrationRule *ir,
+                   const Vector &coeff,
+                   const Array<int> &markers,
+                   Vector &y)
+{
+   Mesh *mesh = fes.GetMesh();
+   const int vdim = fes.GetVDim();
+   const bool byVDIM = fes.GetOrdering() == Ordering::byVDIM;
+
+   const FiniteElement &el = *fes.GetFE(0);
+   constexpr int flags = GeometricFactors::JACOBIANS |
+                         GeometricFactors::DETERMINANTS;
+   const MemoryType mt = Device::GetDeviceMemoryType();
+   const GeometricFactors *geom = mesh->GetGeometricFactors(*ir, flags, mt);
+   const DofToQuad &maps = el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes.GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   MFEM_ASSERT(ER, "Not supported!");
+
+   const int *M = markers.Read();
+   const double *B = maps.B.Read();
+   const double *G = maps.G.Read();
+   const double *J = geom->J.Read();
+   const double *detJ = geom->detJ.Read();
+   const int *I = ER->GatherMap().Read();
+   const double *W = ir->GetWeights().Read();
+   double *Y = y.ReadWrite();
+
+   const int ND = fes.GetNDofs();
+   const int NE = fes.GetMesh()->GetNE();
+
+   const int d = maps.ndof;
+   const int q = maps.nqpt;
+
+   kernel(vdim, byVDIM, ND, NE, d, q, M, B, G, I, J, detJ, W, coeff, Y);
+}
+
+/// Internal assembly kernel for the 2D (Vector)DomainLFIntegrator
+template<int D=0, int Q=0> static
+void VectorDomainLFIntegratorAssemble2D(const int vdim,
+                                        const bool byVDIM,
+                                        const int ND,
+                                        const int NE,
+                                        const int d,
+                                        const int q,
+                                        const int *markers,
+                                        const double *b,
+                                        const double *, // g
+                                        const int *idx,
+                                        const double *, // jacobians
+                                        const double *detJ,
+                                        const double *weights,
+                                        const Vector &coeff,
+                                        double *y)
+{
+   constexpr bool USE_SMEM = D > 0 && Q > 0;
+
+   const bool cst_coeff = coeff.Size() == vdim;
+
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, NE);
+   const auto B = Reshape(b, q,d);
+   const auto DetJ = Reshape(detJ, q,q, NE);
+   const auto W = Reshape(weights, q,q);
+   const auto I = Reshape(idx, d,d, NE);
+   const auto C = cst_coeff ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,NE);
+
+   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
+
+   const int sm_size = 2*q*(d+q);
+   constexpr int GRID = USE_SMEM ? 0 : 128;
+   double *gmem = ScratchMem<GRID>(sm_size);
+
+   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
+   {
+      if (M(e) == 0) { /* ignore */ return; }
+
+      const int bid = MFEM_BLOCK_ID(x);
+      constexpr int SM_SIZE = 2*Q*(D+Q);
+      constexpr bool USE_SMEM = D > 0 && Q > 0;
+      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
+      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
+      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), q,d);
+      const DeviceMatrix QQ(DeviceMemAlloc(sm,q*q), q,q);
+      const DeviceMatrix QD(DeviceMemAlloc(sm,q*d), q,d);
+
+      kernels::internal::LoadB(d,q,B,Bt);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const double cst_val = C(c,0,0,0);
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               const double detJ = DetJ(x,y,e);
+               const double coeff_val = cst_coeff ? cst_val : C(c,x,y,e);
+               QQ(y,x) = W(x,y) * coeff_val * detJ;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Atomic2DEvalTranspose(d,q,Bt,QQ,QD,I,Y,c,e,byVDIM);
+      }
+   });
+}
+
+/// Internal assembly kernel for the 2D (Vector)DomainLFIntegrator
+template<int D=0, int Q=0> static
+void VectorDomainLFIntegratorAssemble3D(const int vdim,
+                                        const bool byVDIM,
+                                        const int ND,
+                                        const int NE,
+                                        const int d,
+                                        const int q,
+                                        const int *markers,
+                                        const double *b,
+                                        const double *, // g
+                                        const int *idx,
+                                        const double *, // jacobians
+                                        const double *detJ,
+                                        const double *weights,
+                                        const Vector &coeff,
+                                        double *y)
+{
+   constexpr bool USE_SMEM = D > 0 && Q > 0;
+
+   const bool cst_coeff = coeff.Size() == vdim;
+
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, NE);
+   const auto B = Reshape(b, q,d);
+   const auto DetJ = Reshape(detJ, q,q,q, NE);
+   const auto W = Reshape(weights, q,q,q);
+   const auto I = Reshape(idx, d,d,d, NE);
+   const auto C = cst_coeff ? Reshape(F,vdim,1,1,1,1):Reshape(F,vdim,q,q,q,NE);
+
+   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
+
+   const int sm_size = q*d + q*q*q;
+   const int GRID = USE_SMEM ? 0 : 128;
+   double *gmem = ScratchMem<GRID>(sm_size);
+   MFEM_VERIFY(q < 32, "Unsupported quadrature order!");
+
+   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
+   {
+      if (M(e) == 0) { /* ignore */ return; }
+
+      double u[Q>0?Q:32];
+
+      const int bid = MFEM_BLOCK_ID(x);
+      constexpr int SM_SIZE = Q*D + Q*Q*Q;
+      constexpr bool USE_SMEM = D > 0 && Q > 0;
+      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
+      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
+      const DeviceCube QQQ(DeviceMemAlloc(sm,q*q*q), q,q,q);
+      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), q,d);
+      kernels::internal::LoadB(d,q,B,Bt);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const double cst_val = C(c,0,0,0,0);
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  const double detJ = DetJ(x,y,z,e);
+                  const double coeff_val = cst_coeff ? cst_val : C(c,x,y,z,e);
+                  QQQ(z,y,x) = W(x,y,z) * coeff_val * detJ;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Atomic3DEvalTranspose(d,q,u,Bt,QQQ,I,Y,c,e,byVDIM);
+      }
+   });
+}
+
+} // namespace linearform_extension
+
+} // namespace internal
+
+} // namespace mfem

fem/lininteg_domain_grad.cpp

@@ -0,0 +1,107 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "lininteg_domain.hpp"
+#include "lininteg_domain_grad.hpp"
+
+namespace mfem
+{
+
+using namespace internal::linearform_extension;
+
+void DomainLFGradIntegrator::AssembleFull(const FiniteElementSpace &fes,
+                                          const Array<int> &markers,
+                                          Vector &y)
+{
+   MFEM_VERIFY(fes.GetVDim()==1, "vdim != 1");
+   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
+   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
+
+   Vector coeff;
+   const int NQ = ir->GetNPoints();
+   const int NE = fes.GetMesh()->GetNE();
+
+   if (VectorConstantCoefficient *vcQ =
+          dynamic_cast<VectorConstantCoefficient*>(&Q))
+   {
+      coeff = vcQ->GetVec();
+   }
+   else if (VectorQuadratureFunctionCoefficient *vqfQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qfun = vqfQ->GetQuadFunction();
+      MFEM_VERIFY(qfun.Size() == NE*NQ,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different.\n");
+      qfun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
+   }
+   else
+   {
+      const int qvdim = Q.GetVDim();
+      Vector Qvec(qvdim);
+      coeff.SetSize(qvdim * NQ * NE);
+      auto C = Reshape(coeff.HostWrite(), qvdim, NQ, NE);
+      for (int e = 0; e < NE; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < NQ; ++q)
+         {
+            Q.Eval(Qvec, T, ir->IntPoint(q));
+            for (int c=0; c<qvdim; ++c)
+            {
+               C(c,q,e) = Qvec[c];
+            }
+         }
+      }
+   }
+
+   const int id = GetKernelId(fes,ir);
+   const int dim = fes.GetMesh()->Dimension();
+
+   LinearFormExtensionKernel_f ker = nullptr;
+   if (dim==2) { ker=VectorDomainLFGradIntegratorAssemble2D; }
+   if (dim==3) { ker=VectorDomainLFGradIntegratorAssemble3D; }
+
+   switch (id)
+   {
+      // 2D kernels, q=p+1
+      case 0x222: ker=VectorDomainLFGradIntegratorAssemble2D<2,2>; break;
+      case 0x233: ker=VectorDomainLFGradIntegratorAssemble2D<3,3>; break;
+      case 0x244: ker=VectorDomainLFGradIntegratorAssemble2D<4,4>; break;
+      case 0x255: ker=VectorDomainLFGradIntegratorAssemble2D<5,5>; break;
+
+      // 2D kernels, q=p+2
+      case 0x223: ker=VectorDomainLFGradIntegratorAssemble2D<2,3>; break;
+      case 0x234: ker=VectorDomainLFGradIntegratorAssemble2D<3,4>; break;
+      case 0x245: ker=VectorDomainLFGradIntegratorAssemble2D<4,5>; break;
+      case 0x256: ker=VectorDomainLFGradIntegratorAssemble2D<5,6>; break;
+
+      // 3D kernels, q=p+1
+      case 0x322: ker=VectorDomainLFGradIntegratorAssemble3D<2,2>; break;
+      case 0x333: ker=VectorDomainLFGradIntegratorAssemble3D<3,3>; break;
+      case 0x344: ker=VectorDomainLFGradIntegratorAssemble3D<4,4>; break;
+      case 0x355: ker=VectorDomainLFGradIntegratorAssemble3D<5,5>; break;
+
+      // 3D kernels, q=p+2
+      case 0x323: ker=VectorDomainLFGradIntegratorAssemble3D<2,3>; break;
+      case 0x334: ker=VectorDomainLFGradIntegratorAssemble3D<3,4>; break;
+      case 0x345: ker=VectorDomainLFGradIntegratorAssemble3D<4,5>; break;
+      case 0x356: ker=VectorDomainLFGradIntegratorAssemble3D<5,6>; break;
+   }
+   MFEM_VERIFY(ker, "Unexpected kernel error!");
+   Launch(ker,fes,ir,coeff,markers,y);
+}
+
+} // namespace mfem

fem/lininteg_domain_grad.hpp

@@ -0,0 +1,239 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Internal header, included only by .cpp files
+
+#include "fem.hpp"
+
+#include "../general/forall.hpp"
+#include "../fem/kernels.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+namespace linearform_extension
+{
+
+////////////////////////////////////////////////////////////////////////////////
+template<int D=0, int Q=0> static
+void VectorDomainLFGradIntegratorAssemble2D(const int vdim,
+                                            const bool byVDIM,
+                                            const int ND,
+                                            const int NE,
+                                            const int d,
+                                            const int q,
+                                            const int *markers,
+                                            const double *b,
+                                            const double *g,
+                                            const int *idx,
+                                            const double *jacobians,
+                                            const double *detJ,
+                                            const double *weights,
+                                            const Vector &coeff,
+                                            double *y)
+{
+   constexpr int DIM = 2;
+   constexpr bool USE_SMEM = D > 0 && Q > 0;
+
+   const bool cst_coeff = coeff.Size() == vdim*DIM;
+
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, NE);
+   const auto B = Reshape(b, q,d);
+   const auto G = Reshape(g, q,d);
+   const auto J = Reshape(jacobians, q,q, DIM,DIM, NE);
+   const auto DetJ = Reshape(detJ, q,q, NE);
+   const auto W = Reshape(weights, q,q);
+   const auto I = Reshape(idx, d,d, NE);
+   const auto C = cst_coeff ?
+                  Reshape(F,DIM,vdim,1,1,1):
+                  Reshape(F,DIM,vdim,q,q,NE);
+
+   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
+
+   const int sm_size = 2*d*q + 4*q*q;
+   const int GRID = USE_SMEM ? 0 : 128;
+   double *gmem = ScratchMem<GRID>(sm_size);
+
+   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
+   {
+      if (M(e) == 0) { /* ignore */ return; }
+
+      const int bid = MFEM_BLOCK_ID(x);
+      constexpr int SM_SIZE = 2*D*Q + 4*Q*Q;
+      constexpr bool USE_SMEM = D > 0 && Q > 0;
+      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
+      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
+
+      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), d,q);
+      const DeviceMatrix Gt(DeviceMemAlloc(sm,q*d), d,q);
+      kernels::internal::LoadBGt(d,q,B,G,Bt,Gt);
+
+      const DeviceMatrix QQ0(DeviceMemAlloc(sm,q*q), q,q);
+      const DeviceMatrix QQ1(DeviceMemAlloc(sm,q*q), q,q);
+
+      const DeviceMatrix DQ0(DeviceMemAlloc(sm,d*q), d,q);
+      const DeviceMatrix DQ1(DeviceMemAlloc(sm,d*q), d,q);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const double cst_val0 = C(0,c,0,0,0);
+         const double cst_val1 = C(1,c,0,0,0);
+
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               double Jloc[4], Jinv[4];
+               Jloc[0] = J(x,y,0,0,e);
+               Jloc[1] = J(x,y,1,0,e);
+               Jloc[2] = J(x,y,0,1,e);
+               Jloc[3] = J(x,y,1,1,e);
+               const double detJ = DetJ(x,y,e);
+               kernels::CalcInverse<2>(Jloc, Jinv);
+               const double weight = W(x,y);
+               const double u = cst_coeff ? cst_val0 : C(0,c,x,y,e);
+               const double v = cst_coeff ? cst_val1 : C(1,c,x,y,e);
+               QQ0(y,x) = Jinv[0]*u + Jinv[2]*v;
+               QQ1(y,x) = Jinv[1]*u + Jinv[3]*v;
+               QQ0(y,x) *= weight * detJ;
+               QQ1(y,x) *= weight * detJ;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Atomic2DGradTranspose(d,q,Bt,Gt,
+                                                  QQ0,QQ1,DQ0,DQ1,
+                                                  I,Y,c,e,byVDIM);
+      }
+   });
+}
+
+template<int D=0, int Q=0> static
+void VectorDomainLFGradIntegratorAssemble3D(const int vdim,
+                                            const bool byVDIM,
+                                            const int ND,
+                                            const int NE,
+                                            const int d,
+                                            const int q,
+                                            const int *markers,
+                                            const double *b,
+                                            const double *g,
+                                            const int *idx,
+                                            const double *jacobians,
+                                            const double *detJ,
+                                            const double *weights,
+                                            const Vector &coeff,
+                                            double *y)
+{
+   constexpr int DIM = 3;
+   constexpr bool USE_SMEM = D > 0 && Q > 0;
+
+   const bool cst_coeff = coeff.Size() == vdim*DIM;
+
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, NE);
+   const auto B = Reshape(b, q,d);
+   const auto G = Reshape(g, q,d);
+   const auto J = Reshape(jacobians, q,q,q, DIM,DIM, NE);
+   const auto DetJ = Reshape(detJ, q,q,q, NE);
+   const auto W = Reshape(weights, q,q,q);
+   const auto I = Reshape(idx, d,d,d, NE);
+   const auto C = cst_coeff ?
+                  Reshape(F,DIM,vdim,1,1,1,1):
+                  Reshape(F,DIM,vdim,q,q,q,NE);
+
+   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
+
+   const int sm_size = 2*q*d + 6*q*q*q;
+
+   const int GRID = USE_SMEM ? 0 : 128;
+   double *gmem = ScratchMem<GRID>(sm_size);
+
+   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
+   {
+      if (M(e) == 0) { /* ignore */ return; }
+
+      const int bid = MFEM_BLOCK_ID(x);
+      constexpr int SM_SIZE = 2*Q*D + 6*Q*Q*Q;
+      constexpr bool USE_SMEM = D > 0 && Q > 0;
+      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
+      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
+
+      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), d,q);
+      const DeviceMatrix Gt(DeviceMemAlloc(sm,q*d), d,q);
+      kernels::internal::LoadBGt(d,q,B,G,Bt,Gt);
+
+      const DeviceCube QQ0(DeviceMemAlloc(sm,q*q*q), q,q,q);
+      const DeviceCube QQ1(DeviceMemAlloc(sm,q*q*q), q,q,q);
+      const DeviceCube QQ2(DeviceMemAlloc(sm,q*q*q), q,q,q);
+
+      const DeviceCube QD0(DeviceMemAlloc(sm,q*q*q), q,q,d);
+      const DeviceCube QD1(DeviceMemAlloc(sm,q*q*q), q,q,d);
+      const DeviceCube QD2(DeviceMemAlloc(sm,q*q*q), q,q,d);
+
+      const DeviceCube DD0(QQ0,q,d,d);
+      const DeviceCube DD1(QQ1,q,d,d);
+      const DeviceCube DD2(QQ2,q,d,d);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const double cst_val_0 = C(0,c,0,0,0,0);
+         const double cst_val_1 = C(1,c,0,0,0,0);
+         const double cst_val_2 = C(2,c,0,0,0,0);
+
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  double Jloc[9], Jinv[9];
+                  for (int j = 0; j < 3; j++)
+                  {
+                     for (int i = 0; i < 3; i++)
+                     {
+                        Jloc[i+3*j] = J(x,y,z,i,j,e);
+                     }
+                  }
+                  const double detJ = DetJ(x,y,z,e);
+                  kernels::CalcInverse<3>(Jloc, Jinv);
+                  const double weight = W(x,y,z);
+                  const double u = cst_coeff ? cst_val_0 : C(0,c,x,y,z,e);
+                  const double v = cst_coeff ? cst_val_1 : C(1,c,x,y,z,e);
+                  const double w = cst_coeff ? cst_val_2 : C(2,c,x,y,z,e);
+                  QQ0(z,y,x) = Jinv[0]*u + Jinv[3]*v + Jinv[6]*w;
+                  QQ1(z,y,x) = Jinv[1]*u + Jinv[4]*v + Jinv[7]*w;
+                  QQ2(z,y,x) = Jinv[2]*u + Jinv[5]*v + Jinv[8]*w;
+                  QQ0(z,y,x) *= weight * detJ;
+                  QQ1(z,y,x) *= weight * detJ;
+                  QQ2(z,y,x) *= weight * detJ;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Atomic3DGrad(d,q,Bt,Gt,
+                                         QQ0,QQ1,QQ2,
+                                         QD0,QD1,QD2,
+                                         DD0,DD1,DD2,
+                                         I,Y,c,e,byVDIM);
+      }
+   });
+}
+
+} // namespace linearform_extension
+
+} // namespace internal
+
+} // namespace mfem

fem/lininteg_vector_domain.cpp

@@ -0,0 +1,102 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "lininteg_domain.hpp"
+
+namespace mfem
+{
+
+using namespace internal::linearform_extension;
+
+void VectorDomainLFIntegrator::AssembleFull(const FiniteElementSpace &fes,
+                                            const Array<int> &markers,
+                                            Vector &y)
+{
+   const int vdim = fes.GetVDim();
+   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
+   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
+
+   Vector coeff;
+   const int NQ = ir->GetNPoints();
+   const int NE = fes.GetMesh()->GetNE();
+
+   if (VectorConstantCoefficient *vcQ =
+          dynamic_cast<VectorConstantCoefficient*>(&Q))
+   {
+      coeff = vcQ->GetVec();
+   }
+   else if (VectorQuadratureFunctionCoefficient *vQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qfun = vQ->GetQuadFunction();
+      MFEM_VERIFY(qfun.Size() == vdim*NE*NQ,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different.\n");
+      qfun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
+   }
+   else
+   {
+      Vector Qvec(vdim);
+      coeff.SetSize(vdim * NQ * NE);
+      auto C = Reshape(coeff.HostWrite(), vdim, NQ, NE);
+      for (int e = 0; e < NE; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < NQ; ++q)
+         {
+            Q.Eval(Qvec, T, ir->IntPoint(q));
+            for (int c=0; c<vdim; ++c) { C(c,q,e) = Qvec[c]; }
+         }
+      }
+   }
+
+   LinearFormExtensionKernel_f ker = nullptr;
+   const int id = GetKernelId(fes,ir);
+   const int dim = fes.GetMesh()->Dimension();
+
+   if (dim==2) { ker = VectorDomainLFIntegratorAssemble2D<>; }
+   if (dim==3) { ker = VectorDomainLFIntegratorAssemble3D<>; }
+
+   switch (id)
+   {
+      // 2D kernels, q=p+1
+      case 0x222: ker=VectorDomainLFIntegratorAssemble2D<2,2>; break;
+      case 0x233: ker=VectorDomainLFIntegratorAssemble2D<3,3>; break;
+      case 0x244: ker=VectorDomainLFIntegratorAssemble2D<4,4>; break;
+      case 0x255: ker=VectorDomainLFIntegratorAssemble2D<5,5>; break;
+
+      // 2D kernels, q=p+2
+      case 0x223: ker=VectorDomainLFIntegratorAssemble2D<2,3>; break;
+      case 0x234: ker=VectorDomainLFIntegratorAssemble2D<3,4>; break;
+      case 0x245: ker=VectorDomainLFIntegratorAssemble2D<4,5>; break;
+      case 0x256: ker=VectorDomainLFIntegratorAssemble2D<5,6>; break;
+
+      // 3D kernels, q=p+1
+      case 0x322: ker=VectorDomainLFIntegratorAssemble3D<2,2>; break;
+      case 0x333: ker=VectorDomainLFIntegratorAssemble3D<3,3>; break;
+      case 0x344: ker=VectorDomainLFIntegratorAssemble3D<4,4>; break;
+      case 0x355: ker=VectorDomainLFIntegratorAssemble3D<5,5>; break;
+
+      // 3D kernels, q=p+2
+      case 0x323: ker=VectorDomainLFIntegratorAssemble3D<2,3>; break;
+      case 0x334: ker=VectorDomainLFIntegratorAssemble3D<3,4>; break;
+      case 0x345: ker=VectorDomainLFIntegratorAssemble3D<4,5>; break;
+      case 0x356: ker=VectorDomainLFIntegratorAssemble3D<5,6>; break;
+   }
+   MFEM_VERIFY(ker, "Unexpected kernel error!");
+   Launch(ker,fes,ir,coeff,markers,y);
+}
+
+} // namespace mfem

fem/lininteg_vector_domain_grad.cpp

@@ -0,0 +1,116 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "lininteg_domain.hpp"
+#include "lininteg_domain_grad.hpp"
+
+namespace mfem
+{
+
+using namespace internal::linearform_extension;
+
+void VectorDomainLFGradIntegrator::AssembleFull(const FiniteElementSpace &fes,
+                                                const Array<int> &markers,
+                                                Vector &y)
+{
+   const int vdim = fes.GetVDim();
+   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
+   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
+
+   Vector coeff;
+   const int NQ = ir->GetNPoints();
+   const int NE = fes.GetMesh()->GetNE();
+   const int NS = fes.GetMesh()->SpaceDimension();
+
+   if (VectorConstantCoefficient *vcQ =
+          dynamic_cast<VectorConstantCoefficient*>(&Q))
+   {
+      coeff = vcQ->GetVec();
+   }
+   else if (QuadratureFunctionCoefficient *qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qfun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qfun.Size() == NE*NQ,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different.\n");
+      qfun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
+   }
+   else if (VectorQuadratureFunctionCoefficient* vqfQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == vdim * NS * NQ * NE,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      Vector Qvec(vdim);
+      coeff.SetSize(vdim * NQ * NE);
+      auto C = Reshape(coeff.HostWrite(), vdim, NQ, NE);
+      for (int e = 0; e < NE; ++e)
+      {
+         ElementTransformation &Tr = *fes.GetElementTransformation(e);
+         for (int q = 0; q < NQ; ++q)
+         {
+            Q.Eval(Qvec, Tr, ir->IntPoint(q));
+            for (int c = 0; c<vdim; ++c) { C(c,q,e) = Qvec[c]; }
+         }
+      }
+   }
+
+   LinearFormExtensionKernel_f ker = nullptr;
+   const int id = GetKernelId(fes,ir);
+   const int dim = fes.GetMesh()->Dimension();
+
+   if (dim==2) { ker=VectorDomainLFGradIntegratorAssemble2D; }
+   if (dim==3) { ker=VectorDomainLFGradIntegratorAssemble3D; }
+
+   switch (id)
+   {
+      // 2D kernels, q=p+1
+      case 0x222: ker=VectorDomainLFGradIntegratorAssemble2D<2,2>; break;
+      case 0x233: ker=VectorDomainLFGradIntegratorAssemble2D<3,3>; break;
+      case 0x244: ker=VectorDomainLFGradIntegratorAssemble2D<4,4>; break;
+      case 0x255: ker=VectorDomainLFGradIntegratorAssemble2D<5,5>; break;
+
+      // 2D kernels, q=p+2
+      case 0x223: ker=VectorDomainLFGradIntegratorAssemble2D<2,3>; break;
+      case 0x234: ker=VectorDomainLFGradIntegratorAssemble2D<3,4>; break;
+      case 0x245: ker=VectorDomainLFGradIntegratorAssemble2D<4,5>; break;
+      case 0x256: ker=VectorDomainLFGradIntegratorAssemble2D<5,6>; break;
+
+      // 3D kernels, q=p+1
+      case 0x322: ker=VectorDomainLFGradIntegratorAssemble3D<2,2>; break;
+      case 0x333: ker=VectorDomainLFGradIntegratorAssemble3D<3,3>; break;
+      case 0x344: ker=VectorDomainLFGradIntegratorAssemble3D<4,4>; break;
+      case 0x355: ker=VectorDomainLFGradIntegratorAssemble3D<5,5>; break;
+
+      // 3D kernels, q=p+2
+      case 0x323: ker=VectorDomainLFGradIntegratorAssemble3D<2,3>; break;
+      case 0x334: ker=VectorDomainLFGradIntegratorAssemble3D<3,4>; break;
+      case 0x345: ker=VectorDomainLFGradIntegratorAssemble3D<4,5>; break;
+      case 0x356: ker=VectorDomainLFGradIntegratorAssemble3D<5,6>; break;
+   }
+   MFEM_VERIFY(ker, "Unexpected kernel error!");
+   Launch(ker,fes,ir,coeff,markers,y);
+}
+
+} // namespace mfem

fem/lor.cpp

@@ -10,8 +10,21 @@
 // CONTRIBUTING.md for details.
 
 #include "lor.hpp"
+#include "lor_assembly.hpp"
+#include "restriction.hpp"
 #include "pbilinearform.hpp"
 
+#include "../mfem-performance.hpp"
+
+#include "../general/forall.hpp"
+
+#define MFEM_DEBUG_COLOR 220
+#include "../general/debug.hpp"
+
+#define MFEM_NVTX_COLOR Turquoise
+#include "../general/nvtx.hpp"
+#include <limits.h>
+
 namespace mfem
 {
 
@@ -21,12 +34,15 @@ void LORBase::AddIntegrators(BilinearForm &a_from,
                              AddIntegratorFn add_integrator,
                              const IntegrationRule *ir)
 {
+   MFEM_NVTX;
    Array<BilinearFormIntegrator*> *integrators = (a_from.*get_integrators)();
    for (int i=0; i<integrators->Size(); ++i)
    {
-      (a_to.*add_integrator)((*integrators)[i]);
-      ir_map[(*integrators)[i]] = ((*integrators)[i])->GetIntegrationRule();
-      if (ir) { ((*integrators)[i])->SetIntegrationRule(*ir); }
+      BilinearFormIntegrator *integrator = (*integrators)[i];
+      if (!integrator->SupportsBatchedLOR()) { supports_batched_assembly = false; }
+      (a_to.*add_integrator)(integrator);
+      ir_map[integrator] = integrator->GetIntegrationRule();
+      if (ir) { integrator->SetIntegrationRule(*ir); }
    }
 }
 
@@ -38,21 +54,24 @@ void LORBase::AddIntegratorsAndMarkers(BilinearForm &a_from,
                                        AddIntegratorFn add_integrator,
                                        const IntegrationRule *ir)
 {
+   MFEM_NVTX;
    Array<BilinearFormIntegrator*> *integrators = (a_from.*get_integrators)();
    Array<Array<int>*> *markers = (a_from.*get_markers)();
 
    for (int i=0; i<integrators->Size(); ++i)
    {
+      BilinearFormIntegrator *integrator = (*integrators)[i];
       if (*markers[i])
       {
-         (a_to.*add_integrator_marker)((*integrators)[i], *(*markers[i]));
+         (a_to.*add_integrator_marker)(integrator, *(*markers[i]));
       }
       else
       {
-         (a_to.*add_integrator)((*integrators)[i]);
+         (a_to.*add_integrator)(integrator);
       }
-      ir_map[(*integrators)[i]] = ((*integrators)[i])->GetIntegrationRule();
-      if (ir) { ((*integrators)[i])->SetIntegrationRule(*ir); }
+      if (!integrator->SupportsBatchedLOR()) { supports_batched_assembly = false; }
+      ir_map[integrator] = integrator->GetIntegrationRule();
+      if (ir) { integrator->SetIntegrationRule(*ir); }
    }
 }
 
@@ -84,6 +103,7 @@ int LORBase::GetLOROrder() const
 
 void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
 {
+   MFEM_NVTX;
    FESpaceType type = GetFESpaceType();
    MFEM_VERIFY(type != H1 && type != L2, "");
 
@@ -203,6 +223,7 @@ void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
 
 void LORBase::ConstructDofPermutation() const
 {
+   MFEM_NVTX;
    FESpaceType type = GetFESpaceType();
    if (type == H1 || type == L2)
    {
@@ -261,8 +282,24 @@ const OperatorHandle &LORBase::GetAssembledSystem() const
    return A;
 }
 
+const LORRestriction *LORBase::GetLORRestriction() const
+{
+   if (R_lor == NULL)
+   {
+      R_lor = new LORRestriction(fes_ho);
+   }
+   return R_lor;
+}
+
 void LORBase::AssembleSystem_(BilinearForm &a_ho, const Array<int> &ess_dofs)
 {
+   dbg();
+   MFEM_NVTX;
+   // By default, we want to use "batched assembly", however this is only
+   // supported for certain integrators. We set it to true here, and then when
+   // we loop through the integrators, if we encounter unsupported integrators,
+   // we set it to false.
+   supports_batched_assembly = true;
    a->UseExternalIntegrators();
    AddIntegrators(a_ho, *a, &BilinearForm::GetDBFI,
                   &BilinearForm::AddDomainIntegrator, ir_el);
@@ -276,8 +313,35 @@ void LORBase::AssembleSystem_(BilinearForm &a_ho, const Array<int> &ess_dofs)
                             &BilinearForm::GetBFBFI_Marker,
                             &BilinearForm::AddBdrFaceIntegrator,
                             &BilinearForm::AddBdrFaceIntegrator, ir_face);
-   a->Assemble();
-   a->FormSystemMatrix(ess_dofs, A);
+
+   if (supports_batched_assembly)
+   {
+      dbg("supports_batched_assembly");
+      fes_ho.GetMesh()->EnsureNodes();
+#ifdef MFEM_USE_MPI
+      ParFiniteElementSpace *pfes_ho =
+         dynamic_cast<ParFiniteElementSpace*>(&fes_ho);
+      if (pfes_ho)
+      {
+         dbg("=> PARALLEL AssembleBatchedLOR");
+         ParAssembleBatchedLOR(*this, *a, fes_ho, ess_dofs, A);
+      }
+      else
+      {
+         dbg("=> SEQUENTIAL AssembleBatchedLOR");
+         AssembleBatchedLOR(*this, *a, fes_ho, ess_dofs, A);
+      }
+#else
+      AssembleBatchedLOR(*this, *a, fes_ho, ess_dofs, A);
+#endif
+   }
+   else
+   {
+      dbg("NOT supports_batched_assembly");
+      a->Assemble();
+      a->FormSystemMatrix(ess_dofs, A);
+   }
+
    ResetIntegrationRules(&BilinearForm::GetDBFI);
    ResetIntegrationRules(&BilinearForm::GetFBFI);
    ResetIntegrationRules(&BilinearForm::GetBBFI);
@@ -286,6 +350,7 @@ void LORBase::AssembleSystem_(BilinearForm &a_ho, const Array<int> &ess_dofs)
 
 void LORBase::SetupProlongationAndRestriction()
 {
+   MFEM_NVTX;
    if (!HasSameDofNumbering())
    {
       Array<int> p;
@@ -348,6 +413,7 @@ void CheckBasisType(const FiniteElementSpace &fes)
 LORBase::LORBase(FiniteElementSpace &fes_ho_)
    : irs(0, Quadrature1D::GaussLobatto), fes_ho(fes_ho_)
 {
+   MFEM_NVTX;
    Mesh &mesh_ = *fes_ho_.GetMesh();
    int dim = mesh_.Dimension();
    Array<Geometry::Type> geoms;
@@ -363,6 +429,8 @@ LORBase::LORBase(FiniteElementSpace &fes_ho_)
       ir_face = NULL;
    }
    a = NULL;
+   R_lor = NULL;
+   supports_batched_assembly = true;
 }
 
 LORBase::~LORBase()
@@ -378,14 +446,17 @@ LORDiscretization::LORDiscretization(BilinearForm &a_ho_,
                                      int ref_type)
    : LORDiscretization(*a_ho_.FESpace(), ref_type)
 {
+   MFEM_NVTX;
    AssembleSystem(a_ho_, ess_tdof_list);
 }
 
 LORDiscretization::LORDiscretization(FiniteElementSpace &fes_ho,
                                      int ref_type) : LORBase(fes_ho)
 {
+   MFEM_NVTX;
    CheckBasisType(fes_ho);
 
+#if 0
    Mesh &mesh_ho = *fes_ho.GetMesh();
    // For H1, ND and RT spaces, use refinement = element order, for DG spaces,
    // use refinement = element order + 1 (since LOR is p = 0 in this case).
@@ -400,21 +471,29 @@ LORDiscretization::LORDiscretization(FiniteElementSpace &fes_ho,
    fec = fes_ho.FEColl()->Clone(GetLOROrder());
    fes = new FiniteElementSpace(mesh, fec);
    SetupProlongationAndRestriction();
-
+#else
+   mesh = NULL;
+   fec = NULL;
+   fes = NULL;
+#endif
    A.SetType(Operator::MFEM_SPARSEMAT);
 }
 
 void LORDiscretization::AssembleSystem(BilinearForm &a_ho,
                                        const Array<int> &ess_dofs)
 {
+   dbg();
+   MFEM_NVTX;
    delete a;
-   a = new BilinearForm(&GetFESpace());
+   // a = new BilinearForm(&GetFESpace());
+   a = new BilinearForm(&fes_ho);
    AssembleSystem_(a_ho, ess_dofs);
 }
 
 SparseMatrix &LORDiscretization::GetAssembledMatrix() const
 {
-   MFEM_VERIFY(a != NULL && A.Ptr() != NULL, "No LOR system assembled");
+   dbg();
+   MFEM_VERIFY(a != nullptr && A.Ptr() != nullptr, "No LOR system assembled");
    return *A.As<SparseMatrix>();
 }
 
@@ -425,20 +504,26 @@ ParLORDiscretization::ParLORDiscretization(ParBilinearForm &a_ho_,
                                            int ref_type)
    : ParLORDiscretization(*a_ho_.ParFESpace(), ref_type)
 {
+   dbg();
+   MFEM_NVTX;
    AssembleSystem(a_ho_, ess_tdof_list);
 }
 
 ParLORDiscretization::ParLORDiscretization(ParFiniteElementSpace &fes_ho,
                                            int ref_type) : LORBase(fes_ho)
 {
+   dbg();
+   MFEM_NVTX;
    if (fes_ho.GetMyRank() == 0) { CheckBasisType(fes_ho); }
    // TODO: support variable-order spaces in parallel
    MFEM_VERIFY(!fes_ho.IsVariableOrder(),
                "Cannot construct LOR operators on variable-order spaces");
 
+#if 0
    int order = fes_ho.GetMaxElementOrder();
    if (GetFESpaceType() == L2) { ++order; }
 
+   NVTX("ParMesh");
    ParMesh &mesh_ho = *fes_ho.GetParMesh();
    ParMesh *pmesh = new ParMesh(ParMesh::MakeRefined(mesh_ho, order, ref_type));
    mesh = pmesh;
@@ -447,6 +532,11 @@ ParLORDiscretization::ParLORDiscretization(ParFiniteElementSpace &fes_ho,
    ParFiniteElementSpace *pfes = new ParFiniteElementSpace(pmesh, fec);
    fes = pfes;
    SetupProlongationAndRestriction();
+#else
+   mesh = NULL;
+   fec = NULL;
+   fes = NULL;
+#endif
 
    A.SetType(Operator::Hypre_ParCSR);
 }
@@ -454,14 +544,19 @@ ParLORDiscretization::ParLORDiscretization(ParFiniteElementSpace &fes_ho,
 void ParLORDiscretization::AssembleSystem(ParBilinearForm &a_ho,
                                           const Array<int> &ess_dofs)
 {
+   dbg();
+   MFEM_NVTX;
    delete a;
-   a = new ParBilinearForm(&GetParFESpace());
+   // a = new ParBilinearForm(&GetParFESpace());
+   a = new ParBilinearForm(&dynamic_cast<ParFiniteElementSpace&>(fes_ho));
    AssembleSystem_(a_ho, ess_dofs);
 }
 
 HypreParMatrix &ParLORDiscretization::GetAssembledMatrix() const
 {
-   MFEM_VERIFY(a != NULL && A.Ptr() != NULL, "No LOR system assembled");
+   dbg();
+   MFEM_NVTX;
+   MFEM_VERIFY(a != nullptr && A.Ptr() != nullptr, "No LOR system assembled");
    return *A.As<HypreParMatrix>();
 }
 
@@ -470,6 +565,368 @@ ParFiniteElementSpace &ParLORDiscretization::GetParFESpace() const
    return static_cast<ParFiniteElementSpace&>(*fes);
 }
 
-#endif
+#endif // MFEM_USE_MPI
+
+int LORRestriction::GetNRefinedElements(const FiniteElementSpace &fes)
+{
+   int ref = fes.GetMaxElementOrder();
+   int dim = fes.GetMesh()->Dimension();
+   return pow(ref, dim);
+}
+
+FiniteElementCollection *LORRestriction::GetLowOrderFEC(
+   const FiniteElementSpace &fes)
+{
+   return fes.FEColl()->Clone(1);
+}
+
+LORRestriction::LORRestriction(const FiniteElementSpace &fes_ho)
+   : fes_ho(fes_ho),
+     fec_lo(GetLowOrderFEC(fes_ho)),
+     geom(fes_ho.GetMesh()->GetElementGeometry(0)),
+     ne_ref(GetNRefinedElements(fes_ho)),
+     ne(fes_ho.GetNE()*ne_ref),
+     vdim(fes_ho.GetVDim()),
+     byvdim(fes_ho.GetOrdering() == Ordering::byVDIM),
+     ndofs(fes_ho.GetNDofs()),
+     dof(fec_lo->GetFE(geom, 1)->GetDof()),
+
+     offsets(ndofs+1),
+     indices(ne*dof),
+     gatherMap(ne*dof),
+
+     dof_glob2loc(),
+     dof_glob2loc_offsets(),
+     el_dof_lex()
+{
+   MFEM_NVTX;
+   SetupLocalToElement();
+   SetupGlobalToLocal();
+
+   NVTX("EnsureNodes");
+}
+
+void LORRestriction::SetupLocalToElement()
+{
+   MFEM_NVTX;
+   MFEM_VERIFY(ne>0, "ne==0 not supported");
+   const FiniteElement *fe = fec_lo->GetFE(geom, 1);
+   const TensorBasisElement* el =
+      dynamic_cast<const TensorBasisElement*>(fe);
+   MFEM_VERIFY(el, "!TensorBasisElement");
+
+   const Array<int> &fe_dof_map = el->GetDofMap();
+   MFEM_VERIFY(fe_dof_map.Size() > 0, "invalid dof map");
+
+   const FiniteElement *fe_ho = fes_ho.GetFE(0);
+   const TensorBasisElement* tel_ho =
+      dynamic_cast<const TensorBasisElement*>(fe_ho);
+   MFEM_VERIFY(tel_ho, "!TensorBasisElement");
+   const Array<int> &fe_dof_map_ho = tel_ho->GetDofMap();
+
+   int order = fes_ho.GetMaxElementOrder();
+   RefinedGeometry &RG = *GlobGeometryRefiner.Refine(geom, order);
+   Array<int> local_dof_map(dof*ne_ref);
+   for (int ie_lo = 0; ie_lo < ne_ref; ++ie_lo)
+   {
+      for (int i = 0; i < dof; ++i)
+      {
+         int cart_idx = RG.RefGeoms[i + dof*ie_lo]; // local Cartesian index
+         local_dof_map[i + dof*ie_lo] = fe_dof_map_ho[cart_idx];
+      }
+   }
+
+   const Table& e2dTable_ho = fes_ho.GetElementToDofTable();
+
+   auto d_offsets = offsets.Write();
+   const int NDOFS = ndofs;
+   dbg();
+   MFEM_FORALL(i, NDOFS+1, d_offsets[i] = 0;);
+
+   const Memory<int> &J = e2dTable_ho.GetJMemory();
+   const MemoryClass mc = Device::GetDeviceMemoryClass();
+   const int *d_elementMap = J.Read(mc, J.Capacity());
+   const int *d_local_dof_map = local_dof_map.Read();
+   const int DOF = dof;
+   const int DOF_ho = fe_ho->GetDof();
+   const int NE = ne;
+   const int NR_REF = ne_ref;
+
+   dbg();
+   MFEM_FORALL(e, NE,
+   {
+      const int e_ho = e/NR_REF;
+      const int i_ref = e%NR_REF;
+      for (int d = 0; d < DOF; ++d)
+      {
+         const int d_ho = d_local_dof_map[d + i_ref*DOF];
+         const int sgid = d_elementMap[DOF_ho*e_ho + d_ho];  // signed
+         const int gid = (sgid >= 0) ? sgid : -1 - sgid;
+         AtomicAdd(d_offsets[gid+1], 1);
+      }
+   });
+
+   // Aggregate to find offsets for each global dof
+   offsets.HostReadWrite();
+   for (int i = 1; i <= ndofs; ++i) { offsets[i] += offsets[i - 1]; }
+
+   // For each global dof, fill in all local nodes that point to it
+   auto d_gather = gatherMap.Write();
+   auto d_indices = indices.Write();
+   auto drw_offsets = offsets.ReadWrite();
+   const auto dof_map_mem = fe_dof_map.GetMemory();
+   const auto d_dof_map = fe_dof_map.GetMemory().Read(mc,dof_map_mem.Capacity());
+
+   dbg();
+   MFEM_FORALL(e, NE,
+   {
+      const int e_ho = e/NR_REF;
+      const int i_ref = e%NR_REF;
+      for (int d = 0; d < DOF; ++d)
+      {
+         int d_ho = d_local_dof_map[d + i_ref*DOF];
+         const int sdid = d_dof_map[d];  // signed
+         // const int did = d;
+         const int sgid = d_elementMap[DOF_ho*e_ho + d_ho];  // signed
+         const int gid = (sgid >= 0) ? sgid : -1-sgid;
+         const int lid = DOF*e + d;
+         const bool plus = (sgid >= 0 && sdid >= 0) || (sgid < 0 && sdid < 0);
+         d_gather[lid] = plus ? gid : -1-gid;
+         d_indices[AtomicAdd(drw_offsets[gid], 1)] = plus ? lid : -1-lid;
+      }
+   });
+
+   dbg();
+   offsets.HostReadWrite();
+   for (int i = ndofs; i > 0; --i) { offsets[i] = offsets[i - 1]; }
+   offsets[0] = 0;
+}
+
+void LORRestriction::SetupGlobalToLocal()
+{
+   MFEM_NVTX;
+   const int ndof = fes_ho.GetVSize();
+   const int nel_ho = fes_ho.GetMesh()->GetNE();
+   const int order = fes_ho.GetMaxElementOrder();
+   const int dim = fes_ho.GetMesh()->Dimension();
+   MFEM_VERIFY(dim==3, "Not supported");
+   const int nd1d = order + 1;
+   const int ndof_per_el = nd1d*nd1d*nd1d;
+
+   dof_glob2loc.SetSize(2*ndof_per_el*nel_ho);
+   dof_glob2loc_offsets.SetSize(ndof+1);
+   el_dof_lex.SetSize(ndof_per_el*nel_ho);
+
+   Array<int> dofs;
+
+   const Array<int> &lex_map =
+      dynamic_cast<const NodalFiniteElement&>
+      (*fes_ho.GetFE(0)).GetLexicographicOrdering();
+
+   dof_glob2loc_offsets = 0;
+   const Memory<int> &I = fes_ho.GetElementToDofTable().GetIMemory();
+   const Memory<int> &J = fes_ho.GetElementToDofTable().GetJMemory();
+   I.Read(MemoryClass::HOST, I.Capacity());
+   J.Read(MemoryClass::HOST, J.Capacity());
+
+   for (int iel_ho=0; iel_ho<nel_ho; ++iel_ho)
+   {
+      fes_ho.GetElementDofs(iel_ho, dofs);
+      for (int i=0; i<ndof_per_el; ++i)
+      {
+         const int dof = dofs[lex_map[i]];
+         el_dof_lex[i + iel_ho*ndof_per_el] = dof;
+         dof_glob2loc_offsets[dof+1] += 2;
+      }
+   }
+
+   dof_glob2loc_offsets.PartialSum();
+
+   // Sanity check
+   MFEM_VERIFY(dof_glob2loc_offsets[ndof] == dof_glob2loc.Size(), "");
+
+   Array<int> dof_ptr(ndof);
+
+   for (int i=0; i<ndof; ++i) { dof_ptr[i] = dof_glob2loc_offsets[i]; }
+
+   for (int iel_ho=0; iel_ho<nel_ho; ++iel_ho)
+   {
+      fes_ho.GetElementDofs(iel_ho, dofs);
+      for (int i=0; i<ndof_per_el; ++i)
+      {
+         const int dof = dofs[lex_map[i]];
+         dof_glob2loc[dof_ptr[dof]++] = iel_ho;
+         dof_glob2loc[dof_ptr[dof]++] = i;
+      }
+   }
+}
+
+static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
+                                      const int *nbr_elts, const int nbrNbElts)
+{
+   // Find the minimal element index found in both my_elts[] and nbr_elts[]
+   int min_el = INT_MAX;
+   for (int i = 0; i < nbElts; i++)
+   {
+      const int e_i = my_elts[i];
+      if (e_i >= min_el) { continue; }
+      for (int j = 0; j < nbrNbElts; j++)
+      {
+         if (e_i==nbr_elts[j])
+         {
+            min_el = e_i; // we already know e_i < min_el
+            break;
+         }
+      }
+   }
+   return min_el;
+}
+
+int LORRestriction::FillI(SparseMatrix &mat) const
+{
+   MFEM_NVTX;
+   static constexpr int Max = 16;
+   const int all_dofs = ndofs;
+   const int vd = vdim;
+   const int elt_dofs = dof;
+   auto I = mat.ReadWriteI();
+   auto d_offsets = offsets.Read();
+   auto d_indices = indices.Read();
+   auto d_gatherMap = gatherMap.Read();
+   MFEM_FORALL(i_L, vd*all_dofs+1, { I[i_L] = 0; });
+   MFEM_FORALL(e, ne,
+   {
+      for (int i = 0; i < elt_dofs; i++)
+      {
+         int i_elts[Max];
+         const int i_E = e*elt_dofs + i;
+         const int i_L = d_gatherMap[i_E];
+         const int i_offset = d_offsets[i_L];
+         const int i_nextOffset = d_offsets[i_L+1];
+         const int i_nbElts = i_nextOffset - i_offset;
+         for (int e_i = 0; e_i < i_nbElts; ++e_i)
+         {
+            const int i_E = d_indices[i_offset+e_i];
+            i_elts[e_i] = i_E/elt_dofs;
+         }
+         for (int j = 0; j < elt_dofs; j++)
+         {
+            const int j_E = e*elt_dofs + j;
+            const int j_L = d_gatherMap[j_E];
+            const int j_offset = d_offsets[j_L];
+            const int j_nextOffset = d_offsets[j_L+1];
+            const int j_nbElts = j_nextOffset - j_offset;
+            if (i_nbElts == 1 || j_nbElts == 1) // no assembly required
+            {
+               AtomicAdd(I[i_L],1);
+            }
+            else // assembly required
+            {
+               int j_elts[Max];
+               for (int e_j = 0; e_j < j_nbElts; ++e_j)
+               {
+                  const int j_E = d_indices[j_offset+e_j];
+                  const int elt = j_E/elt_dofs;
+                  j_elts[e_j] = elt;
+               }
+               const int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
+               if (e == min_e) // add the nnz only once
+               {
+                  AtomicAdd(I[i_L],1);
+               }
+            }
+         }
+      }
+   });
+   // We need to sum the entries of I, we do it on CPU as it is very sequential.
+   auto h_I = mat.HostReadWriteI();
+   const int nTdofs = vd*all_dofs;
+   int sum = 0;
+   for (int i = 0; i < nTdofs; i++)
+   {
+      const int nnz = h_I[i];
+      h_I[i] = sum;
+      sum+=nnz;
+   }
+   h_I[nTdofs] = sum;
+   // We return the number of nnz
+   return h_I[nTdofs];
+}
+
+void LORRestriction::FillJAndZeroData(SparseMatrix &mat) const
+{
+   MFEM_NVTX;
+   static constexpr int Max = 8;
+   const int all_dofs = ndofs;
+   const int vd = vdim;
+   const int elt_dofs = dof;
+   auto I = mat.ReadWriteI();
+   auto J = mat.WriteJ();
+   auto Data = mat.WriteData();
+   const int NE = ne;
+   auto d_offsets = offsets.Read();
+   auto d_indices = indices.Read();
+   auto d_gatherMap = gatherMap.Read();
+
+   MFEM_FORALL(e, NE,
+   {
+      for (int i = 0; i < elt_dofs; i++)
+      {
+         int i_elts[Max];
+         const int i_E = e*elt_dofs + i;
+         const int i_L = d_gatherMap[i_E];
+         const int i_offset = d_offsets[i_L];
+         const int i_nextOffset = d_offsets[i_L+1];
+         const int i_nbElts = i_nextOffset - i_offset;
+         for (int e_i = 0; e_i < i_nbElts; ++e_i)
+         {
+            const int i_E = d_indices[i_offset+e_i];
+            i_elts[e_i] = i_E/elt_dofs;
+         }
+         for (int j = 0; j < elt_dofs; j++)
+         {
+            const int j_E = e*elt_dofs + j;
+            const int j_L = d_gatherMap[j_E];
+            const int j_offset = d_offsets[j_L];
+            const int j_nextOffset = d_offsets[j_L+1];
+            const int j_nbElts = j_nextOffset - j_offset;
+            if (i_nbElts == 1 || j_nbElts == 1) // no assembly required
+            {
+               const int nnz = AtomicAdd(I[i_L],1);
+               J[nnz] = j_L;
+               Data[nnz] = 0.0;
+            }
+            else // assembly required
+            {
+               int j_elts[Max];
+               for (int e_j = 0; e_j < j_nbElts; ++e_j)
+               {
+                  const int j_E = d_indices[j_offset+e_j];
+                  const int elt = j_E/elt_dofs;
+                  j_elts[e_j] = elt;
+               }
+               const int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
+               if (e == min_e) // add the nnz only once
+               {
+                  const int nnz = AtomicAdd(I[i_L],1);
+                  J[nnz] = j_L;
+                  Data[nnz] = 0.0;
+               }
+            }
+         }
+      }
+   });
+   // We need to shift again the entries of I, we do it on CPU as it is very
+   // sequential.
+   auto h_I = mat.HostReadWriteI();
+   const int size = vd*all_dofs;
+   for (int i = 0; i < size; i++) { h_I[size-i] = h_I[size-(i+1)]; }
+   h_I[0] = 0;
+}
+
+LORRestriction::~LORRestriction()
+{
+   delete fec_lo;
+}
 
 } // namespace mfem

fem/lor.hpp

@@ -69,6 +69,10 @@ protected:
    BilinearForm *a;
    OperatorHandle A;
    mutable Array<int> perm;
+   bool supports_batched_assembly;
+
+   /// The LOR element restriction operator.
+   mutable class LORRestriction *R_lor;
 
    /// Constructs the local DOF (ldof) permutation. In parallel this is used as
    /// an intermediate step in computing the DOF permutation (see
@@ -120,6 +124,8 @@ public:
    /// Returns the low-order refined finite element space.
    FiniteElementSpace &GetFESpace() const { return *fes; }
 
+   /// Returns the low-order restriction.
+   const LORRestriction *GetLORRestriction() const;
    ~LORBase();
 };
 
@@ -259,6 +265,51 @@ public:
    ~LORSolver() { if (own_lor) { delete lor; } }
 };
 
+
+/// Create a low-order refined version of a Restriction.
+/// Only used here for the FillI and FillJAndZeroData methods.
+class LORRestriction
+{
+   const FiniteElementSpace &fes_ho;
+   FiniteElementCollection *fec_lo;
+   const Geometry::Type geom;
+   const int ne_ref;
+   const int ne;
+   const int vdim;
+   const bool byvdim;
+   const int ndofs;
+   const int dof;
+
+   Array<int> offsets;
+   Array<int> indices;
+   Array<int> gatherMap;
+
+   Array<int> dof_glob2loc;
+   Array<int> dof_glob2loc_offsets;
+   Array<int> el_dof_lex;
+
+protected:
+   static int GetNRefinedElements(const FiniteElementSpace &fes);
+   static FiniteElementCollection *GetLowOrderFEC(const FiniteElementSpace &fes);
+
+public:
+   LORRestriction(const FiniteElementSpace &fes_ho);
+
+   int FillI(SparseMatrix &mat) const;
+   void FillJAndZeroData(SparseMatrix &mat) const;
+
+   const Array<int> &GatherMap() const { return el_dof_lex; }
+   const Array<int> &Indices() const { return dof_glob2loc; }
+   const Array<int> &Offsets() const { return dof_glob2loc_offsets; }
+
+   ~LORRestriction();
+
+   // Device lambda cannot have private or protected access
+public:
+   void SetupLocalToElement();
+   void SetupGlobalToLocal();
+};
+
 } // namespace mfem
 
 #endif

fem/lor_assembly.cpp

@@ -0,0 +1,412 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "lor.hpp"
+#include "lor_assembly.hpp"
+#include "../linalg/dtensor.hpp"
+#include "../general/forall.hpp"
+
+#define MFEM_DEBUG_COLOR 226
+#include "../general/debug.hpp"
+
+#define MFEM_NVTX_COLOR DarkOrchid
+#include "../general/nvtx.hpp"
+
+#include <thrust/execution_policy.h>
+#include <thrust/scan.h>
+
+namespace mfem
+{
+
+// Defined in lor_assembly_ker.cpp
+template <int order, bool use_smem = true>
+void Assemble3DBatchedLOR(const Array<int> &dof_glob2loc,
+                          const Array<int> &dof_glob2loc_offsets,
+                          const Array<int> &el_dof_lex,
+                          Mesh &mesh_ho,
+                          SparseMatrix &A_mat);
+
+static void AssembleBatchedLORWithoutBC(LORBase &lor_disc,
+                                        BilinearForm &form_lor,
+                                        FiniteElementSpace &fes_ho,
+                                        OperatorHandle &Ah)
+{
+   MFEM_NVTX;
+   Mesh &mesh_ho = *fes_ho.GetMesh();
+   const int dim = mesh_ho.Dimension();
+   const int order = fes_ho.GetMaxElementOrder();
+   dbg("dim:%d order:%d",dim,order);
+
+   const bool has_to_init = Ah.Ptr() == nullptr;
+   dbg("has_to_init: %s", has_to_init?"yes":"no");
+   SparseMatrix *A = has_to_init ? nullptr : Ah.As<SparseMatrix>();
+
+   dbg("GetLORRestriction");
+   const LORRestriction *R = lor_disc.GetLORRestriction();
+   MFEM_VERIFY(R,"LOR Restriction error!");
+
+   if (has_to_init)
+   {
+      MFEM_VERIFY(UsesTensorBasis(fes_ho),
+                  "Batched LOR assembly requires tensor basis");
+      if (Device::IsEnabled()||true)
+      {
+         dbg("Device::IsEnabled()");
+#ifdef MFEM_USE_MPI
+         ParFiniteElementSpace *pfes_ho =
+            dynamic_cast<ParFiniteElementSpace*>(&fes_ho);
+         if (pfes_ho && pfes_ho->GetNRanks() > 1)
+         {
+            dbg("Device::IsEnabled() and multiple ranks!");
+            const int width = pfes_ho->GetVSize();
+            const int height = pfes_ho->GetVSize();
+            dbg("HxW: %dx%d",height,width);
+            A = new SparseMatrix(height, width, 0);
+         }
+         else
+#endif
+         {
+            dbg("Device::IsEnabled() but one rank!");
+            const int width = fes_ho.GetVSize();
+            const int height = fes_ho.GetVSize();
+            dbg("HxW: %dx%d",height,width);
+            A = new SparseMatrix(height, width, 0);
+         }
+         A->GetMemoryI().New(A->Height()+1, A->GetMemoryI().GetMemoryType());
+         const int nnz = R->FillI(*A);
+         A->GetMemoryJ().New(nnz, A->GetMemoryJ().GetMemoryType());
+         A->GetMemoryData().New(nnz, A->GetMemoryData().GetMemoryType());
+         R->FillJAndZeroData(*A); // J, A = 0.0
+      }
+      else
+      {
+         MFEM_ABORT("");
+         dbg("NOT Device::IsEnabled()");
+         // the sparsity pattern is defined from the map: element->dof
+         const int ndofs = fes_ho.GetVSize();
+         dbg("ndofs:%d",ndofs);
+         const Table &elem_dof = form_lor.FESpace()->GetElementToDofTable();
+         Table dof_dof, dof_elem;
+         Transpose(elem_dof, dof_elem, ndofs);
+         mfem::Mult(dof_elem, elem_dof, dof_dof);
+         dof_dof.SortRows();
+         int *I = dof_dof.GetI();
+         int *J = dof_dof.GetJ();
+         double *data = Memory<double>(I[ndofs]);
+         A = new SparseMatrix(I,J,data,ndofs,ndofs,true,true,true);
+         dof_dof.LoseData();
+         *A = 0.0;
+         dbg("done");
+      }
+   }
+
+   void (*Kernel)(const Array<int> &dof_glob2loc,
+                  const Array<int> &dof_glob2loc_offsets,
+                  const Array<int> &el_dof_lex,
+                  Mesh &mesh_ho,
+                  SparseMatrix &A_mat) = nullptr;
+
+   if (dim == 2) { MFEM_ABORT("Unsuported!"); }
+   else if (dim == 3)
+   {
+      switch (order)
+      {
+         case 1: Kernel = Assemble3DBatchedLOR<1>; break;
+         case 2: Kernel = Assemble3DBatchedLOR<2>; break;
+         case 3: Kernel = Assemble3DBatchedLOR<3>; break;
+         case 4: Kernel = Assemble3DBatchedLOR<4>; break;
+         case 5: Kernel = Assemble3DBatchedLOR<5>; break;
+         case 6: Kernel = Assemble3DBatchedLOR<6,false>; break;/*
+         case 7: Kernel = Assemble3DBatchedLOR<7,false>; break;
+         case 8: Kernel = Assemble3DBatchedLOR<8,false>; break;
+         case 9: Kernel = Assemble3DBatchedLOR<9,false>; break;
+         case 10: Kernel = Assemble3DBatchedLOR<10,false>; break;
+         case 11: Kernel = Assemble3DBatchedLOR<11,false>; break;
+         case 12: Kernel = Assemble3DBatchedLOR<12,false>; break;
+         case 13: Kernel = Assemble3DBatchedLOR<13,false>; break;
+         case 14: Kernel = Assemble3DBatchedLOR<14,false>; break;
+         case 15: Kernel = Assemble3DBatchedLOR<15,false>; break;
+         case 16: Kernel = Assemble3DBatchedLOR<16,false>; break;*/
+         default: MFEM_ABORT("Kernel not ready!");
+      }
+   }
+
+   Kernel(R->Indices(),
+          R->Offsets(),
+          R->GatherMap(),
+          mesh_ho, *A);
+
+   A->Finalize();
+
+   if (has_to_init) { Ah.Reset(A); } // A now owns A_mat
+}
+
+
+void AssembleBatchedLOR(LORBase &lor_disc,
+                        BilinearForm &form_lor,
+                        FiniteElementSpace &fes_ho,
+                        const Array<int> &ess_dofs,
+                        OperatorHandle &Ah)
+{
+   MFEM_NVTX;
+   AssembleBatchedLORWithoutBC(lor_disc, form_lor, fes_ho, Ah);
+
+   // Set essential dofs to 0.0
+   const int n_ess_dofs = ess_dofs.Size();
+   const auto ess_dofs_d = ess_dofs.Read();
+
+   const auto I = Ah.As<SparseMatrix>()->ReadI();
+   const auto J = Ah.As<SparseMatrix>()->ReadJ();
+   auto dA = Ah.As<SparseMatrix>()->ReadWriteData();
+
+   MFEM_FORALL(i, n_ess_dofs,
+   {
+      const int idof = ess_dofs_d[i];
+      for (int j=I[idof]; j<I[idof+1]; ++j)
+      {
+         const int jdof = J[j];
+         if (jdof != idof)
+         {
+            dA[j] = 0.0;
+            for (int k=I[jdof]; k<I[jdof+1]; ++k)
+            {
+               if (J[k] == idof)
+               {
+                  dA[k] = 0.0;
+                  break;
+               }
+            }
+         }
+      }
+   });
+}
+
+
+#ifdef MFEM_USE_MPI
+
+void ParAssembleBatchedLOR(LORBase &lor_disc,
+                           BilinearForm &form_lor,
+                           FiniteElementSpace &fes_ho,
+                           const Array<int> &ess_dofs,
+                           OperatorHandle &Ah)
+{
+   dbg();
+   MFEM_NVTX;
+   ParFiniteElementSpace *pfes_ho =
+      dynamic_cast<ParFiniteElementSpace*>(&fes_ho);
+   assert(pfes_ho);
+
+   OperatorHandle A_local(Operator::MFEM_SPARSEMAT);
+   AssembleBatchedLORWithoutBC(lor_disc, form_lor, fes_ho, A_local);
+   MFEM_VERIFY(A_local.As<SparseMatrix>()->Finalized(),
+               "the local matrix must be finalized");
+
+   NVTX("Parallel");
+   OperatorHandle dA(Operator::Hypre_ParCSR),
+                  Ph(Operator::Hypre_ParCSR);
+   {
+      NVTX("MakeSquareBlockDiag");
+      dA.MakeSquareBlockDiag(pfes_ho->GetComm(),
+                             pfes_ho->GlobalVSize(),
+                             pfes_ho->GetDofOffsets(),
+                             A_local.As<SparseMatrix>());
+   }
+   Ph.ConvertFrom(pfes_ho->Dof_TrueDof_Matrix());
+
+   {
+      NVTX("MakePtAP");
+      Ah.MakePtAP(dA, Ph);
+   }
+
+   {
+      dbg("EliminateRowsCols");
+      NVTX("EliminateRowsCols");
+      HypreParMatrix *A_mat = Ah.As<HypreParMatrix>();
+      hypre_ParCSRMatrix *A = *A_mat;
+      A_mat->HypreReadWrite();
+
+      hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(A);
+      hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(A);
+
+      HYPRE_Int diag_nrows = hypre_CSRMatrixNumRows(diag);
+      HYPRE_Int offd_ncols = hypre_CSRMatrixNumCols(offd);
+      dbg("diag_nrows:%d offd_ncols:%d", diag_nrows, offd_ncols);
+
+      const int n_ess_dofs = ess_dofs.Size();
+      const auto ess_dofs_d = ess_dofs.Read();
+      dbg("n_ess_dofs:%d", n_ess_dofs);
+
+      // Start communication to figure out which columns need to be eliminated in
+      // the off-diagonal block
+      hypre_ParCSRCommHandle *comm_handle;
+      HYPRE_Int *int_buf_data, *eliminate_row, *eliminate_col;
+      {
+         eliminate_row = hypre_CTAlloc(HYPRE_Int, diag_nrows, HYPRE_MEMORY_DEVICE);
+         eliminate_col = hypre_CTAlloc(HYPRE_Int, offd_ncols, HYPRE_MEMORY_DEVICE);
+
+         // Get the communication package for A, creating it if it does not
+         // already exist.
+         hypre_ParCSRCommPkg *comm_pkg = hypre_ParCSRMatrixCommPkg(A);
+         if (!comm_pkg)
+         {
+            hypre_MatvecCommPkgCreate(A);
+            comm_pkg = hypre_ParCSRMatrixCommPkg(A);
+         }
+
+         MFEM_FORALL(i, diag_nrows,
+         {
+            eliminate_row[i] = 0;
+         });
+         MFEM_FORALL(i, n_ess_dofs,
+         {
+            eliminate_row[ess_dofs_d[i]] = 1;
+         });
+
+         // Use a matvec communication pattern to find (in eliminate_col) which of
+         // the local offd columns are to be eliminated
+         HYPRE_Int num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
+         HYPRE_Int int_buf_sz = hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends);
+         dbg("num_sends:%d", num_sends);
+         int_buf_data = hypre_CTAlloc(HYPRE_Int, int_buf_sz, HYPRE_MEMORY_DEVICE);
+
+         hypre_ParCSRCommPkgCopySendMapElmtsToDevice(comm_pkg);
+         HYPRE_Int *send_map_elmts = hypre_ParCSRCommPkgDeviceSendMapElmts(comm_pkg);
+         MFEM_FORALL(i, int_buf_sz,
+         {
+            int k = send_map_elmts[i];
+            int_buf_data[i] = eliminate_row[k];
+         });
+
+         // Try to use device-aware MPI for the communication
+         comm_handle = hypre_ParCSRCommHandleCreate_v2(
+            11, comm_pkg, HYPRE_MEMORY_DEVICE, int_buf_data,
+            HYPRE_MEMORY_DEVICE, eliminate_col);
+      }
+
+      // Eliminate rows and columns in the diagonal block
+      {
+         dbg("Eliminate rows and columns in the diagonal block");
+         const auto I = diag->i;
+         const auto J = diag->j;
+         auto data = diag->data;
+
+         MFEM_FORALL(i, n_ess_dofs,
+         {
+            const int idof = ess_dofs_d[i];
+            for (int j=I[idof]; j<I[idof+1]; ++j)
+            {
+               const int jdof = J[j];
+               if (jdof != idof)
+               {
+                  data[j] = 0.0;
+                  for (int k=I[jdof]; k<I[jdof+1]; ++k)
+                  {
+                     if (J[k] == idof)
+                     {
+                        data[k] = 0.0;
+                        break;
+                     }
+                  }
+               }
+            }
+         });
+      }
+
+      // Eliminate rows in the off-diagonal block
+      {
+         dbg("Eliminate rows in the off-diagonal block");
+         const auto I = offd->i;
+         auto data = offd->data;
+         MFEM_FORALL(i, n_ess_dofs,
+         {
+            const int idof = ess_dofs_d[i];
+            for (int j=I[idof]; j<I[idof+1]; ++j)
+            {
+               data[j] = 0.0;
+            }
+         });
+      }
+
+      // Wait for MPI communication to finish
+      Array<HYPRE_Int> cols_to_eliminate;
+      {
+         dbg("Wait for MPI communication to finish");
+         hypre_ParCSRCommHandleDestroy(comm_handle);
+
+         // Convert a "column marker" array to a "column index" array.
+
+         // First we need to know how many columns are marked, and do a partial
+         // sum to figure out which index they correspond to.
+         HYPRE_Int *col_idx = hypre_CTAlloc(HYPRE_Int, offd_ncols, HYPRE_MEMORY_DEVICE);
+         thrust::exclusive_scan(
+            thrust::device, eliminate_col, eliminate_col + offd_ncols, col_idx
+         );
+
+         // Because we do an exclusive scan above, the last entry is missing
+         // from the partial sum. To compute the total sum, get last entry of
+         // both arrays and add them.
+         HYPRE_Int partial_sum, last_element;
+         hypre_TMemcpy(&partial_sum, col_idx + offd_ncols - 1, HYPRE_Int, 1, HYPRE_MEMORY_HOST, HYPRE_MEMORY_DEVICE);
+         hypre_TMemcpy(&last_element, eliminate_col + offd_ncols - 1, HYPRE_Int, 1, HYPRE_MEMORY_HOST, HYPRE_MEMORY_DEVICE);
+         HYPRE_Int ncols_to_eliminate = partial_sum + last_element;
+
+         // Perform the conversion to "column index" array.
+         cols_to_eliminate.SetSize(ncols_to_eliminate);
+         HYPRE_Int *cols = cols_to_eliminate.Write();
+         MFEM_FORALL(i, offd_ncols,
+         {
+            if (eliminate_col[i])
+            {
+               cols[col_idx[i]] = i;
+            }
+         });
+
+         hypre_TFree(int_buf_data, HYPRE_MEMORY_DEVICE);
+         hypre_TFree(eliminate_row, HYPRE_MEMORY_DEVICE);
+         hypre_TFree(eliminate_col, HYPRE_MEMORY_DEVICE);
+         hypre_TFree(col_idx, HYPRE_MEMORY_DEVICE);
+      }
+
+      // Eliminate columns in the off-diagonal block
+      {
+         dbg("Eliminate columns in the off-diagonal block");
+         const int ncols_to_eliminate = cols_to_eliminate.Size();
+         const int nrows_offd = hypre_CSRMatrixNumRows(offd);
+         const auto cols = cols_to_eliminate.Read();
+         const auto I = offd->i;
+         const auto J = offd->j;
+         auto data = offd->data;
+         dbg("ncols_to_eliminate:%d nrows_offd:%d", ncols_to_eliminate, nrows_offd);
+         // Note: could also try a different strategy, looping over nnz in the
+         // matrix and then doing a binary search in ncols_to_eliminate to see if
+         // the column should be eliminated.
+         MFEM_FORALL(idx, ncols_to_eliminate,
+         {
+            const int j = cols[idx];
+            for (int i=0; i<nrows_offd; ++i)
+            {
+               for (int jj=I[i]; jj<I[i+1]; ++jj)
+               {
+                  if (J[jj] == j)
+                  {
+                     data[jj] = 0.0;
+                     break;
+                  }
+               }
+            }
+         });
+      }
+   }
+}
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem

fem/lor_assembly.hpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LOR_ASSEMBLY
+#define MFEM_LOR_ASSEMBLY
+
+#include "lor.hpp"
+#include "bilinearform.hpp"
+#include "pbilinearform.hpp"
+
+namespace mfem
+{
+
+void AssembleBatchedLOR(LORBase &lor_disc,
+                        BilinearForm &form_lo,
+                        FiniteElementSpace &fes_ho,
+                        const Array<int> &ess_dofs,
+                        OperatorHandle &A);
+
+#ifdef MFEM_USE_MPI
+
+void ParAssembleBatchedLOR(LORBase &lor_disc,
+                           BilinearForm &form_lo,
+                           FiniteElementSpace &fes_ho,
+                           const Array<int> &ess_dofs,
+                           OperatorHandle &A);
+
+#endif // MFEM_USE_MPI
+
+} // namespace mfem
+
+#endif

fem/lor_assembly_ker.cpp

@@ -0,0 +1,533 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "../general/forall.hpp"
+
+#define MFEM_DEBUG_COLOR 187
+#include "../general/debug.hpp"
+
+#define MFEM_NVTX_COLOR SlateBlue
+#include "../general/nvtx.hpp"
+
+namespace mfem
+{
+
+template<int D1D, int Q1D>
+void NodalInterpolation3D(const int NE,
+                          const Vector& localL, Vector& localH,
+                          const Array<double>& B);
+
+template <int order, bool USE_SMEM = true>
+void Assemble3DBatchedLOR(const Array<int> &dof_glob2loc_,
+                          const Array<int> &dof_glob2loc_offsets_,
+                          const Array<int> &el_dof_lex_,
+                          Mesh &mesh_ho,
+                          SparseMatrix &A_mat)
+{
+   const int nel_ho = mesh_ho.GetNE();
+
+   static constexpr int nv = 8;
+   static constexpr int dim = 3;
+   static constexpr int ddm2 = (dim*(dim+1))/2;
+   static constexpr int nd1d = order + 1;
+   static constexpr int ndof_per_el = nd1d*nd1d*nd1d;
+   static constexpr int nnz_per_row = 27;
+   static constexpr int nnz_per_el = nnz_per_row * ndof_per_el;
+   static constexpr int sz_grad_A = 3*3*2*2*2*2;
+   static constexpr int sz_grad_B = sz_grad_A*2;
+   static constexpr int sz_local_mat = 8*8;
+
+   static constexpr int GRID = USE_SMEM ? 0 : 128;
+   double *GM = nullptr;
+   static Vector *d_buffer = nullptr;
+   if (!USE_SMEM)
+   {
+      if (!d_buffer)
+      {
+         d_buffer = new Vector();
+         d_buffer->UseDevice(true);
+      }
+      d_buffer->SetSize(nnz_per_el*GRID);
+      GM = d_buffer->Write();
+   }
+
+   const auto el_dof_lex = Reshape(el_dof_lex_.Read(), ndof_per_el, nel_ho);
+   const auto dof_glob2loc = dof_glob2loc_.Read();
+   const auto K = dof_glob2loc_offsets_.Read();
+
+   const auto I = A_mat.ReadI();
+   const auto J = A_mat.ReadJ();
+   auto A = A_mat.ReadWriteData();
+
+   const GridFunction *nodal_gf = mesh_ho.GetNodes();
+   const FiniteElementSpace *nodal_fes = nodal_gf->FESpace();
+   const Operator *nodal_restriction = nodal_fes->GetElementRestriction(
+                                          ElementDofOrdering::LEXICOGRAPHIC);
+   const int nodal_nd1d = nodal_fes->GetMaxElementOrder() + 1;
+
+   IntegrationRules irs(0, Quadrature1D::GaussLobatto);
+   const IntegrationRule &ir = irs.Get(Geometry::Type::CUBE, 2*nd1d - 3);
+   MFEM_VERIFY(ir.Size() == ndof_per_el, "");
+
+   // Get the map from mesh nodes to LOR vertices
+   const DofToQuad& maps =
+      nodal_fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
+
+   // Map from nodal E-vector to L-vector
+   Vector nodes_loc(nodal_restriction->Height());
+   nodes_loc.UseDevice(true);
+   nodal_restriction->Mult(*nodal_gf, nodes_loc);
+
+   // Get nodal points at the LOR vertices
+   Vector X_loc(dim*ndof_per_el*nel_ho);
+   X_loc.UseDevice(true);
+
+   // Get the LOR vertex coordinates
+   MFEM_VERIFY(nd1d==order+1, "nd1d!=order+1");
+   switch (nodal_nd1d)
+   {
+      case 2:
+      {
+         NodalInterpolation3D<2,nd1d>(nel_ho, nodes_loc, X_loc, maps.B);
+         break;
+      }
+      case 4:
+      {
+         NodalInterpolation3D<4,nd1d>(nel_ho, nodes_loc, X_loc, maps.B);
+         break;
+      }
+      case 6:
+      {
+         NodalInterpolation3D<6,nd1d>(nel_ho, nodes_loc, X_loc, maps.B);
+         break;
+      }
+      default: MFEM_ABORT("Unsuported mesh order!");
+   }
+   auto X = X_loc.Read();
+
+   // Last GRID dimension is lowered to avoid too many resources
+   MFEM_FORALL_3D_GRID(iel_ho, nel_ho, order, order, USE_SMEM?order:1, GRID,
+   {
+      const int bid = MFEM_BLOCK_ID(x);
+      MFEM_SHARED double smem[USE_SMEM ? nnz_per_el : 1];
+      double *V_ = USE_SMEM ? smem : GM + nnz_per_el*bid;
+      DeviceTensor<4> V(V_, nnz_per_row, nd1d, nd1d, nd1d);
+
+      // Assemble a sparse matrix over the macro-element by looping over each
+      // subelement.
+      // V(j,i) stores the jth nonzero in the ith row of the sparse matrix.
+      MFEM_FOREACH_THREAD(iz,z,nd1d)
+      {
+         MFEM_FOREACH_THREAD(iy,y,nd1d)
+         {
+            MFEM_FOREACH_THREAD(ix,x,nd1d)
+            {
+               //MFEM_UNROLL(27)
+               for (int j=0; j<nnz_per_row; ++j)
+               {
+                  V(j,ix,iy,iz) = 0.0;
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Compute geometric factors at quadrature points
+      MFEM_FOREACH_THREAD(kz,z,order)
+      {
+         MFEM_FOREACH_THREAD(ky,y,order)
+         {
+            MFEM_FOREACH_THREAD(kx,x,order)
+            {
+               double Q_[ddm2*nv];
+               double grad_A_[sz_grad_A];
+               double grad_B_[sz_grad_B];
+               double local_mat_[sz_local_mat];
+
+               DeviceTensor<4> Q(Q_, ddm2,2,2,2);
+               DeviceTensor<2> local_mat(local_mat_, 8, 8);
+               DeviceTensor<6> grad_A(grad_A_, 3, 3, 2, 2, 2, 2);
+               DeviceTensor<7> grad_B(grad_B_, 3, 3, 2, 2, 2, 2, 2);
+
+               // local_mat is the local (dense) stiffness matrix
+               for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }
+
+               // Intermediate quantities
+               // (see e.g. Mora and Demkowicz for notation).
+               for (int i=0; i<sz_grad_A; ++i) { grad_A[i] = 0.0; }
+               for (int i=0; i<sz_grad_B; ++i) { grad_B[i] = 0.0; }
+
+               const int v0 = kx + nd1d*(ky + nd1d*kz);
+               const int v1 = kx + 1 + nd1d*(ky + nd1d*kz);
+               const int v2 = kx + 1 + nd1d*(ky + 1 + nd1d*kz);
+               const int v3 = kx + nd1d*(ky + 1 + nd1d*kz);
+               const int v4 = kx + nd1d*(ky + nd1d*(kz + 1));
+               const int v5 = kx + 1 + nd1d*(ky + nd1d*(kz + 1));
+               const int v6 = kx + 1 + nd1d*(ky + 1 + nd1d*(kz + 1));
+               const int v7 = kx + nd1d*(ky + 1 + nd1d*(kz + 1));
+
+               const int e0 = dim*(v0 + ndof_per_el*iel_ho);
+               const int e1 = dim*(v1 + ndof_per_el*iel_ho);
+               const int e2 = dim*(v2 + ndof_per_el*iel_ho);
+               const int e3 = dim*(v3 + ndof_per_el*iel_ho);
+               const int e4 = dim*(v4 + ndof_per_el*iel_ho);
+               const int e5 = dim*(v5 + ndof_per_el*iel_ho);
+               const int e6 = dim*(v6 + ndof_per_el*iel_ho);
+               const int e7 = dim*(v7 + ndof_per_el*iel_ho);
+
+               const double v0x = X[e0 + 0];
+               const double v0y = X[e0 + 1];
+               const double v0z = X[e0 + 2];
+
+               const double v1x = X[e1 + 0];
+               const double v1y = X[e1 + 1];
+               const double v1z = X[e1 + 2];
+
+               const double v2x = X[e2 + 0];
+               const double v2y = X[e2 + 1];
+               const double v2z = X[e2 + 2];
+
+               const double v3x = X[e3 + 0];
+               const double v3y = X[e3 + 1];
+               const double v3z = X[e3 + 2];
+
+               const double v4x = X[e4 + 0];
+               const double v4y = X[e4 + 1];
+               const double v4z = X[e4 + 2];
+
+               const double v5x = X[e5 + 0];
+               const double v5y = X[e5 + 1];
+               const double v5z = X[e5 + 2];
+
+               const double v6x = X[e6 + 0];
+               const double v6y = X[e6 + 1];
+               const double v6z = X[e6 + 2];
+
+               const double v7x = X[e7 + 0];
+               const double v7y = X[e7 + 1];
+               const double v7z = X[e7 + 2];
+
+               //MFEM_UNROLL(2)
+               for (int iqz=0; iqz<2; ++iqz)
+               {
+                  //MFEM_UNROLL(2)
+                  for (int iqy=0; iqy<2; ++iqy)
+                  {
+                     //MFEM_UNROLL(2)
+                     for (int iqx=0; iqx<2; ++iqx)
+                     {
+
+                        const double x = iqx;
+                        const double y = iqy;
+                        const double z = iqz;
+                        const double w = 1.0/8.0;
+
+                        // c: (1-x)(1-y)(1-z)v0[c] + x (1-y)(1-z)v1[c] + x y (1-z)v2[c] + (1-x) y (1-z)v3[c]
+                        //  + (1-x)(1-y) z   v4[c] + x (1-y) z   v5[c] + x y z    v6[c] + (1-x) y z    v7[c]
+                        const double J11 = -(1-y)*(1-z)*v0x
+                                           + (1-y)*(1-z)*v1x + y*(1-z)*v2x - y*(1-z)*v3x
+                                           - (1-y)*z*v4x + (1-y)*z*v5x + y*z*v6x - y*z*v7x;
+
+                        const double J12 = -(1-x)*(1-z)*v0x
+                                           - x*(1-z)*v1x + x*(1-z)*v2x + (1-x)*(1-z)*v3x
+                                           - (1-x)*z*v4x - x*z*v5x + x*z*v6x + (1-x)*z*v7x;
+
+                        const double J13 = -(1-x)*(1-y)*v0x - x*(1-y)*v1x
+                                           - x*y*v2x - (1-x)*y*v3x + (1-x)*(1-y)*v4x
+                                           + x*(1-y)*v5x + x*y*v6x + (1-x)*y*v7x;
+
+                        const double J21 = -(1-y)*(1-z)*v0y + (1-y)*(1-z)*v1y
+                                           + y*(1-z)*v2y - y*(1-z)*v3y - (1-y)*z*v4y
+                                           + (1-y)*z*v5y + y*z*v6y - y*z*v7y;
+
+                        const double J22 = -(1-x)*(1-z)*v0y - x*(1-z)*v1y
+                                           + x*(1-z)*v2y + (1-x)*(1-z)*v3y- (1-x)*z*v4y -
+                                           x*z*v5y + x*z*v6y + (1-x)*z*v7y;
+
+                        const double J23 = -(1-x)*(1-y)*v0y - x*(1-y)*v1y
+                                           - x*y*v2y - (1-x)*y*v3y + (1-x)*(1-y)*v4y
+                                           + x*(1-y)*v5y + x*y*v6y + (1-x)*y*v7y;
+
+                        const double J31 = -(1-y)*(1-z)*v0z + (1-y)*(1-z)*v1z
+                                           + y*(1-z)*v2z - y*(1-z)*v3z- (1-y)*z*v4z +
+                                           (1-y)*z*v5z + y*z*v6z - y*z*v7z;
+
+                        const double J32 = -(1-x)*(1-z)*v0z - x*(1-z)*v1z
+                                           + x*(1-z)*v2z + (1-x)*(1-z)*v3z - (1-x)*z*v4z
+                                           - x*z*v5z + x*z*v6z + (1-x)*z*v7z;
+
+                        const double J33 = -(1-x)*(1-y)*v0z - x*(1-y)*v1z
+                                           - x*y*v2z - (1-x)*y*v3z + (1-x)*(1-y)*v4z
+                                           + x*(1-y)*v5z + x*y*v6z + (1-x)*y*v7z;
+
+                        const double detJ = J11 * (J22 * J33 - J32 * J23) -
+                                            J21 * (J12 * J33 - J32 * J13) +
+                                            J31 * (J12 * J23 - J22 * J13);
+                        const double w_detJ = w/detJ;
+
+                        // adj(J)
+                        const double A11 = (J22 * J33) - (J23 * J32);
+                        const double A12 = (J32 * J13) - (J12 * J33);
+                        const double A13 = (J12 * J23) - (J22 * J13);
+                        const double A21 = (J31 * J23) - (J21 * J33);
+                        const double A22 = (J11 * J33) - (J13 * J31);
+                        const double A23 = (J21 * J13) - (J11 * J23);
+                        const double A31 = (J21 * J32) - (J31 * J22);
+                        const double A32 = (J31 * J12) - (J11 * J32);
+                        const double A33 = (J11 * J22) - (J12 * J21);
+
+                        Q(0,iqz,iqy,iqx) = w_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+                        Q(1,iqz,iqy,iqx) = w_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+                        Q(2,iqz,iqy,iqx) = w_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+                        Q(3,iqz,iqy,iqx) = w_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+                        Q(4,iqz,iqy,iqx) = w_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+                        Q(5,iqz,iqy,iqx) = w_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
+                     }
+                  }
+               }
+
+               //MFEM_UNROLL(2)
+               for (int iqx=0; iqx<2; ++iqx)
+               {
+                  //MFEM_UNROLL(2)
+                  for (int jz=0; jz<2; ++jz)
+                  {
+                     // Note loop starts at iz=jz here, taking advantage of
+                     // symmetries.
+                     //MFEM_UNROLL(2)
+                     for (int iz=jz; iz<2; ++iz)
+                     {
+                        //MFEM_UNROLL(2)
+                        for (int iqy=0; iqy<2; ++iqy)
+                        {
+                           //MFEM_UNROLL(2)
+                           for (int iqz=0; iqz<2; ++iqz)
+                           {
+                              const double biz = (iz == iqz) ? 1.0 : 0.0;
+                              const double giz = (iz == 0) ? -1.0 : 1.0;
+
+                              const double bjz = (jz == iqz) ? 1.0 : 0.0;
+                              const double gjz = (jz == 0) ? -1.0 : 1.0;
+
+                              const double J11 = Q(0,iqz,iqy,iqx);
+                              const double J21 = Q(1,iqz,iqy,iqx);
+                              const double J31 = Q(2,iqz,iqy,iqx);
+                              const double J12 = J21;
+                              const double J22 = Q(3,iqz,iqy,iqx);
+                              const double J32 = Q(4,iqz,iqy,iqx);
+                              const double J13 = J31;
+                              const double J23 = J32;
+                              const double J33 = Q(5,iqz,iqy,iqx);
+
+                              grad_A(0,0,iqy,iz,jz,iqx) += J11*biz*bjz;
+                              grad_A(1,0,iqy,iz,jz,iqx) += J21*biz*bjz;
+                              grad_A(2,0,iqy,iz,jz,iqx) += J31*giz*bjz;
+                              grad_A(0,1,iqy,iz,jz,iqx) += J12*biz*bjz;
+                              grad_A(1,1,iqy,iz,jz,iqx) += J22*biz*bjz;
+                              grad_A(2,1,iqy,iz,jz,iqx) += J32*giz*bjz;
+                              grad_A(0,2,iqy,iz,jz,iqx) += J13*biz*gjz;
+                              grad_A(1,2,iqy,iz,jz,iqx) += J23*biz*gjz;
+                              grad_A(2,2,iqy,iz,jz,iqx) += J33*giz*gjz;
+                           }
+                           //MFEM_UNROLL(2)
+                           for (int jy=0; jy<2; ++jy)
+                           {
+                              //MFEM_UNROLL(2)
+                              for (int iy=0; iy<2; ++iy)
+                              {
+                                 const double biy = (iy == iqy) ? 1.0 : 0.0;
+                                 const double giy = (iy == 0) ? -1.0 : 1.0;
+
+                                 const double bjy = (jy == iqy) ? 1.0 : 0.0;
+                                 const double gjy = (jy == 0) ? -1.0 : 1.0;
+
+                                 grad_B(0,0,iy,jy,iz,jz,iqx) += biy*bjy*grad_A(0,0,iqy,iz,jz,iqx);
+                                 grad_B(1,0,iy,jy,iz,jz,iqx) += giy*bjy*grad_A(1,0,iqy,iz,jz,iqx);
+                                 grad_B(2,0,iy,jy,iz,jz,iqx) += biy*bjy*grad_A(2,0,iqy,iz,jz,iqx);
+                                 grad_B(0,1,iy,jy,iz,jz,iqx) += biy*gjy*grad_A(0,1,iqy,iz,jz,iqx);
+                                 grad_B(1,1,iy,jy,iz,jz,iqx) += giy*gjy*grad_A(1,1,iqy,iz,jz,iqx);
+                                 grad_B(2,1,iy,jy,iz,jz,iqx) += biy*gjy*grad_A(2,1,iqy,iz,jz,iqx);
+                                 grad_B(0,2,iy,jy,iz,jz,iqx) += biy*bjy*grad_A(0,2,iqy,iz,jz,iqx);
+                                 grad_B(1,2,iy,jy,iz,jz,iqx) += giy*bjy*grad_A(1,2,iqy,iz,jz,iqx);
+                                 grad_B(2,2,iy,jy,iz,jz,iqx) += biy*bjy*grad_A(2,2,iqy,iz,jz,iqx);
+                              }
+                           }
+                        }
+                        //MFEM_UNROLL(2)
+                        for (int jy=0; jy<2; ++jy)
+                        {
+                           //MFEM_UNROLL(2)
+                           for (int jx=0; jx<2; ++jx)
+                           {
+                              //MFEM_UNROLL(2)
+                              for (int iy=0; iy<2; ++iy)
+                              {
+                                 //MFEM_UNROLL(2)
+                                 for (int ix=0; ix<2; ++ix)
+                                 {
+                                    const double bix = (ix == iqx) ? 1.0 : 0.0;
+                                    const double gix = (ix == 0) ? -1.0 : 1.0;
+
+                                    const double bjx = (jx == iqx) ? 1.0 : 0.0;
+                                    const double gjx = (jx == 0) ? -1.0 : 1.0;
+
+                                    int ii_loc = ix + 2*iy + 4*iz;
+                                    int jj_loc = jx + 2*jy + 4*jz;
+
+                                    // Only store the lower-triangular part of
+                                    // the matrix (by symmetry).
+                                    if (jj_loc > ii_loc) { continue; }
+
+                                    double val = 0.0;
+                                    val += gix*gjx*grad_B(0,0,iy,jy,iz,jz,iqx);
+                                    val += bix*gjx*grad_B(1,0,iy,jy,iz,jz,iqx);
+                                    val += bix*gjx*grad_B(2,0,iy,jy,iz,jz,iqx);
+                                    val += gix*bjx*grad_B(0,1,iy,jy,iz,jz,iqx);
+                                    val += bix*bjx*grad_B(1,1,iy,jy,iz,jz,iqx);
+                                    val += bix*bjx*grad_B(2,1,iy,jy,iz,jz,iqx);
+                                    val += gix*bjx*grad_B(0,2,iy,jy,iz,jz,iqx);
+                                    val += bix*bjx*grad_B(2,2,iy,jy,iz,jz,iqx);
+                                    val += bix*bjx*grad_B(1,2,iy,jy,iz,jz,iqx);
+
+                                    local_mat(ii_loc, jj_loc) += val;
+                                 }
+                              }
+                           }
+                        }
+                     }
+                  }
+               }
+               // Assemble the local matrix into the macro-element sparse matrix
+               // in a format similar to coordinate format. The (I,J) arrays
+               // are implicit (not stored explicitly).
+               //MFEM_UNROLL(8)
+               for (int ii_loc=0; ii_loc<8; ++ii_loc)
+               {
+                  const int ix = ii_loc%2;
+                  const int iy = (ii_loc/2)%2;
+                  const int iz = ii_loc/2/2;
+
+                  for (int jj_loc=0; jj_loc<8; ++jj_loc)
+                  {
+                     const int jx = jj_loc%2;
+                     const int jy = (jj_loc/2)%2;
+                     const int jz = jj_loc/2/2;
+                     const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
+
+                     if (jj_loc <= ii_loc)
+                     {
+                        AtomicAdd(V(jj_off, ix+kx, iy+ky, iz+kz), local_mat(ii_loc, jj_loc));
+                     }
+                     else
+                     {
+                        AtomicAdd(V(jj_off, ix+kx, iy+ky, iz+kz), local_mat(jj_loc, ii_loc));
+                     }
+                  }
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Place the macro-element sparse matrix into the global sparse matrix.
+      MFEM_FOREACH_THREAD(iz,z,nd1d)
+      {
+         MFEM_FOREACH_THREAD(iy,y,nd1d)
+         {
+            MFEM_FOREACH_THREAD(ix,x,nd1d)
+            {
+               double col_ptr[nnz_per_row]; // 27
+
+               const int ii_el = ix + nd1d*(iy + nd1d*iz);
+               const int ii = el_dof_lex(ii_el, iel_ho);
+
+               // Set column pointer to avoid searching in the row
+               for (int j = I[ii], end = I[ii+1]; j < end; j++)
+               {
+                  const int jj = J[j];
+                  int jj_el = -1;
+                  for (int k = K[jj], k_end = K[jj+1]; k < k_end; k += 2)
+                  {
+                     if (dof_glob2loc[k] == iel_ho)
+                     {
+                        jj_el = dof_glob2loc[k+1];
+                        break;
+                     }
+                  }
+                  if (jj_el < 0) { continue; }
+                  const int jx = jj_el%nd1d;
+                  const int jy = (jj_el/nd1d)%nd1d;
+                  const int jz = jj_el/nd1d/nd1d;
+                  const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
+                  col_ptr[jj_off] = j;
+               }
+
+               const int jx_begin = (ix > 0) ? ix - 1 : 0;
+               const int jx_end = (ix < order) ? ix + 1 : order;
+
+               const int jy_begin = (iy > 0) ? iy - 1 : 0;
+               const int jy_end = (iy < order) ? iy + 1 : order;
+
+               const int jz_begin = (iz > 0) ? iz - 1 : 0;
+               const int jz_end = (iz < order) ? iz + 1 : order;
+
+               for (int jz=jz_begin; jz<=jz_end; ++jz)
+               {
+                  for (int jy=jy_begin; jy<=jy_end; ++jy)
+                  {
+                     for (int jx=jx_begin; jx<=jx_end; ++jx)
+                     {
+                        const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
+                        const double Vji = V(jj_off, ix, iy, iz);
+                        const int col_ptr_jj = col_ptr[jj_off];
+                        if ((ix == 0 && jx == 0) || (ix == order && jx == order) ||
+                            (iy == 0 && jy == 0) || (iy == order && jy == order) ||
+                            (iz == 0 && jz == 0) || (iz == order && jz == order))
+                        {
+                           AtomicAdd(A[col_ptr_jj], Vji);
+                        }
+                        else
+                        {
+                           A[col_ptr_jj] += Vji;
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
+#define LOR_KERNEL_INSTANCE(order,use_smem) \
+template void Assemble3DBatchedLOR<order,use_smem>\
+    (const Array<int> &,const Array<int> &, const Array<int> &,\
+     Mesh &, SparseMatrix &)
+
+LOR_KERNEL_INSTANCE(1,true);
+LOR_KERNEL_INSTANCE(2,true);
+LOR_KERNEL_INSTANCE(3,true);
+LOR_KERNEL_INSTANCE(4,true);
+LOR_KERNEL_INSTANCE(5,true);
+LOR_KERNEL_INSTANCE(6,false);/*
+LOR_KERNEL_INSTANCE(7,false);
+LOR_KERNEL_INSTANCE(8,false);
+LOR_KERNEL_INSTANCE(9,false);
+LOR_KERNEL_INSTANCE(10,false);
+LOR_KERNEL_INSTANCE(11,false);
+LOR_KERNEL_INSTANCE(12,false);
+LOR_KERNEL_INSTANCE(13,false);
+LOR_KERNEL_INSTANCE(14,false);
+LOR_KERNEL_INSTANCE(15,false);
+LOR_KERNEL_INSTANCE(16,false);*/
+
+} // namespace mfem

fem/lor_assembly_ni3.cpp

@@ -0,0 +1,128 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "../general/forall.hpp"
+
+#define MFEM_DEBUG_COLOR 227
+#include "../general/debug.hpp"
+
+#define MFEM_NVTX_COLOR MediumVioletRed
+#include "../general/nvtx.hpp"
+
+namespace mfem
+{
+
+#define M1D 8
+
+template<int D1D, int Q1D>
+void NodalInterpolation3D(const int NE,
+                          const Vector& localL, Vector& localH,
+                          const Array<double> &B)
+{
+   MFEM_NVTX;
+   dbg("D1D:%d Q1D:%d", D1D, Q1D);
+
+   static constexpr int VDIM = 3;
+
+   const auto x_ = Reshape(localL.Read(), D1D, D1D, D1D, VDIM, NE);
+   const auto B_ = Reshape(B.Read(), Q1D, D1D);
+
+   auto y_ = Reshape(localH.Write(), VDIM, Q1D, Q1D, Q1D, NE);
+
+   {
+      NVTX("localH = 0.0");
+      localH = 0.0;
+   }
+
+   {
+      NVTX("NodalInterpolation3D Kernel");
+      MFEM_FORALL(e, NE,
+      {
+         for (int vd = 0; vd < VDIM; ++vd)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               double sol_xy[M1D][M1D];
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     sol_xy[qy][qx] = 0.0;
+                  }
+               }
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  double sol_x[M1D];
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     sol_x[qx] = 0;
+                  }
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     const double s = x_(dx, dy, dz, vd, e);
+                     for (int qx = 0; qx < Q1D; ++qx)
+                     {
+                        sol_x[qx] += B_(qx, dx) * s;
+                     }
+                  }
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     const double wy = B_(qy, dy);
+                     for (int qx = 0; qx < Q1D; ++qx)
+                     {
+                        sol_xy[qy][qx] += wy * sol_x[qx];
+                     }
+                  }
+               }
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  const double wz = B_(qz, dz);
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     for (int qx = 0; qx < Q1D; ++qx)
+                     {
+                        y_(vd, qx, qy, qz, e) += wz * sol_xy[qy][qx];
+                     }
+                  }
+               }
+            }
+         }
+      });
+   }
+}
+
+#define NODAL_INTERP_3D_INSTANCE(D1D,Q1D) \
+template void NodalInterpolation3D<D1D,Q1D>\
+    (const int, const Vector&, Vector&,const Array<double>&)
+
+NODAL_INTERP_3D_INSTANCE(2,2);
+NODAL_INTERP_3D_INSTANCE(2,3);
+NODAL_INTERP_3D_INSTANCE(2,4);
+NODAL_INTERP_3D_INSTANCE(2,5);
+NODAL_INTERP_3D_INSTANCE(2,6);
+NODAL_INTERP_3D_INSTANCE(2,7);
+
+NODAL_INTERP_3D_INSTANCE(4,2);
+NODAL_INTERP_3D_INSTANCE(4,3);
+NODAL_INTERP_3D_INSTANCE(4,4);
+NODAL_INTERP_3D_INSTANCE(4,5);
+NODAL_INTERP_3D_INSTANCE(4,6);
+NODAL_INTERP_3D_INSTANCE(4,7);
+
+NODAL_INTERP_3D_INSTANCE(6,2);
+NODAL_INTERP_3D_INSTANCE(6,3);
+NODAL_INTERP_3D_INSTANCE(6,4);
+NODAL_INTERP_3D_INSTANCE(6,5);
+NODAL_INTERP_3D_INSTANCE(6,6);
+NODAL_INTERP_3D_INSTANCE(6,7);
+
+} // namespace mfem

fem/nonlininteg.hpp

@@ -21,6 +21,14 @@
 namespace mfem
 {
 
+// E or L I/O vectors
+enum class ActionType
+{
+   E2E,   ///< E => E kernels
+   L2E,   ///< L => E kernels
+   L2L    ///< L => L kernels
+};
+
 /** @brief This class is used to express the local action of a general nonlinear
     finite element operator. In addition it may provide the capability to
     assemble the local gradient operator and to compute the local energy. */
@@ -32,7 +40,9 @@ protected:
    // CEED extension
    ceed::Operator* ceedOp;
 
-   MemoryType pa_mt = MemoryType::DEFAULT;
+   MemoryType memory_type = MemoryType::DEFAULT;
+
+   ActionType action_type = ActionType::E2E;
 
    NonlinearFormIntegrator(const IntegrationRule *ir = NULL)
       : IntRule(ir), ceedOp(NULL) { }
@@ -46,8 +56,11 @@ public:
    void SetIntegrationRule(const IntegrationRule &ir) { SetIntRule(&ir); }
 
    /// Set the memory type used for GeometricFactors and other large allocations
-   /// in PA extensions.
-   void SetPAMemoryType(MemoryType mt) { pa_mt = mt; }
+   void SetMemoryType(MemoryType mt) { memory_type = mt; }
+   MFEM_DEPRECATED void SetPAMemoryType(MemoryType mt) { memory_type = mt; }
+
+   /// Set the kernel type used in PA extensions.
+   ActionType GetActionType() const { return action_type; }
 
    /// Get the integration rule of the integrator (possibly NULL).
    const IntegrationRule *GetIntegrationRule() const { return IntRule; }

fem/pfespace.cpp

@@ -23,8 +23,6 @@
 #include <limits>
 #include <list>
 
-using namespace std;
-
 namespace mfem
 {
 
@@ -1012,36 +1010,9 @@ void ParFiniteElementSpace::GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
 {
    FiniteElementSpace::GetEssentialVDofs(bdr_attr_is_ess, ess_dofs, component);
 
-   if (Conforming())
-   {
-      // Make sure that processors without boundary elements mark
-      // their boundary dofs (if they have any).
-      Synchronize(ess_dofs);
-   }
-}
-
-void ParFiniteElementSpace::GetEssentialVDofs(EntitySets::EntityType type,
-                                              int set_index,
-                                              Array<int> &ess_dofs,
-                                              int component) const
-{
-   FiniteElementSpace::GetEssentialVDofs(type, set_index, ess_dofs, component);
-
-   if (Conforming())
-   {
-      // Make sure that processors without boundary elements mark
-      // their boundary dofs (if they have any).
-      Synchronize(ess_dofs);
-   }
-}
-
-void ParFiniteElementSpace::GetEssentialVDofs(EntitySets::EntityType type,
-                                              const string & set_name,
-                                              Array<int> &ess_vdofs,
-                                              int component) const
-{
-   GetEssentialVDofs(type, pmesh->ent_sets->GetSetIndex(type, set_name),
-                     ess_vdofs, component);
+   // Make sure that processors without boundary elements mark
+   // their boundary dofs (if they have any).
+   Synchronize(ess_dofs);
 }
 
 void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
@@ -1067,33 +1038,13 @@ void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
    {
       if (bool(ted[i]) != bool(true_ess_dofs2[i])) { counter++; }
    }
-   MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter);
+   MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter
+               << ", rank = " << MyRank);
 #endif
 
    MarkerToList(true_ess_dofs, ess_tdof_list);
 }
 
-void ParFiniteElementSpace::GetEssentialTrueDofs(EntitySets::EntityType type,
-                                                 int set_index,
-                                                 Array<int> &ess_tdof_list,
-                                                 int component)
-{
-   Array<int> ess_dofs, true_ess_dofs;
-
-   GetEssentialVDofs(type, set_index, ess_dofs, component);
-   GetRestrictionMatrix()->BooleanMult(ess_dofs, true_ess_dofs);
-   MarkerToList(true_ess_dofs, ess_tdof_list);
-}
-
-void ParFiniteElementSpace::GetEssentialTrueDofs(EntitySets::EntityType type,
-                                                 const string & set_name,
-                                                 Array<int> &ess_tdof_list,
-                                                 int component)
-{
-   GetEssentialTrueDofs(type, pmesh->ent_sets->GetSetIndex(type, set_name),
-                        ess_tdof_list, component);
-}
-
 int ParFiniteElementSpace::GetLocalTDofNumber(int ldof) const
 {
    if (Nonconforming())

fem/pfespace.hpp

@@ -355,38 +355,12 @@ public:
                                   Array<int> &ess_dofs,
                                   int component = -1) const;
 
-   /** Mark degrees of freedom associated with the entity set with the
-       specified entity type and set index. */
-   virtual void GetEssentialVDofs(EntitySets::EntityType type, int set_index,
-                                  Array<int> &ess_vdofs,
-                                  int component = -1) const;
-
-   /** Mark degrees of freedom associated with the entity set with the
-       specified entity type and set index. */
-   virtual void GetEssentialVDofs(EntitySets::EntityType type,
-                                  const std::string & set_name,
-                                  Array<int> &ess_vdofs,
-                                  int component = -1) const;
-
    /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
        boundary attributes marked in the array bdr_attr_is_ess. */
    virtual void GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
                                      Array<int> &ess_tdof_list,
                                      int component = -1);
 
-   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
-       entity set specified by the given entity type and set index. */
-   virtual void GetEssentialTrueDofs(EntitySets::EntityType type, int set_index,
-                                     Array<int> &ess_tdof_list,
-                                     int component = -1);
-
-   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
-       entity set specified by the given entity type and set name. */
-   virtual void GetEssentialTrueDofs(EntitySets::EntityType type,
-                                     const std::string & set_name,
-                                     Array<int> &ess_tdof_list,
-                                     int component = -1);
-
    /** If the given ldof is owned by the current processor, return its local
        tdof number, otherwise return -1 */
    int GetLocalTDofNumber(int ldof) const;

fem/qinterp/eval.hpp

@@ -65,13 +65,13 @@ static void Values2D(const int NE,
       MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
       MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
       ConstDeviceMatrix B(sB, D1D,Q1D);
       DeviceMatrix DD(sm0[tidz], MD1, MD1);
       DeviceMatrix DQ(sm1[tidz], MD1, MQ1);
       DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
       for (int c = 0; c < VDIM; c++)
       {
          kernels::internal::LoadX(e,D1D,c,x,DD);
@@ -126,14 +126,14 @@ static void Values3D(const int NE,
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
       MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
       ConstDeviceMatrix B(sB, D1D,Q1D);
       DeviceCube DDD(sm0, MD1,MD1,MD1);
       DeviceCube DDQ(sm1, MD1,MD1,MQ1);
       DeviceCube DQQ(sm0, MD1,MQ1,MQ1);
       DeviceCube QQQ(sm1, MQ1,MQ1,MQ1);
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
       for (int c = 0; c < VDIM; c++)
       {
          kernels::internal::LoadX(e,D1D,c,x,DDD);

fem/restriction.hpp

@@ -74,6 +74,23 @@ public:
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
        pattern given by this ElementRestriction, and the values of ea_data. */
    void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
+
+   /**
+    * @brief GatherMap
+    * @return the mapping from L dofs to E dofs.
+    */
+   const Array<int> &GatherMap() const { return gatherMap; }
+
+   /**
+    * @brief Indices
+    * @return the local to global indices.
+    */
+   const Array<int> &Indices() const { return indices; }
+   /**
+    * @brief Offsets
+    * @return the offsets for each global dof.
+    */
+   const Array<int> &Offsets() const { return offsets; }
 };
 
 /// Operator that converts L2 FiniteElementSpace L-vectors to E-vectors.

fem/tbilininteg.hpp

@@ -384,9 +384,9 @@ struct TDiffusionKernel<2,2,complex_t>
          const complex_t w_det_J = Q.get(q,i,k) / (J11 * J22 - J21 * J12);
          internal::MatrixOps<2,2>::Symm<Symm>::Set(
             A.layout.ind1(i), A,
-            + w_det_J * (J12*J12 + J22*J22), // (1,1)
+            w_det_J * (J12*J12 + J22*J22), // (1,1)
             - w_det_J * (J11*J12 + J21*J22), // (2,1)
-            + w_det_J * (J11*J11 + J21*J21)  // (2,2)
+            w_det_J * (J11*J11 + J21*J21)  // (2,2)
          );
       }
    }

fem/tintrules.hpp

@@ -19,6 +19,8 @@
 namespace mfem
 {
 
+static IntegrationRules LobattoIntRules(0, Quadrature1D::GaussLobatto);
+
 // Templated integration rules, cf. intrules.?pp
 
 template <Geometry::Type G, int Q, int Order, typename real_t>
@@ -52,7 +54,7 @@ public:
 
    static const IntegrationRule &GetIntRule()
    {
-      return IntRules.Get(geom, order);
+      return LobattoIntRules.Get(geom, order);
    }
 
    // Multi-component weight assignment. qpt_layout_t must be (qpts x n1 x ...)
@@ -201,10 +203,10 @@ public:
 
 template <int Dim, int Q, typename real_t>
 class GaussIntegrationRule
-   : public TProductIntegrationRule<Dim, Q, 2*Q-1, real_t>
+   : public TProductIntegrationRule<Dim, Q, 2*Q-3, real_t>
 {
 public:
-   typedef TProductIntegrationRule<Dim,Q,2*Q-1,real_t> base_class;
+   typedef TProductIntegrationRule<Dim,Q,2*Q-3,real_t> base_class;
 
    using base_class::geom;
    using base_class::order;
@@ -226,11 +228,11 @@ public:
 
    static const IntegrationRule &Get1DIntRule()
    {
-      return IntRules.Get(Geometry::SEGMENT, order);
+      return LobattoIntRules.Get(Geometry::SEGMENT, order);
    }
    static const IntegrationRule &GetIntRule()
    {
-      return IntRules.Get(geom, order);
+      return LobattoIntRules.Get(geom, order);
    }
 };
 
@@ -239,15 +241,15 @@ class TIntegrationRule;
 
 template <int Order, typename real_t>
 class TIntegrationRule<Geometry::SEGMENT, Order, real_t>
-   : public GaussIntegrationRule<1, Order/2+1, real_t> { };
+   : public GaussIntegrationRule<1, Order/2+2, real_t> { };
 
 template <int Order, typename real_t>
 class TIntegrationRule<Geometry::SQUARE, Order, real_t>
-   : public GaussIntegrationRule<2, Order/2+1, real_t> { };
+   : public GaussIntegrationRule<2, Order/2+2, real_t> { };
 
 template <int Order, typename real_t>
 class TIntegrationRule<Geometry::CUBE, Order, real_t>
-   : public GaussIntegrationRule<3, Order/2+1, real_t> { };
+   : public GaussIntegrationRule<3, Order/2+2, real_t> { };
 
 // Triangle integration rules (based on intrules.cpp)
 // These specializations define the number of quadrature points for each rule as

fem/tmop.hpp

@@ -1133,7 +1133,7 @@ protected:
 
    // Evaluation of the discrete target specification on different meshes.
    // Owned.
-   AdaptivityEvaluator *adapt_eval;
+   AdaptivityEvaluator *adapt_lim_eval;
 
    void SetDiscreteTargetBase(const GridFunction &tspec_);
    void SetTspecAtIndex(int idx, const GridFunction &tspec_);
@@ -1156,7 +1156,7 @@ public:
 #endif
         amr_el(-1), lim_min_size(-0.1),
         good_tspec(false), good_tspec_grad(false), good_tspec_hess(false),
-        adapt_eval(NULL) { }
+        adapt_lim_eval(NULL) { }
 
    virtual ~DiscreteAdaptTC();
 
@@ -1232,8 +1232,8 @@ public:
 
    void SetAdaptivityEvaluator(AdaptivityEvaluator *ae)
    {
-      if (adapt_eval) { delete adapt_eval; }
-      adapt_eval = ae;
+      if (adapt_lim_eval) { delete adapt_lim_eval; }
+      adapt_lim_eval = ae;
    }
 
    const Vector &GetTspecPert1H()   { return tspec_pert1h; }
@@ -1315,15 +1315,15 @@ protected:
    int integ_order;
 
    // Weight Coefficient multiplying the quality metric term.
-   Coefficient *coeff1; // not owned, if NULL -> coeff1 is 1.
+   Coefficient *metric_coeff; // not owned, if NULL -> metric_coeff is 1.
    // Normalization factor for the metric term.
    double metric_normal;
 
    // Nodes and weight Coefficient used for "limiting" the TMOP_Integrator.
    // These are both NULL when there is no limiting.
-   // The class doesn't own nodes0 and coeff0.
-   const GridFunction *nodes0;
-   Coefficient *coeff0;
+   // The class doesn't own lim_nodes0 and lim_coeff.
+   const GridFunction *lim_nodes0;
+   Coefficient *lim_coeff;
    // Limiting reference distance. Not owned.
    const GridFunction *lim_dist;
    // Limiting function. Owned.
@@ -1332,20 +1332,21 @@ protected:
    double lim_normal;
 
    // Adaptive limiting.
-   const GridFunction *zeta_0;       // Not owned.
+   const GridFunction *adapt_lim_gf0;    // Not owned.
 #ifdef MFEM_USE_MPI
-   const ParGridFunction *pzeta_0;
+   const ParGridFunction *adapt_lim_pgf0;
 #endif
-   GridFunction *zeta;               // Owned. Updated by adapt_eval.
-   Coefficient *coeff_zeta;          // Not owned.
-   AdaptivityEvaluator *adapt_eval;  // Not owned.
+   GridFunction *adapt_lim_gf;           // Owned. Updated by adapt_lim_eval.
+   Coefficient *adapt_lim_coeff;         // Not owned.
+   AdaptivityEvaluator *adapt_lim_eval;  // Not owned.
 
    // Surface fitting.
-   GridFunction *sigma, *sigma_bar;  // Owned. Updated by sigma_eval.
-   const Array<bool> *sigma_marker;  // Not owned.
-   Coefficient *coeff_sigma;         // Not owned.
-   AdaptivityEvaluator *sigma_eval;  // Not owned.
-   double sigma_normal;
+   GridFunction *surf_fit_gf,
+                *surf_fit_gf_bar;       // Owned, Updated by surf_fit_eval.
+   const Array<bool> *surf_fit_marker;  // Not owned.
+   Coefficient *surf_fit_coeff;         // Not owned.
+   AdaptivityEvaluator *surf_fit_eval;  // Not owned.
+   double surf_fit_normal;
 
    DiscreteAdaptTC *discr_tc;
 
@@ -1416,7 +1417,7 @@ protected:
 
    void ComputeNormalizationEnergies(const GridFunction &x,
                                      double &metric_energy, double &lim_energy,
-                                     double &sigma_energy);
+                                     double &surf_fit_gf_energy);
 
    void AssembleElementVectorExact(const FiniteElement &el,
                                    ElementTransformation &T,
@@ -1471,7 +1472,7 @@ protected:
 
    void DisableLimiting()
    {
-      nodes0 = NULL; coeff0 = NULL; lim_dist = NULL;
+      lim_nodes0 = NULL; lim_coeff = NULL; lim_dist = NULL;
       delete lim_func; lim_func = NULL;
    }
 
@@ -1531,12 +1532,14 @@ public:
    TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc,
                    TMOP_QualityMetric *hm)
       : h_metric(hm), metric(m), targetC(tc), IntegRules(NULL),
-        integ_order(-1), coeff1(NULL), metric_normal(1.0),
-        nodes0(NULL), coeff0(NULL),
+        integ_order(-1), metric_coeff(NULL), metric_normal(1.0),
+        lim_nodes0(NULL), lim_coeff(NULL),
         lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
-        zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
-        sigma(NULL), sigma_bar(NULL), sigma_marker(NULL), coeff_sigma(NULL),
-        sigma_eval(NULL), sigma_normal(1.0),
+        adapt_lim_gf0(NULL), adapt_lim_gf(NULL), adapt_lim_coeff(NULL),
+        adapt_lim_eval(NULL),
+        surf_fit_gf(NULL), surf_fit_gf_bar(NULL), surf_fit_marker(NULL),
+        surf_fit_coeff(NULL),
+        surf_fit_eval(NULL), surf_fit_normal(1.0),
         discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
    { PA.enabled = false; }
@@ -1564,7 +1567,7 @@ public:
 
        Note that the Coefficient is evaluated in the physical configuration and
        not in the target configuration which may be undefined. */
-   void SetCoefficient(Coefficient &w1) { coeff1 = &w1; }
+   void SetCoefficient(Coefficient &w1) { metric_coeff = &w1; }
 
    /** @brief Limiting of the mesh displacements (general version).
 
@@ -1631,7 +1634,7 @@ public:
    void GetSurfaceFittingErrors(double &err_avg, double &err_max);
 
    /// Update the original/reference nodes used for limiting.
-   void SetLimitingNodes(const GridFunction &n0) { nodes0 = &n0; }
+   void SetLimitingNodes(const GridFunction &n0) { lim_nodes0 = &n0; }
 
    /** @brief Computes the integral of W(Jacobian(Trt)) over a target zone.
        @param[in] el     Type of FiniteElement.

fem/tmop/tmop_pa.cpp

@@ -38,23 +38,23 @@ void TMOP_Integrator::AssembleGradPA(const Vector &xe,
    if (PA.dim == 2)
    {
       AssembleGradPA_2D(xe);
-      if (coeff0) { AssembleGradPA_C0_2D(xe); }
+      if (lim_coeff) { AssembleGradPA_C0_2D(xe); }
    }
 
    if (PA.dim == 3)
    {
       AssembleGradPA_3D(xe);
-      if (coeff0) { AssembleGradPA_C0_3D(xe); }
+      if (lim_coeff) { AssembleGradPA_C0_3D(xe); }
    }
 }
 
 void TMOP_Integrator::AssemblePA_Limiting()
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : memory_type;
    // Return immediately if limiting is not enabled
-   if (coeff0 == nullptr) { return; }
-   MFEM_VERIFY(nodes0, "internal error");
+   if (lim_coeff == nullptr) { return; }
+   MFEM_VERIFY(lim_nodes0, "internal error");
 
    MFEM_VERIFY(PA.enabled, "AssemblePA_Limiting but PA is not enabled!");
    MFEM_VERIFY(lim_func, "No TMOP_LimiterFunction specification!")
@@ -68,14 +68,14 @@ void TMOP_Integrator::AssemblePA_Limiting()
 
    const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
 
-   // H0 for coeff0, (dim x dim) Q-vector
+   // H0 for lim_coeff, (dim x dim) Q-vector
    PA.H0.UseDevice(true);
    PA.H0.SetSize(PA.dim * PA.dim * PA.nq * NE, mt);
 
-   // coeff0 -> PA.C0 (Q-vector)
+   // lim_coeff -> PA.C0 (Q-vector)
    PA.C0.UseDevice(true);
    if (ConstantCoefficient* cQ =
-          dynamic_cast<ConstantCoefficient*>(coeff0))
+          dynamic_cast<ConstantCoefficient*>(lim_coeff))
    {
       PA.C0.SetSize(1, Device::GetMemoryType());
       PA.C0.HostWrite();
@@ -90,17 +90,17 @@ void TMOP_Integrator::AssemblePA_Limiting()
          ElementTransformation& T = *fes->GetElementTransformation(e);
          for (int q = 0; q < ir.GetNPoints(); ++q)
          {
-            C0(q,e) = coeff0->Eval(T, ir.IntPoint(q));
+            C0(q,e) = lim_coeff->Eval(T, ir.IntPoint(q));
          }
       }
    }
 
-   // nodes0 -> PA.X0 (E-vector)
-   MFEM_VERIFY(nodes0->FESpace() == fes, "");
+   // lim_nodes0 -> PA.X0 (E-vector)
+   MFEM_VERIFY(lim_nodes0->FESpace() == fes, "");
    const Operator *n0_R = fes->GetElementRestriction(ordering);
    PA.X0.SetSize(n0_R->Height(), Device::GetMemoryType());
    PA.X0.UseDevice(true);
-   n0_R->Mult(*nodes0, PA.X0);
+   n0_R->Mult(*lim_nodes0, PA.X0);
 
    // Limiting distances: lim_dist -> PA.LD (E-vector)
    // TODO: remove the hack for the case lim_dist == NULL.
@@ -177,8 +177,8 @@ void TMOP_Integrator::ComputeAllElementTargets(const Vector &xe) const
 
 void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : memory_type;
    PA.enabled = true;
    PA.fes = &fes;
    Mesh *mesh = fes.GetMesh();
@@ -217,8 +217,8 @@ void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
    PA.Jtr_needs_update = true;
    PA.Jtr_debug_grad = false;
 
-   // Limiting: coeff0 -> PA.C0, nodes0 -> PA.X0, lim_dist -> PA.LD, PA.H0
-   if (coeff0) { AssemblePA_Limiting(); }
+   // Limiting: lim_coeff -> PA.C0, lim_nodes0 -> PA.X0, lim_dist -> PA.LD, PA.H0
+   if (lim_coeff) { AssemblePA_Limiting(); }
 }
 
 void TMOP_Integrator::AssembleGradDiagonalPA(Vector &de) const
@@ -236,13 +236,13 @@ void TMOP_Integrator::AssembleGradDiagonalPA(Vector &de) const
    if (PA.dim == 2)
    {
       AssembleDiagonalPA_2D(de);
-      if (coeff0) { AssembleDiagonalPA_C0_2D(de); }
+      if (lim_coeff) { AssembleDiagonalPA_C0_2D(de); }
    }
 
    if (PA.dim == 3)
    {
       AssembleDiagonalPA_3D(de);
-      if (coeff0) { AssembleDiagonalPA_C0_3D(de); }
+      if (lim_coeff) { AssembleDiagonalPA_C0_3D(de); }
    }
 }
 
@@ -258,13 +258,13 @@ void TMOP_Integrator::AddMultPA(const Vector &xe, Vector &ye) const
    if (PA.dim == 2)
    {
       AddMultPA_2D(xe,ye);
-      if (coeff0) { AddMultPA_C0_2D(xe,ye); }
+      if (lim_coeff) { AddMultPA_C0_2D(xe,ye); }
    }
 
    if (PA.dim == 3)
    {
       AddMultPA_3D(xe,ye);
-      if (coeff0) { AddMultPA_C0_3D(xe,ye); }
+      if (lim_coeff) { AddMultPA_C0_3D(xe,ye); }
    }
 }
 
@@ -283,13 +283,13 @@ void TMOP_Integrator::AddMultGradPA(const Vector &re, Vector &ce) const
    if (PA.dim == 2)
    {
       AddMultGradPA_2D(re,ce);
-      if (coeff0) { AddMultGradPA_C0_2D(re,ce); }
+      if (lim_coeff) { AddMultGradPA_C0_2D(re,ce); }
    }
 
    if (PA.dim == 3)
    {
       AddMultGradPA_3D(re,ce);
-      if (coeff0) { AddMultGradPA_C0_3D(re,ce); }
+      if (lim_coeff) { AddMultGradPA_C0_3D(re,ce); }
    }
 }
 
@@ -307,13 +307,13 @@ double TMOP_Integrator::GetLocalStateEnergyPA(const Vector &xe) const
    if (PA.dim == 2)
    {
       energy = GetLocalStateEnergyPA_2D(xe);
-      if (coeff0) { energy += GetLocalStateEnergyPA_C0_2D(xe); }
+      if (lim_coeff) { energy += GetLocalStateEnergyPA_C0_2D(xe); }
    }
 
    if (PA.dim == 3)
    {
       energy = GetLocalStateEnergyPA_3D(xe);
-      if (coeff0) { energy += GetLocalStateEnergyPA_C0_3D(xe); }
+      if (lim_coeff) { energy += GetLocalStateEnergyPA_C0_3D(xe); }
    }
 
    return energy;

fem/tmop/tmop_pa.hpp

@@ -181,9 +181,9 @@ template<int T_D1D, int T_Q1D, int T_MAX> return_t kernel(__VA_ARGS__)
 if (K##kernel.Find(id)) { return K##kernel.At(id)(__VA_ARGS__,0,0); }\
 else {\
    constexpr int T_MAX = 4;\
-   const int D1D = (id>>4)&0xF, Q1D = id&0xF;\
-   MFEM_VERIFY(D1D <= MAX_D1D && Q1D <= MAX_Q1D, "Max size error!");\
-   return kernel<0,0,T_MAX>(__VA_ARGS__,D1D,Q1D); }
+   const int d1d = (id>>4)&0xF, q1d = id&0xF;\
+   MFEM_VERIFY(d1d <= MAX_D1D && q1d <= MAX_Q1D, "Max size error!");\
+   return kernel<0,0,T_MAX>(__VA_ARGS__,d1d,q1d); }
 
 } // namespace kernels
 

fem/tmop/tmop_pa_da3.cpp

@@ -55,6 +55,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, DatcSize,
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
       MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
       ConstDeviceMatrix B(sB, D1D, Q1D);
       DeviceCube DDD(sm0, MD1,MD1,MD1);
       DeviceCube DDQ(sm1, MD1,MD1,MQ1);
@@ -88,7 +90,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, DatcSize,
       }
       min = min_size[0];
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
       kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
       kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);

fem/tmop/tmop_pa_h2d.cpp

@@ -97,9 +97,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_2D,
                   const double *Jtr = &J(0,0,qx,qy,e);
 
                   // Jrt = Jtr^{-1}
-                  double j[4];
-                  ConstDeviceMatrix Jrt(j,2,2);
-                  kernels::CalcInverse<2>(Jtr, j);
+                  double jrt_data[4];
+                  ConstDeviceMatrix Jrt(jrt_data,2,2);
+                  kernels::CalcInverse<2>(Jtr, jrt_data);
 
                   const double gg = G(qy,dy) * G(qy,dy);
                   const double gb = G(qy,dy) * B(qy,dy);

fem/tmop/tmop_pa_h2m_c0.cpp

@@ -68,8 +68,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_2D,
             double Xh[2];
             kernels::internal::PullEval<MQ1,NBZ>(Q1D,qx,qy,QQ,Xh);
 
-            double B[4];
-            DeviceMatrix H(B,2,2);
+            double H_data[4];
+            DeviceMatrix H(H_data,2,2);
             for (int i = 0; i < DIM; i++)
             {
                for (int j = 0; j < DIM; j++)
@@ -78,9 +78,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_2D,
                }
             }
 
-            // p2 = B . Xh
+            // p2 = H . Xh
             double p2[2];
-            kernels::Mult(2,2,B,Xh,p2);
+            kernels::Mult(2,2,H_data,Xh,p2);
             kernels::internal::PushEval<MQ1,NBZ>(Q1D,qx,qy,p2,QQ);
          }
       }

fem/tmop/tmop_pa_h3m_c0.cpp

@@ -70,8 +70,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_3D,
                double Xh[3];
                kernels::internal::PullEval<MQ1>(Q1D,qx,qy,qz,QQQ,Xh);
 
-               double B[9];
-               DeviceMatrix H(B,3,3);
+               double H_data[9];
+               DeviceMatrix H(H_data,3,3);
                for (int i = 0; i < DIM; i++)
                {
                   for (int j = 0; j < DIM; j++)
@@ -80,9 +80,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_3D,
                   }
                }
 
-               // p2 = B . Xh
+               // p2 = H . Xh
                double p2[3];
-               kernels::Mult(3,3,B,Xh,p2);
+               kernels::Mult(3,3,H_data,Xh,p2);
                kernels::internal::PushEval<MQ1>(Q1D,qx,qy,qz,p2,QQQ);
             }
          }

fem/tmop/tmop_pa_h3s_c0.cpp

@@ -55,6 +55,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_Kernel_C0_3D,
       constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
 
       MFEM_SHARED double sBLD[MQ1*MD1];
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -66,8 +67,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_Kernel_C0_3D,
 
       kernels::internal::LoadX(e,D1D,LD,DDD);
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
-
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);
       kernels::internal::EvalZ(D1D,Q1D,BLD,DQQ,QQQ);

fem/tmop/tmop_pa_p3_c0.cpp

@@ -62,6 +62,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_3D,
 
       MFEM_SHARED double B[MQ1*MD1];
       MFEM_SHARED double sBLD[MQ1*MD1];
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -86,7 +87,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_3D,
       kernels::internal::LoadX<MD1>(e,D1D,X1,DDD1);
 
       kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
 
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);

fem/tmop/tmop_pa_w3_c0.cpp

@@ -63,6 +63,7 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_3D,
 
       MFEM_SHARED double B[MQ1*MD1];
       MFEM_SHARED double sBLD[MQ1*MD1];
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -87,7 +88,6 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_3D,
       kernels::internal::LoadX<MD1>(e,D1D,X1,DDD1);
 
       kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
 
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);

fem/tmop_amr.cpp

@@ -837,26 +837,26 @@ void TMOPHRSolver::ParUpdate()
 }
 #endif
 
-void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
+void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh_, NonlinearForm *nlf_)
 {
-   const FiniteElementSpace &fes = *mesh->GetNodalFESpace();
+   const FiniteElementSpace &fes = *mesh_->GetNodalFESpace();
 
    // Update Nonlinear form and Set Essential BC
-   nlf->Update();
+   nlf_->Update();
    const int dim = fes.GetFE(0)->GetDim();
    if (move_bnd == false)
    {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      Array<int> ess_bdr(mesh_->bdr_attributes.Max());
       ess_bdr = 1;
-      nlf->SetEssentialBC(ess_bdr);
+      nlf_->SetEssentialBC(ess_bdr);
    }
    else
    {
       const int nd  = fes.GetBE(0)->GetDof();
       int n = 0;
-      for (int i = 0; i < mesh->GetNBE(); i++)
+      for (int i = 0; i < mesh_->GetNBE(); i++)
       {
-         const int attr = mesh->GetBdrElement(i)->GetAttribute();
+         const int attr = mesh_->GetBdrElement(i)->GetAttribute();
          MFEM_VERIFY(!(dim == 2 && attr == 3),
                      "Boundary attribute 3 must be used only for 3D meshes. "
                      "Adjust the attributes (1/2/3/4 for fixed x/y/z/all "
@@ -866,9 +866,9 @@ void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
       }
       Array<int> ess_vdofs(n), vdofs;
       n = 0;
-      for (int i = 0; i < mesh->GetNBE(); i++)
+      for (int i = 0; i < mesh_->GetNBE(); i++)
       {
-         const int attr = mesh->GetBdrElement(i)->GetAttribute();
+         const int attr = mesh_->GetBdrElement(i)->GetAttribute();
          fes.GetBdrElementVDofs(i, vdofs);
          if (attr == 1) // Fix x components.
          {
@@ -891,7 +891,7 @@ void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
             { ess_vdofs[n++] = vdofs[j]; }
          }
       }
-      nlf->SetEssentialVDofs(ess_vdofs);
+      nlf_->SetEssentialVDofs(ess_vdofs);
    }
 }
 

fem/tmop_tools.cpp

@@ -240,7 +240,7 @@ ParAdvectorCGOper::ParAdvectorCGOper(const Vector &x_start,
    ConvectionIntegrator *Kinteg = new ConvectionIntegrator(u_coeff);
    if (al == AssemblyLevel::PARTIAL)
    {
-      Kinteg->SetPAMemoryType(mt);
+      Kinteg->SetMemoryType(mt);
    }
    K.AddDomainIntegrator(Kinteg);
    K.SetAssemblyLevel(al);
@@ -250,7 +250,7 @@ ParAdvectorCGOper::ParAdvectorCGOper(const Vector &x_start,
    MassIntegrator *Minteg = new MassIntegrator;
    if (al == AssemblyLevel::PARTIAL)
    {
-      Minteg->SetPAMemoryType(mt);
+      Minteg->SetMemoryType(mt);
    }
    M.AddDomainIntegrator(Minteg);
    M.SetAssemblyLevel(al);
@@ -444,15 +444,19 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (untangling == false && min_detT_out < 0.0)
       {
          // No untangling, and detJ got negative -- no good.
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
+         if (print_options.iterations)
+         {
+            mfem::out << "Scale = " << scale << " Neg det(J) found.\n";
+         }
          scale *= detJ_factor; continue;
       }
       if (untangling == true && min_detT_out < *min_det_ptr)
       {
          // Untangling, and detJ got even more negative -- no good.
-         if (print_level >= 0)
-         { mfem::out << "Scale = " << scale << " Neg det(J) decreased.\n"; }
+         if (print_options.iterations)
+         {
+            mfem::out << "Scale = " << scale << " Neg det(J) decreased.\n";
+         }
          scale *= detJ_factor; continue;
       }
 
@@ -477,7 +481,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (energy_out > energy_in + 0.2*fabs(energy_in) ||
           std::isnan(energy_out) != 0)
       {
-         if (print_level >= 0)
+         if (print_options.iterations)
          {
             mfem::out << "Scale = " << scale << " Increasing energy: "
                       << energy_in << " --> " << energy_out << '\n';
@@ -492,7 +496,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
       if (norm_out > 1.2*norm_in)
       {
-         if (print_level >= 0)
+         if (print_options.iterations)
          {
             mfem::out << "Scale = " << scale << " Norm increased: "
                       << norm_in << " --> " << norm_out << '\n';
@@ -508,13 +512,15 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (min_detT_out > 0.0)
       {
          *min_det_ptr = 0.0;
-         if (print_level >= 0)
+         if (print_options.summary || print_options.iterations ||
+             print_options.first_and_last)
          { mfem::out << "The mesh has been untangled at the used points!\n"; }
       }
       else { *min_det_ptr = untangle_factor * min_detT_out; }
    }
 
-   if (print_level >= 0)
+   if (print_options.summary || print_options.iterations ||
+       print_options.first_and_last)
    {
       if (untangling)
       {
@@ -568,9 +574,9 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
    if (parallel)
    {
 #ifdef MFEM_USE_MPI
-      const ParNonlinearForm *nlf =
+      const ParNonlinearForm *pnlf =
          dynamic_cast<const ParNonlinearForm *>(oper);
-      const ParFiniteElementSpace *pfesc = nlf->ParFESpace();
+      const ParFiniteElementSpace *pfesc = pnlf->ParFESpace();
       Vector x_loc(pfesc->GetVSize());
       pfesc->GetProlongationMatrix()->Mult(x, x_loc);
       for (int i = 0; i < integs.Size(); i++)

fem/transfer.cpp

@@ -1192,6 +1192,114 @@ void Prolongation2D(const int NE, const int D1D, const int Q1D,
    });
 }
 
+template <int D1D, int Q1D, int NBZ = 1>
+void FastProlongation3D(const int lND,
+                        const int hNQ,
+                        const int NE,
+                        const int *lmap,
+                        const Vector& x,
+                        const int *hmap,
+                        Vector& y,
+                        const Array<double>& B,
+                        const Vector& mask)
+{
+   const auto B_ = Reshape(B.Read(), Q1D, D1D);
+   const auto lMAP = Reshape(lmap, D1D,D1D,D1D, NE);
+   const auto hMAP = Reshape(hmap, Q1D,Q1D,Q1D, NE);
+   const auto X = Reshape(x.Read(), lND);
+   const auto M = Reshape(mask.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), hNQ);
+
+   y = 0.0;
+
+   assert(Q1D<=8);
+
+   MFEM_FORALL_3D(be, (NE+NBZ-1)/NBZ, Q1D, Q1D, NBZ,
+   {
+      double u[Q1D];
+      const int tz = MFEM_THREAD_ID(z);
+      const int e = be * MFEM_THREAD_SIZE(z) + tz;
+
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
+
+      // Load input, B & X interpolation
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            if (tz == 0) { s_B[qx][dy] = B_(qx,dy); }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double Bx = B_(qx,dx);
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  const int gid = lMAP(dx,dy,dz,e);
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  u[dz] += X(idx)* Bx;
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Y interpolation
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Z interpolation & Q-function
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Z interpolation
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
+            }
+
+            // Q-function
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const int gid = hMAP(qx,qy,qz,e);
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               const double value = M(qx, qy, qz, e) * u[qz];
+               AtomicAdd(Y[idx], value);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 void Prolongation3D(const int NE, const int D1D, const int Q1D,
                     const Vector& localL, Vector& localH,
                     const Array<double>& B, const Vector& mask)
@@ -1303,6 +1411,7 @@ void Restriction2D(const int NE, const int D1D, const int Q1D,
       }
    });
 }
+
 void Restriction3D(const int NE, const int D1D, const int Q1D,
                    const Vector& localH, Vector& localL,
                    const Array<double>& Bt, const Vector& mask)
@@ -1364,6 +1473,129 @@ void Restriction3D(const int NE, const int D1D, const int Q1D,
       }
    });
 }
+
+template<int D1D, int Q1D, int NBZ = 1>
+void FastRestriction3D(const int lND,
+                       const int hNQ,
+                       const int NE,
+                       const int *hmap,
+                       const Vector& x,
+                       const int *lmap,
+                       Vector& y,
+                       const Array<double>& Bt,
+                       const Vector& mask)
+{
+   const auto Bt_ = Reshape(Bt.Read(), D1D, Q1D);
+   const auto lMAP = Reshape(lmap, D1D,D1D,D1D, NE);
+   const auto hMAP = Reshape(hmap, Q1D,Q1D,Q1D, NE);
+   const auto M = Reshape(mask.Read(), Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x.Read(), hNQ);
+   auto Y = Reshape(y.ReadWrite(), lND);
+
+   y = 0.0;
+
+   assert(Q1D<=8);
+
+   MFEM_FORALL_3D(be, (NE+NBZ-1)/NBZ, Q1D, Q1D, NBZ,
+   {
+      double u[Q1D];
+      const int tz = MFEM_THREAD_ID(z);
+      const int e = be * MFEM_THREAD_SIZE(z) + tz;
+
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
+
+      // Load B
+      if (tz == 0)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               { s_B[qx][dy] = Bt_(dy,qx); }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Q-function & Zt projection
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Q-function
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const int gid = hMAP(qx,qy,qz,e);
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               s_q[tz][qz][qy][qx] = M(qx, qy, qz, e) * X(idx);
+            }
+
+            // Zt projection
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double ZYX = s_q[tz][qz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Yt projection
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Xt projection & save output
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const int gid = lMAP(dx,dy,dz,e);
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               const double value = u[dx];
+               AtomicAdd(Y[idx], value);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 } // namespace TransferKernels
 
 
@@ -1380,22 +1612,69 @@ void TensorProductPRefinementTransferOperator::Mult(const Vector& x,
       return;
    }
 
-   elem_restrict_lex_l->Mult(x, localL);
-   if (dim == 2)
+   if (Device::FastKernelsEnabled())
    {
-      TransferKernels::Prolongation2D(NE, D1D, Q1D, localL, localH, B, mask);
-   }
-   else if (dim == 3)
-   {
-      TransferKernels::Prolongation3D(NE, D1D, Q1D, localL, localH, B, mask);
+      assert(dim==3);
+      constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+      // lFESpace map idx
+      const int lND = lFESpace.GetNDofs();
+      const Operator *lERop = lFESpace.GetElementRestriction(ordering);
+      const ElementRestriction* lER = dynamic_cast<const ElementRestriction*>(lERop);
+      assert(lER);
+      const int *lmap = lER->GatherMap().Read();
+      assert(lER->GatherMap().Size() == D1D*D1D*D1D*NE);
+      // hFESpace map idx
+      const int hNQ = hFESpace.GetNDofs();
+      const Operator *hERop = elem_restrict_lex_h;
+      const ElementRestriction* hER = dynamic_cast<const ElementRestriction*>(hERop);
+      assert(hER);
+      const int *hmap = hER->GatherMap().Read();
+      assert(hER->GatherMap().Size() == Q1D*Q1D*Q1D*NE);
+
+      void (*Ker)(const int lND,
+                  const int hNQ,
+                  const int NE,
+                  const int *lmap,
+                  const Vector& x,
+                  const int *hmap,
+                  Vector& y,
+                  const Array<double>& B,
+                  const Vector& mask) = nullptr;
+
+      const int id = (D1D << 4) | Q1D;
+      switch (id) // orders 1~8
+      {
+         case 0x23: Ker=TransferKernels::FastProlongation3D<2,3>; break; // 1
+         case 0x34: Ker=TransferKernels::FastProlongation3D<3,4>; break; // 2
+         case 0x35: Ker=TransferKernels::FastProlongation3D<3,5>; break;
+         case 0x45: Ker=TransferKernels::FastProlongation3D<4,5>; break; // 3
+         case 0x56: Ker=TransferKernels::FastProlongation3D<5,6>; break; // 4
+         case 0x57: Ker=TransferKernels::FastProlongation3D<5,7>; break;
+         case 0x67: Ker=TransferKernels::FastProlongation3D<6,7>; break; // 5
+         case 0x78: Ker=TransferKernels::FastProlongation3D<7,8>; break; // 6
+         default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
+      }
+      Ker(lND, hNQ, NE, lmap, x, hmap, y, B, mask);
    }
    else
    {
-      MFEM_ABORT("TensorProductPRefinementTransferOperator::Mult not "
-                 "implemented for dim = "
-                 << dim);
+      elem_restrict_lex_l->Mult(x, localL);
+      if (dim == 2)
+      {
+         TransferKernels::Prolongation2D(NE, D1D, Q1D, localL, localH, B, mask);
+      }
+      else if (dim == 3)
+      {
+         TransferKernels::Prolongation3D(NE, D1D, Q1D, localL, localH, B, mask);
+      }
+      else
+      {
+         MFEM_ABORT("TensorProductPRefinementTransferOperator::Mult not "
+                    "implemented for dim = "
+                    << dim);
+      }
+      elem_restrict_lex_h->MultTranspose(localH, y);
    }
-   elem_restrict_lex_h->MultTranspose(localH, y);
 }
 
 void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
@@ -1406,22 +1685,68 @@ void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
       return;
    }
 
-   elem_restrict_lex_h->Mult(x, localH);
-   if (dim == 2)
+   if (Device::FastKernelsEnabled())
    {
-      TransferKernels::Restriction2D(NE, D1D, Q1D, localH, localL, Bt, mask);
-   }
-   else if (dim == 3)
-   {
-      TransferKernels::Restriction3D(NE, D1D, Q1D, localH, localL, Bt, mask);
+      assert(dim==3);
+      // lFESpace map idx
+      const int lND = lFESpace.GetNDofs();
+      constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+      const Operator *lERop = lFESpace.GetElementRestriction(ordering);
+      const ElementRestriction* lER = dynamic_cast<const ElementRestriction*>(lERop);
+      assert(lER);
+      const int *lmap = lER->GatherMap().Read();
+      // hFESpace map idx
+      const int hNQ = hFESpace.GetNDofs();
+      const Operator *hERop = elem_restrict_lex_h;
+      const ElementRestriction* hER = dynamic_cast<const ElementRestriction*>(hERop);
+      assert(hER);
+      const int *hmap = hER->GatherMap().Read();
+      assert(hER->GatherMap().Size() == Q1D*Q1D*Q1D*NE);
+
+      void (*Ker)(const int lND,
+                  const int hNQ,
+                  const int NE,
+                  const int *hmap,
+                  const Vector& x,
+                  const int *lmap,
+                  Vector& y,
+                  const Array<double>& Bt,
+                  const Vector& mask) = nullptr;
+
+      const int id = (D1D << 4) | Q1D;
+      switch (id) // orders 1~8
+      {
+         case 0x23: Ker=TransferKernels::FastRestriction3D<2,3>; break; // 1
+         case 0x34: Ker=TransferKernels::FastRestriction3D<3,4>; break; // 2
+         case 0x35: Ker=TransferKernels::FastRestriction3D<3,5>; break;
+         case 0x45: Ker=TransferKernels::FastRestriction3D<4,5>; break; // 3
+         case 0x56: Ker=TransferKernels::FastRestriction3D<5,6>; break; // 4
+         case 0x57: Ker=TransferKernels::FastRestriction3D<5,7>; break;
+         case 0x67: Ker=TransferKernels::FastRestriction3D<6,7>; break; // 5
+         case 0x78: Ker=TransferKernels::FastRestriction3D<7,8>; break; // 6
+         default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
+      }
+      Ker(lND, hNQ, NE, hmap, x, lmap, y, Bt, mask);
    }
    else
    {
-      MFEM_ABORT("TensorProductPRefinementTransferOperator::MultTranspose not "
-                 "implemented for dim = "
-                 << dim);
+      elem_restrict_lex_h->Mult(x, localH);
+      if (dim == 2)
+      {
+         TransferKernels::Restriction2D(NE, D1D, Q1D, localH, localL, Bt, mask);
+      }
+      else if (dim == 3)
+      {
+         TransferKernels::Restriction3D(NE, D1D, Q1D, localH, localL, Bt, mask);
+      }
+      else
+      {
+         MFEM_ABORT("TensorProductPRefinementTransferOperator::MultTranspose not "
+                    "implemented for dim = "
+                    << dim);
+      }
+      elem_restrict_lex_l->MultTranspose(localL, y);
    }
-   elem_restrict_lex_l->MultTranspose(localL, y);
 }
 
 #ifdef MFEM_USE_MPI

general/backends.hpp

@@ -18,6 +18,7 @@
 #include <cusparse.h>
 #include <library_types.h>
 #include <cuda_runtime.h>
+#include <curand.h>
 #include <cuda.h>
 #endif
 #include "cuda.hpp"
@@ -39,20 +40,26 @@
 #endif
 
 #if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+#define MFEM_GLOBAL
 #define MFEM_DEVICE
 #define MFEM_LAMBDA
+#define MFEM_GLOBAL
 #define MFEM_HOST_DEVICE
 // MFEM_DEVICE_SYNC is made available for debugging purposes
 #define MFEM_DEVICE_SYNC
 // MFEM_STREAM_SYNC is used for UVM and MPI GPU-Aware kernels
 #define MFEM_STREAM_SYNC
+#define MFEM_LAUNCH_BOUNDS(...)
+#define MFEM_LAUNCH_KERNEL(Kernel,Grid,Block,Smem,...) Kernel(__VA_ARGS__)
 #endif
 
 #if !((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) || \
       (defined(MFEM_USE_HIP)  && defined(__HIP_DEVICE_COMPILE__)))
 #define MFEM_SHARED
 #define MFEM_SYNC_THREAD
+#define MFEM_GRID_DIM(k) 1
 #define MFEM_BLOCK_ID(k) 0
+#define MFEM_BLOCK_DIM(k) 1
 #define MFEM_THREAD_ID(k) 0
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
@@ -91,4 +98,12 @@ MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 #endif
 }
 
+/// Helper function to return and increment a given pointer with a given size
+template<typename T> MFEM_HOST_DEVICE static
+inline T *DeviceMemAlloc(T* &mem, size_t size) noexcept
+{
+   T* base = mem;
+   return (mem += size, base);
+}
+
 #endif // MFEM_BACKENDS_HPP