Merge remote-tracking branch 'origin/master' into entity-sets-dev

# Conflicts:
#	mesh/pncmesh.hpp

.gitignore

@@ -15,19 +15,6 @@
 CMakeCache.txt
 CMakeFiles/
 
-# NSYS files
-*.nsys-rep
-
-# Benchmark files
-*.tex
-*.org
-*.aux
-*.log
-*.gnuplot
-*.table
-*.pdf
-*.png
-
 # Backup files
 *~
 
@@ -329,9 +316,6 @@ tests/unit/debug_device_tests
 
 # Benchmark binaries
 tests/benchmarks/bench_ceed
-tests/benchmarks/bench_solvers
-tests/benchmarks/bench_linext
-tests/benchmarks/bench_lor_assembly
 tests/benchmarks/bench_tmop
 tests/benchmarks/bench_vector
 tests/benchmarks/bench_virtuals

.gitlab/configs/lassen-config.yml

@@ -45,5 +45,5 @@ variables:
     - echo ${MFEM_DATA_DIR}
     - echo ${SPEC}
     # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 30 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
   needs: [setup]

CHANGELOG

@@ -10,16 +10,10 @@
 
 Version 4.3.1 (development)
 ===========================
-- More explicit and consistent formating of the output of iterative solvers
-  with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
-
-- Added support for automatic differentiation. Users can select between native
-  implementation and external library implementation during configuration. One
-  parallel and two serial examples are implemented in the miniapps/autodiff/
-  directory.
-
-- GridFunctionCoefficient (and the related vector, gradient, divergence, and
-  curl classes) now work properly with LORDiscretization and LORSolver.
+- Added support for automatic differentiation. Users can select between 
+  native implementation and external library implementation at the 
+  configuration phase. A parallel and two serial examples are implemented
+  in the autodiff miniapp directory.
 
 - Added support for mesh preprocessing to resolve fine scale problem data
   before simulation. This feature uses adaptive mesh refinement to control the
@@ -77,23 +71,13 @@ Version 4.3.1 (development)
 
 - Added initial TMOP-based capabilities for surface fitting and tangential
   relaxation in the mesh-optimizer and pmesh-optimizer miniapps.
-
+ 
 - Added ParMesh Adjaceny Set (adjset) creation support to the Conduit Mesh
   Blueprint MFEM wrapper functions in ConduitDataCollection.
 
-- Added full assembly / device support for the LinearForm integrators:
-  * DomainLF: L(v) := (f, v)
-  * VectorDomainLF: L(v) := ((f1,...,fn), (v1,...,vn))
-  * DomainLFGrad: L(v) := (f,grad v)
-  * VectorDomainLFGrad:  L(v) := ((f1x,f1y,f1z,...,fnx,fny,fnz),grad(v1,...,vn))
-  It can be enabled by setting the linear assembly level to
-  'LinearAssemblyLevel::FULL' with the 'LinearForm::SetAssemblyLevel' method.
-
 - `HypreParVector` and `Vector` now support move semantics, and the copy
   constructor for `HypreParVector` now copies the local vector data.
 
-- The HPC versions of ex1 and ex1p (in miniapps/performance) now support
-  runtime selection of either 2D or 3D meshes.
 
 Version 4.3, released on July 29, 2021
 ======================================

CODE_OF_CONDUCT.md

@@ -1,133 +0,0 @@
-# MFEM Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at mfem@llnl.gov.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident. Anyone involved in the reported behavior will recuse
-themselves from the investigation and decision making about the resolution of
-the complaint.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -21,9 +21,6 @@ documentation; new examples and miniapps; HPC performance improvements; etc.
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license.
 
-Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
-in the MFEM community, you agree to abide by its rules.
-
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -141,7 +138,6 @@ The MFEM source code has the following structure:
   │   ├── tools
   │   └── toys
   └── tests
-      ├── benchmarks
       ├── convergence
       ├── gitlab
       ├── mem_manager

README

@@ -12,9 +12,6 @@ to enable high-performance scalable finite element discretization research and
 application development on a wide variety of platforms, ranging from laptops to
 supercomputers.
 
-We welcome contributions and feedback from the community. Please see the file
-CONTRIBUTING.md for additional details about our development process.
-
 * For building instructions, see the file INSTALL, or type "make help".
 
 * Copyright and licensing information can be found in files LICENSE and NOTICE.
@@ -22,6 +19,9 @@ CONTRIBUTING.md for additional details about our development process.
 * The best starting point for new users interested in MFEM's features is to
   review the examples and miniapps at https://mfem.org/examples.
 
+* Developers interested in contributing to the library, should read the
+  instructions and documentation in the CONTRIBUTING.md file.
+
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support

config/defaults.mk

@@ -428,7 +428,7 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse -lnvToolsExt
+CUDA_LIB = -lcusparse
 
 # HIP library configuration (currently not needed)
 HIP_OPT =

examples/ex1.cpp

@@ -120,8 +120,8 @@ int main(int argc, char *argv[])
    //    largest number that gives a final mesh with no more than 50,000
    //    elements.
    {
-      const int NE = Device::IsEnabled() ? 1e5 : 50000.;
-      const int ref_levels = (int)floor(log(NE/mesh.GetNE())/log(2.)/dim);
+      int ref_levels =
+         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh.UniformRefinement();
@@ -171,7 +171,6 @@ int main(int argc, char *argv[])
    LinearForm b(&fespace);
    ConstantCoefficient one(1.0);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
    b.Assemble();
 
    // 8. Define the solution vector x as a finite element grid function
@@ -185,10 +184,7 @@ int main(int argc, char *argv[])
    //    domain integrator.
    BilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   // 'Default' E2E kernel
-   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
-   // Can be E2E, L2L, libCeed, etc.
-   a.AddDomainIntegrator(new MassIntegrator(one));
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 10. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: eliminating boundary
@@ -201,7 +197,7 @@ int main(int argc, char *argv[])
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-   //cout << "Size of linear system: " << A->Height() << endl;
+   cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
    if (!pa)
@@ -227,70 +223,10 @@ int main(int argc, char *argv[])
             ceed::AlgebraicSolver M(a, ess_tdof_list);
             PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
          }
-         /*else
+         else
          {
             OperatorJacobiSmoother M(a, ess_tdof_list);
             PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-         }*/
-         else
-         {
-            const int myid = 0;
-            const int max_it = 50;
-            const int print_lvl = -1;
-            const double rtol = 1e-12;
-
-            CGSolver cg;
-            cg.SetRelTol(rtol);
-            cg.SetOperator(*A);
-
-            // Warm-up CG solve (in case of JIT to avoid timing it)
-            {
-               Vector Y(X);
-               cg.SetMaxIter(2);
-               cg.SetPrintLevel(-1);
-               cg.Mult(B, Y);
-               MFEM_DEVICE_SYNC;
-            }
-
-            // benchmark this problem
-            {
-               tic_toc.Clear();
-               cg.SetMaxIter(max_it);
-               cg.SetPrintLevel(print_lvl);
-               {
-                  tic_toc.Start();
-                  cg.Mult(B, X);
-                  MFEM_DEVICE_SYNC;
-                  tic_toc.Stop();
-               }
-            }
-            // Final norm check
-            const double final_norm = cg.GetFinalNorm();
-            MFEM_VERIFY(final_norm < sqrt(rtol),
-                        "FinalNorm (" << final_norm << ") Error!");
-            // Number of iteration check
-            const int num_iter = cg.GetNumIterations();
-            MFEM_VERIFY(num_iter <= max_it,
-                        "NumIterations (" << num_iter << ") Error!");
-
-            const double rt = tic_toc.RealTime();
-            const double rt_min = rt, rt_max = rt;
-            const int dofs = fespace.GetVSize();
-            const int cg_iter = cg.GetNumIterations();
-            const double mdofs_max = ((1e-6 * dofs) * cg_iter) / rt_max;
-            const double mdofs_min = ((1e-6 * dofs) * cg_iter) / rt_min;
-            if (myid == 0)
-            {
-               mfem::out << "Total CG time:    " << rt_max << " (" << rt_min << ") sec."
-                         << std::endl;
-               mfem::out << "Time per CG step: "
-                         << rt_max / cg_iter << " ("
-                         << rt_min / cg_iter << ") sec." << std::endl;
-               mfem::out << "\033[32m";
-               mfem::out << "\"DOFs/sec\" in CG: " << mdofs_max << " ("
-                         << mdofs_min << ") million.";
-               mfem::out << "\033[m" << std::endl;
-            }
          }
       }
       else

examples/ex10.cpp

@@ -149,7 +149,7 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 void InitialVelocity(const Vector &x, Vector &v);
 
-void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -376,10 +376,10 @@ int main(int argc, char *argv[])
 }
 
 
-void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!os)
+   if (!out)
    {
       return;
    }
@@ -389,25 +389,24 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   os << "solution\n" << *mesh << *field;
+   out << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      os << "window_size 800 800\n";
-      os << "window_title '" << field_name << "'\n";
+      out << "window_size 800 800\n";
+      out << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         os << "view 0 0\n"; // view from top
-         os << "keys jl\n";  // turn off perspective and light
+         out << "view 0 0\n"; // view from top
+         out << "keys jl\n";  // turn off perspective and light
       }
-      os << "keys cm\n";         // show colorbar and mesh
-      // update value-range; keep mesh-extents fixed
-      os << "autoscale value\n";
-      os << "pause\n";
+      out << "keys cm\n";         // show colorbar and mesh
+      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      out << "pause\n";
    }
-   os << flush;
+   out << flush;
 }
 
 

examples/ex10p.cpp

@@ -154,8 +154,7 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 void InitialVelocity(const Vector &x, Vector &v);
 
-void visualize(ostream &os, ParMesh *mesh,
-               ParGridFunction *deformed_nodes,
+void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -439,11 +438,10 @@ int main(int argc, char *argv[])
    return 0;
 }
 
-void visualize(ostream &os, ParMesh *mesh,
-               ParGridFunction *deformed_nodes,
+void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!os)
+   if (!out)
    {
       return;
    }
@@ -453,27 +451,25 @@ void visualize(ostream &os, ParMesh *mesh,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   os << "parallel " << mesh->GetNRanks()
-      << " " << mesh->GetMyRank() << "\n";
-   os << "solution\n" << *mesh << *field;
+   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
+   out << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      os << "window_size 800 800\n";
-      os << "window_title '" << field_name << "'\n";
+      out << "window_size 800 800\n";
+      out << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         os << "view 0 0\n"; // view from top
-         os << "keys jl\n";  // turn off perspective and light
+         out << "view 0 0\n"; // view from top
+         out << "keys jl\n";  // turn off perspective and light
       }
-      os << "keys cm\n";         // show colorbar and mesh
-      // update value-range; keep mesh-extents fixed
-      os << "autoscale value\n";
-      os << "pause\n";
+      out << "keys cm\n";         // show colorbar and mesh
+      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      out << "pause\n";
    }
-   os << flush;
+   out << flush;
 }
 
 

examples/ex18.hpp

@@ -32,7 +32,7 @@ private:
    mutable DenseTensor flux;
    mutable Vector z;
 
-   void GetFlux(const DenseMatrix &state_, DenseTensor &flux_) const;
+   void GetFlux(const DenseMatrix &state, DenseTensor &flux) const;
 
 public:
    FE_Evolution(FiniteElementSpace &vfes_,
@@ -256,26 +256,26 @@ inline double ComputeMaxCharSpeed(const Vector &state, const int dim)
 }
 
 // Compute the flux at solution nodes.
-void FE_Evolution::GetFlux(const DenseMatrix &x_, DenseTensor &flux_) const
+void FE_Evolution::GetFlux(const DenseMatrix &x, DenseTensor &flux) const
 {
-   const int flux_dof = flux_.SizeI();
-   const int flux_dim = flux_.SizeJ();
+   const int dof = flux.SizeI();
+   const int dim = flux.SizeJ();
 
-   for (int i = 0; i < flux_dof; i++)
+   for (int i = 0; i < dof; i++)
    {
-      for (int k = 0; k < num_equation; k++) { state(k) = x_(i, k); }
-      ComputeFlux(state, flux_dim, f);
+      for (int k = 0; k < num_equation; k++) { state(k) = x(i, k); }
+      ComputeFlux(state, dim, f);
 
-      for (int d = 0; d < flux_dim; d++)
+      for (int d = 0; d < dim; d++)
       {
          for (int k = 0; k < num_equation; k++)
          {
-            flux_(i, d, k) = f(k, d);
+            flux(i, d, k) = f(k, d);
          }
       }
 
       // Update max char speed
-      const double mcs = ComputeMaxCharSpeed(state, flux_dim);
+      const double mcs = ComputeMaxCharSpeed(state, dim);
       if (mcs > max_char_speed) { max_char_speed = mcs; }
    }
 }

examples/ex19.cpp

@@ -171,7 +171,7 @@ public:
 };
 
 // Visualization driver
-void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -542,10 +542,10 @@ RubberOperator::~RubberOperator()
 
 
 // Inline visualization
-void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
                GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!os)
+   if (!out)
    {
       return;
    }
@@ -555,25 +555,23 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   os << "solution\n" << *mesh << *field;
+   out << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      os << "window_size 800 800\n";
-      os << "window_title '" << field_name << "'\n";
+      out << "window_size 800 800\n";
+      out << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         os << "view 0 0\n"; // view from top
-         // turn off perspective and light, +anti-aliasing
-         os << "keys jlA\n";
+         out << "view 0 0\n"; // view from top
+         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
       }
-      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
-      // update value-range; keep mesh-extents fixed
-      os << "autoscale value\n";
+      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
+      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
    }
-   os << flush;
+   out << flush;
 }
 
 void ReferenceConfiguration(const Vector &x, Vector &y)

examples/ex19p.cpp

@@ -185,8 +185,7 @@ public:
 };
 
 // Visualization driver
-void visualize(ostream &os, ParMesh *mesh,
-               ParGridFunction *deformed_nodes,
+void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name = NULL,
                bool init_vis = false);
 
@@ -618,11 +617,10 @@ RubberOperator::~RubberOperator()
 
 
 // Inline visualization
-void visualize(ostream &os, ParMesh *mesh,
-               ParGridFunction *deformed_nodes,
+void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
                ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!os)
+   if (!out)
    {
       return;
    }
@@ -632,27 +630,24 @@ void visualize(ostream &os, ParMesh *mesh,
 
    mesh->SwapNodes(nodes, owns_nodes);
 
-   os << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() <<
-      "\n";
-   os << "solution\n" << *mesh << *field;
+   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
+   out << "solution\n" << *mesh << *field;
 
    mesh->SwapNodes(nodes, owns_nodes);
 
    if (init_vis)
    {
-      os << "window_size 800 800\n";
-      os << "window_title '" << field_name << "'\n";
+      out << "window_size 800 800\n";
+      out << "window_title '" << field_name << "'\n";
       if (mesh->SpaceDimension() == 2)
       {
-         os << "view 0 0\n"; // view from top
-         // turn off perspective and light, +anti-aliasing
-         os << "keys jlA\n";
+         out << "view 0 0\n"; // view from top
+         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
       }
-      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
-      // update value-range; keep mesh-extents fixed
-      os << "autoscale value\n";
+      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
+      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
    }
-   os << flush;
+   out << flush;
 }
 
 void ReferenceConfiguration(const Vector &x, Vector &y)

examples/ex1_es.cpp

@@ -0,0 +1,266 @@
+//                                MFEM Example 1
+//
+// Compile with: make ex1
+//
+// Sample runs:  ex1 -m ../data/square-disc.mesh
+//               ex1 -m ../data/star.mesh
+//               ex1 -m ../data/escher.mesh
+//               ex1 -m ../data/fichera.mesh
+//               ex1 -m ../data/square-disc-p2.vtk -o 2
+//               ex1 -m ../data/square-disc-p3.mesh -o 3
+//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
+//               ex1 -m ../data/disc-nurbs.mesh -o -1
+//               ex1 -m ../data/pipe-nurbs.mesh -o -1
+//               ex1 -m ../data/star-surf.mesh
+//               ex1 -m ../data/square-disc-surf.mesh
+//               ex1 -m ../data/inline-segment.mesh
+//               ex1 -m ../data/amr-quad.mesh
+//               ex1 -m ../data/amr-hex.mesh
+//               ex1 -m ../data/fichera-amr.mesh
+//               ex1 -m ../data/mobius-strip.mesh
+//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "./star-set.mesh";
+   int order = 1;
+   int rs = -1;
+   int ra = 0;
+   int bt = EntitySets::INVALID;
+   const char *bs = "Origin";
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&rs, "-rs", "--refine-serial",
+                  "Number of serial refinement levels");
+   args.AddOption(&ra, "-ra", "--refine-adaptive",
+                  "Number of adaptive refinement levels");
+   args.AddOption(&bt, "-bt", "--bc-entity-type",
+                  "");
+   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
+                  "");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
+   //    largest number that gives a final mesh with no more than 50,000
+   //    elements.
+   {
+      int ref_levels = ( rs >= 0 ) ? rs :
+                       (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+
+   if ( ra > 0 )
+   {
+      cout << "calling EnsureNCMesh" << endl;
+      mesh->EnsureNCMesh();
+      cout << "back from EnsureNCMesh" << endl;
+   }
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+   cout << "Calling RandomRefinement " << ra << " times." << endl;
+   for (int l = 0; l < ra; l++)
+   {
+      mesh->RandomRefinement(0.2);
+   }
+   cout << "Done with refinement" << endl;
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   if ( mesh->ncmesh )
+   {
+      mesh->ncmesh->PrintStats(cout);
+
+      ofstream ofsV("vp.out");
+      ofstream ofsE("ce.out");
+      mesh->ncmesh->PrintVertexParents(ofsV);
+      mesh->ncmesh->PrintCoarseElements(ofsE);
+   }
+
+   // 4. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+      cout << "Using isoparametric FEs: " << fec->Name() << endl;
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of finite element unknowns: "
+        << fespace->GetTrueVSize() << endl;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if ( bt == EntitySets::INVALID )
+   {
+      if (mesh->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(mesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+   }
+   else
+   {
+      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
+                                    ess_tdof_list);
+   }
+
+   cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
+
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 7. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 8. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 9. Assemble the bilinear form and the corresponding linear system,
+   //    applying any necessary transformations such as: eliminating boundary
+   //    conditions, applying conforming constraints for non-conforming AMR,
+   //    static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   SparseMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   cout << "Size of linear system: " << A.Height() << endl;
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system A X = B with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+   // 10. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
+#endif
+
+   // 11. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 12. Save the refined mesh and the solution. This output can be viewed later
+   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh->Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 13. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 14. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;
+
+   return 0;
+}

examples/ex1p.cpp

@@ -128,7 +128,7 @@ int main(int argc, char *argv[])
    //    more than 10,000 elements.
    {
       int ref_levels =
-         (int)floor(log(1000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh.UniformRefinement();
@@ -197,7 +197,6 @@ int main(int argc, char *argv[])
    ParLinearForm b(&fespace);
    ConstantCoefficient one(1.0);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
    b.Assemble();
 
    // 10. Define the solution vector x as a parallel finite element grid
@@ -211,8 +210,7 @@ int main(int argc, char *argv[])
    //     Diffusion domain integrator.
    ParBilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
-   a.AddDomainIntegrator(new MassIntegrator(one));
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 12. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
@@ -247,67 +245,13 @@ int main(int argc, char *argv[])
    {
       prec = new HypreBoomerAMG;
    }
-
-   const int max_it = 50;
-   const int print_lvl = -1;
-   const double rtol = 1e-12;
-
    CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(rtol);
-   //cg.SetMaxIter(max_it);
-   //cg.SetPrintLevel(print_lvl);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
    if (prec) { cg.SetPreconditioner(*prec); }
    cg.SetOperator(*A);
-
-   // Warm-up CG solve (in case of JIT to avoid timing it)
-   {
-      Vector Y(X);
-      cg.SetMaxIter(2);
-      cg.SetPrintLevel(-1);
-      cg.Mult(B, Y);
-      MFEM_DEVICE_SYNC;
-   }
-
-   // benchmark this problem
-   {
-      tic_toc.Clear();
-      cg.SetMaxIter(max_it);
-      cg.SetPrintLevel(print_lvl);
-      {
-         tic_toc.Start();
-         cg.Mult(B, X);
-         MFEM_DEVICE_SYNC;
-         tic_toc.Stop();
-      }
-   }
-   // Final norm check
-   const double final_norm = cg.GetFinalNorm();
-   MFEM_VERIFY(final_norm < sqrt(rtol),
-               "FinalNorm (" << final_norm << ") Error!");
-   // Number of iteration check
-   const int num_iter = cg.GetNumIterations();
-   MFEM_VERIFY(num_iter <= max_it,
-               "NumIterations (" << num_iter << ") Error!");
-
-   const double rt = tic_toc.RealTime();
-   const double rt_min = rt, rt_max = rt;
-   HYPRE_BigInt dofs = fespace.GlobalTrueVSize();
-   const int cg_iter = cg.GetNumIterations();
-   const double mdofs_max = ((1e-6 * dofs) * cg_iter) / rt_max;
-   const double mdofs_min = ((1e-6 * dofs) * cg_iter) / rt_min;
-   if (myid == 0)
-   {
-      mfem::out << "Total CG time:    " << rt_max << " (" << rt_min << ") sec."
-                << std::endl;
-      mfem::out << "Time per CG step: "
-                << rt_max / cg_iter << " ("
-                << rt_min / cg_iter << ") sec." << std::endl;
-      mfem::out << "\033[32m";
-      mfem::out << "\"DOFs/sec\" in CG: " << mdofs_max << " ("
-                << mdofs_min << ") million.";
-      mfem::out << "\033[m" << std::endl;
-   }
-   //cg.Mult(B, X);
+   cg.Mult(B, X);
    delete prec;
 
    // 14. Recover the parallel grid function corresponding to X. This is the

examples/ex1p_es.cpp

@@ -0,0 +1,388 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+//               The following are examples of using EntitySets to define
+//               homogeneous Dirichlet boundary condition.  These examples
+//               require a modified mesh file and a specialized version of
+//               example 1 called "ex1p_es".
+//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 0 -bs Origin
+//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 1 -bs Axes
+//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
+//                                    -bt 1 -bs "Negative Axes"
+//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
+//                                    -bt 2 -bs "Interior Corner"
+//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
+//                                    -bt 2 -bs "Exterior Corner"
+//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
+//                                    -bt 3 -bs "Interior Corner"
+//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
+//                                    -bt 3 -bs "Exterior Corner"
+//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 3 -bs "Steps"
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "./star-set.mesh";
+   int order = 1;
+   int rs = -1;
+   int rp = 2;
+   int ra = 0;
+   int bt = EntitySets::INVALID;
+   const char *bs = "Origin";
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&rs, "-rs", "--refine-serial",
+                  "Number of serial refinement levels");
+   args.AddOption(&rp, "-rp", "--refine-parallel",
+                  "Number of parallel refinement levels");
+   args.AddOption(&ra, "-ra", "--refine-adaptive",
+                  "Number of adaptive refinement levels");
+   args.AddOption(&bt, "-bt", "--bc-entity-type",
+                  "");
+   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
+                  "");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 10,000 elements.
+   {
+      int ref_levels = ( rs >= 0 ) ? rs :
+                       (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
+         mesh->UniformRefinement();
+      }
+      MPI_Barrier(MPI_COMM_WORLD);
+      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
+   }
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+   /*
+     At this point we have a serial mesh containing an EntitySets
+     object which stores the current node/edge/face/element indices
+     for each entity in each set.  This data is duplicated on each MPI
+     rank.
+    */
+   if ( ra > 0 )
+   {
+      cout << "calling EnsureNCMesh" << endl;
+      mesh->EnsureNCMesh();
+      cout << "back from EnsureNCMesh" << endl;
+   }
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+   /*
+     We now have an NCEntitySets object which stores the node indices
+     describing each enity in each node/edge/face set and the element
+     indices for the elements in each element set.  This data is
+     duplicated on each MPI rank.
+    */
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   cout << "creating ParMesh from serial mesh" << endl;
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   cout << "done creating ParMesh from serial mesh" << endl;
+   delete mesh;
+   if ( pmesh->pent_sets )
+   {
+      cout << "pmesh->pent_sets is non NULL" << endl;
+      pmesh->pent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "pmesh->pent_sets is NULL" << endl;
+   }
+   /*
+     We now have a ParEntitySets object which marshals the data stored
+     in EntitySets objects.  The data has now been pruned so that each
+     rank only contains indices of local entities.
+
+     The NCEntitySets object remains unchanged...
+
+     If we have an NC mesh a different path is taken and the
+     EntitySets are ignored.
+
+     1) ParNCMesh is created from NCMesh
+       a) Creates a ParNCEntitySets object from ncmesh (every rank contains
+          information to find every entity)
+     2) ParNCMesh is pruned which involves renumbering elements and vertices
+     3) ParMesh is initialized from ParNCMesh
+     4) ParNCMesh::OnMeshUpdated is called
+     5) Mesh::GenerateNCFaceInfo is called
+    */
+   {
+      int par_ref_levels = rp;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
+         pmesh->UniformRefinement();
+      }
+      MPI_Barrier(MPI_COMM_WORLD);
+      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
+   }
+   /*
+     RandomRefinement will end up calling
+     ParMesh::NonconformingRefinement which will create a new ParMesh
+     object using the ParNCMesh object and then call
+     ParMesh::OnMeshUpdated on this new mesh.
+    */
+
+   for (int l = 0; l < ra; l++)
+   {
+      pmesh->RandomRefinement(0.2);
+   }
+   if ( ra > 0 )
+   {
+      if ( pmesh->pent_sets )
+      {
+         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
+         pmesh->pent_sets->PrintSetInfo(cout);
+      }
+      else
+      {
+         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
+      }
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if ( bt == EntitySets::INVALID )
+   {
+      if (pmesh->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+   }
+   else
+   {
+      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
+                                    ess_tdof_list);
+   }
+   for (int i=0; i<num_procs; i++)
+   {
+      if (myid == i)
+      {
+         cout << "Number of Dirichlet dofs on proc " << i << ": "
+              << ess_tdof_list.Size() << endl;
+      }
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm *b = new ParLinearForm(fespace);
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   //     preconditioner from hypre.
+   HypreSolver *amg = new HypreBoomerAMG(A);
+   HyprePCG *pcg = new HyprePCG(A);
+   pcg->SetTol(1e-12);
+   pcg->SetMaxIter(200);
+   pcg->SetPrintLevel(2);
+   pcg->SetPreconditioner(*amg);
+   pcg->Mult(B, X);
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 14. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 16. Free the used memory.
+   delete pcg;
+   delete amg;
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/ex27.cpp

@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
                    double alpha, double beta, double gamma,
-                   double &error);
+                   double &err);
 
 int main(int argc, char *argv[])
 {
@@ -302,26 +302,26 @@ int main(int argc, char *argv[])
    {
       // Integrate the solution on the Dirichlet boundary and compare to the
       // expected value.
-      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
+      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
 
       bool hom_dbc = (dbc_val == 0.0);
-      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
       mfem::out << "Average of solution on Gamma_dbc:\t"
                 << avg << ", \t"
                 << (hom_dbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
    }
    {
       // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
       // to the expected value.
-      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
+      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
 
       bool hom_nbc = (nbc_val == 0.0);
-      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
       mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
    }
    {
       // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -330,26 +330,25 @@ int main(int argc, char *argv[])
       nbc0_bdr = 0;
       nbc0_bdr[3] = 1;
 
-      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
+      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
 
       bool hom_nbc = true;
       mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
    }
    {
       // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
       // expected value.
-      double error;
-      double avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, error);
+      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
 
       bool hom_rbc = (rbc_b_val == 0.0);
-      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
       mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                 << avg << ", \t"
                 << (hom_rbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
    }
 
    // 14. Save the refined mesh and the solution. This output can be viewed
@@ -638,11 +637,11 @@ Mesh * GenerateSerialMesh(int ref)
 
 double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
                    double alpha, double beta, double gamma,
-                   double &error)
+                   double &err)
 {
    double nrm = 0.0;
    double avg = 0.0;
-   error = 0.0;
+   err = 0.0;
 
    const bool a_is_zero = alpha == 0.0;
    const bool b_is_zero = beta == 0.0;
@@ -706,20 +705,20 @@ double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
 
          // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
          val -= gamma;
-         error += (val*val) * ip.weight * face_weight;
+         err += (val*val) * ip.weight * face_weight;
       }
    }
 
    // Normalize by the length of the boundary
    if (std::abs(nrm) > 0.0)
    {
-      error /= nrm;
+      err /= nrm;
       avg /= nrm;
    }
 
    // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'error')
-   error = (error >= 0.0) ? sqrt(error) : -sqrt(-error);
+   // quadrature weights may produce negative 'err')
+   err = (err >= 0.0) ? sqrt(err) : -sqrt(-err);
 
    // Return the average value of alpha * n.Grad(x) + beta * x
    return avg;

examples/ex27p.cpp

@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
                    double alpha, double beta, double gamma,
-                   double &error);
+                   double &err);
 
 int main(int argc, char *argv[])
 {
@@ -321,26 +321,26 @@ int main(int argc, char *argv[])
    {
       // Integrate the solution on the Dirichlet boundary and compare to the
       // expected value.
-      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
+      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
 
       bool hom_dbc = (dbc_val == 0.0);
-      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
       mfem::out << "Average of solution on Gamma_dbc:\t"
                 << avg << ", \t"
                 << (hom_dbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
    }
    {
       // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
       // to the expected value.
-      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
+      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
 
       bool hom_nbc = (nbc_val == 0.0);
-      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
       mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
    }
    {
       // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -349,26 +349,25 @@ int main(int argc, char *argv[])
       nbc0_bdr = 0;
       nbc0_bdr[3] = 1;
 
-      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
+      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
 
       bool hom_nbc = true;
       mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                 << avg << ", \t"
                 << (hom_nbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
    }
    {
       // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
       // expected value.
-      double error, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val,
-                                      error);
+      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
 
       bool hom_rbc = (rbc_b_val == 0.0);
-      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
       mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                 << avg << ", \t"
                 << (hom_rbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
    }
 
    // 15. Save the refined mesh and the solution in parallel. This output can be
@@ -668,11 +667,11 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
    double loc_vals[3];
    double &nrm = loc_vals[0];
    double &avg = loc_vals[1];
-   double &error = loc_vals[2];
+   double &err = loc_vals[2];
 
    nrm = 0.0;
    avg = 0.0;
-   error = 0.0;
+   err = 0.0;
 
    const bool a_is_zero = alpha == 0.0;
    const bool b_is_zero = beta == 0.0;
@@ -736,7 +735,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
 
          // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
          val -= gamma;
-         error += (val*val) * ip.weight * face_weight;
+         err += (val*val) * ip.weight * face_weight;
       }
    }
 
@@ -755,7 +754,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
    }
 
    // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'error')
+   // quadrature weights may produce negative 'err')
    glb_err = (glb_err >= 0.0) ? sqrt(glb_err) : -sqrt(-glb_err);
 
    // Return the average value of alpha * n.Grad(x) + beta * x

examples/ex29.cpp

@@ -167,9 +167,9 @@ int main(int argc, char *argv[])
 
    // 13. Compute error in the solution and its flux
    FunctionCoefficient uCoef(uExact);
-   double error = x.ComputeL2Error(uCoef);
+   double err = x.ComputeL2Error(uCoef);
 
-   cout << "|u - u_h|_2 = " << error << endl;
+   cout << "|u - u_h|_2 = " << err << endl;
 
    FiniteElementSpace flux_fespace(mesh, &fec, 3);
    GridFunction flux(&flux_fespace);

examples/ex29p.cpp

@@ -197,9 +197,9 @@ int main(int argc, char *argv[])
 
    // 15. Compute error in the solution and its flux
    FunctionCoefficient uCoef(uExact);
-   double error = x.ComputeL2Error(uCoef);
+   double err = x.ComputeL2Error(uCoef);
 
-   if (myid == 0) { cout << "|u - u_h|_2 = " << error << endl; }
+   if (myid == 0) { cout << "|u - u_h|_2 = " << err << endl; }
 
    ParFiniteElementSpace flux_fespace(&pmesh, &fec, 3);
    ParGridFunction flux(&flux_fespace);

examples/ex3p.cpp

@@ -255,10 +255,10 @@ int main(int argc, char *argv[])
 
    // 15. Compute and print the L^2 norm of the error.
    {
-      double error = x.ComputeL2Error(E);
+      double err = x.ComputeL2Error(E);
       if (myid == 0)
       {
-         cout << "\n|| E_h - E ||_{L^2} = " << error << '\n' << endl;
+         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
       }
    }
 

examples/ex3p_es.cpp

@@ -0,0 +1,411 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+//void E_exact(const Vector &, Vector &);
+//void f_exact(const Vector &, Vector &);
+//double freq = 1.0, kappa;
+void f_const(const Vector &, Vector &);
+
+int dim;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/beam-tet.mesh";
+   int order = 1;
+   int rs = -1;
+   int rp = 2;
+   int ra = 0;
+   int bt = EntitySets::INVALID;
+   const char *bs = "Origin";
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   /*
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   */
+   args.AddOption(&rs, "-rs", "--refine-serial",
+                  "Number of serial refinement levels");
+   args.AddOption(&rp, "-rp", "--refine-parallel",
+                  "Number of parallel refinement levels");
+   args.AddOption(&ra, "-ra", "--refine-adaptive",
+                  "Number of adaptive refinement levels");
+   args.AddOption(&bt, "-bt", "--bc-entity-type",
+                  "");
+   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
+                  "");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   // kappa = freq * M_PI;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      int ref_levels = ( rs >= 0 ) ? rs :
+                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
+         mesh->UniformRefinement();
+      }
+      MPI_Barrier(MPI_COMM_WORLD);
+      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
+   }
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+   /*
+     At this point we have a serial mesh containing an EntitySets
+     object which stores the current node/edge/face/element indices
+     for each entity in each set.  This data is duplicated on each MPI
+     rank.
+    */
+   if ( ra > 0 )
+   {
+      cout << "calling EnsureNCMesh" << endl;
+      mesh->EnsureNCMesh();
+      cout << "back from EnsureNCMesh" << endl;
+   }
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+   /*
+     We now have an NCEntitySets object which stores the node indices
+     describing each enity in each node/edge/face set and the element
+     indices for the elements in each element set.  This data is
+     duplicated on each MPI rank.
+    */
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   if ( pmesh->pent_sets )
+   {
+      cout << "pmesh->pent_sets is non NULL" << endl;
+      pmesh->pent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "pmesh->pent_sets is NULL" << endl;
+   }
+   {
+      int par_ref_levels = rp;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
+         pmesh->UniformRefinement();
+      }
+      MPI_Barrier(MPI_COMM_WORLD);
+      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
+   }
+   pmesh->ReorientTetMesh();
+   pmesh->ent_sets->PrintSetInfo(cout);
+
+   for (int l = 0; l < ra; l++)
+   {
+      pmesh->RandomRefinement(0.2);
+   }
+   if ( ra > 0 )
+   {
+      if ( pmesh->pent_sets )
+      {
+         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
+         pmesh->pent_sets->PrintSetInfo(cout);
+      }
+      else
+      {
+         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
+      }
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if ( bt == EntitySets::INVALID )
+   {
+      if (pmesh->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+   }
+   else
+   {
+      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
+                                    ess_tdof_list);
+   }
+   if (myid == 0)
+   {
+      cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
+   }
+
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   VectorFunctionCoefficient f(sdim, f_const);
+   ParLinearForm *b = new ParLinearForm(fespace);
+   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   // VectorFunctionCoefficient E(sdim, E_exact);
+   // x.ProjectCoefficient(E);
+   x = 0.0;
+
+   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+   //     mass domain integrators.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(1.0);
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+   //     preconditioner from hypre.
+   ParFiniteElementSpace *prec_fespace =
+      (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+   HypreSolver *ams = new HypreAMS(A, prec_fespace);
+   HyprePCG *pcg = new HyprePCG(A);
+   pcg->SetTol(1e-12);
+   pcg->SetMaxIter(500);
+   pcg->SetPrintLevel(2);
+   pcg->SetPreconditioner(*ams);
+   pcg->Mult(B, X);
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   /*
+   // 14. Compute and print the L^2 norm of the error.
+   {
+      double err = x.ComputeL2Error(E);
+      if (myid == 0)
+      {
+         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+      }
+   }
+   */
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete pcg;
+   delete ams;
+   delete a;
+   delete sigma;
+   delete muinv;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+/*
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}
+*/
+void f_const(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+      f(0) = 1.0;
+      f(1) = 1.0;
+      f(2) = 1.0;
+   }
+   else
+   {
+      f(0) = 1.0;
+      f(1) = 1.0;
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/ex4p.cpp

@@ -256,10 +256,10 @@ int main(int argc, char *argv[])
 
    // 15. Compute and print the L^2 norm of the error.
    {
-      double error = x.ComputeL2Error(F);
+      double err = x.ComputeL2Error(F);
       if (myid == 0)
       {
-         cout << "\n|| F_h - F ||_{L^2} = " << error << '\n' << endl;
+         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
       }
    }
 

examples/ex4p_es.cpp

@@ -0,0 +1,438 @@
+//                       MFEM Example 4 - Parallel Version
+//
+// Compile with: make ex4p
+//
+// Sample runs:  mpirun -np 4 ex4p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex4p -m ../data/star.mesh
+//               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
+//               mpirun -np 4 ex4p -m ../data/beam-hex-nurbs.mesh -o 3
+//               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
+//               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
+//               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
+//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
+//
+// Description:  This example code solves a simple 2D/3D H(div) diffusion
+//               problem corresponding to the second order definite equation
+//               -grad(alpha div F) + beta F = f with boundary condition F dot n
+//               = <given normal field>. Here, we use a given exact solution F
+//               and compute the corresponding r.h.s. f.  We discretize with
+//               Raviart-Thomas finite elements.
+//
+//               The example demonstrates the use of H(div) finite element
+//               spaces with the grad-div and H(div) vector finite element mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Bilinear form
+//               hybridization and static condensation are also illustrated.
+//
+//               We recommend viewing examples 1-3 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, F, and r.h.s., f. See below for implementation.
+//void F_exact(const Vector &, Vector &);
+//void f_exact(const Vector &, Vector &);
+//double freq = 1.0, kappa;
+void f_const(const Vector &, Vector &);
+
+int dim;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int rs = -1;
+   int rp = 2;
+   int ra = 0;
+   int bt = EntitySets::INVALID;
+   const char *bs = "Origin";
+   bool set_bc = true;
+   bool static_cond = false;
+   bool hybridization = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
+                  "Impose or not essential boundary conditions.");
+   args.AddOption(&rs, "-rs", "--refine-serial",
+                  "Number of serial refinement levels");
+   args.AddOption(&rp, "-rp", "--refine-parallel",
+                  "Number of parallel refinement levels");
+   args.AddOption(&ra, "-ra", "--refine-adaptive",
+                  "Number of adaptive refinement levels");
+   args.AddOption(&bt, "-bt", "--bc-entity-type",
+                  "");
+   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
+                  "");
+   // args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+   //                 " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
+                  "--no-hybridization", "Enable hybridization.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   // kappa = freq * M_PI;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume, as well as periodic meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      int ref_levels = ( rs >= 0 ) ? rs :
+                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
+         mesh->UniformRefinement();
+      }
+      MPI_Barrier(MPI_COMM_WORLD);
+      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
+   }
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+   /*
+     At this point we have a serial mesh containing an EntitySets
+     object which stores the current node/edge/face/element indices
+     for each entity in each set.  This data is duplicated on each MPI
+     rank.
+    */
+   if ( ra > 0 )
+   {
+      cout << "calling EnsureNCMesh" << endl;
+      mesh->EnsureNCMesh();
+      cout << "back from EnsureNCMesh" << endl;
+   }
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+   /*
+     We now have an NCEntitySets object which stores the node indices
+     describing each enity in each node/edge/face set and the element
+     indices for the elements in each element set.  This data is
+     duplicated on each MPI rank.
+    */
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   if ( pmesh->pent_sets )
+   {
+      cout << "pmesh->pent_sets is non NULL" << endl;
+      pmesh->pent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "pmesh->pent_sets is NULL" << endl;
+   }
+   {
+      int par_ref_levels = rp;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   for (int l = 0; l < ra; l++)
+   {
+      pmesh->RandomRefinement(0.2);
+   }
+   if ( ra > 0 )
+   {
+      if ( pmesh->pent_sets )
+      {
+         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
+         pmesh->pent_sets->PrintSetInfo(cout);
+      }
+      else
+      {
+         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
+      }
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Raviart-Thomas finite elements of the specified order.
+   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if ( bt == EntitySets::INVALID )
+   {
+      if (pmesh->bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+         ess_bdr = set_bc ? 1 : 0;
+         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+   }
+   else
+   {
+      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
+                                    ess_tdof_list);
+   }
+   if (myid == 0)
+   {
+      cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   VectorFunctionCoefficient f(sdim, f_const);
+   ParLinearForm *b = new ParLinearForm(fespace);
+   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary faces will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   // VectorFunctionCoefficient F(sdim, F_exact);
+   // x.ProjectCoefficient(F);
+   x = 0.0;
+
+   // 10. Set up the parallel bilinear form corresponding to the H(div)
+   //     diffusion operator grad alpha div + beta I, by adding the div-div and
+   //     the mass domain integrators.
+   Coefficient *alpha = new ConstantCoefficient(1.0);
+   Coefficient *beta  = new ConstantCoefficient(1.0);
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation,
+   //     hybridization, etc.
+   FiniteElementCollection *hfec = NULL;
+   ParFiniteElementSpace *hfes = NULL;
+   if (static_cond)
+   {
+      a->EnableStaticCondensation();
+   }
+   else if (hybridization)
+   {
+      hfec = new DG_Interface_FECollection(order-1, dim);
+      hfes = new ParFiniteElementSpace(pmesh, hfec);
+      a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
+                             ess_tdof_list);
+   }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   HYPRE_Int glob_size = A.GetGlobalNumRows();
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << glob_size << endl;
+   }
+
+   // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
+   //     the 3D ADS preconditioners from hypre. If using hybridization, the
+   //     system is preconditioned with hypre's BoomerAMG.
+   HypreSolver *prec = NULL;
+   CGSolver *pcg = new CGSolver(A.GetComm());
+   pcg->SetOperator(A);
+   pcg->SetRelTol(1e-12);
+   pcg->SetMaxIter(500);
+   pcg->SetPrintLevel(1);
+   if (hybridization) { prec = new HypreBoomerAMG(A); }
+   else
+   {
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
+      else            { prec = new HypreADS(A, prec_fespace); }
+   }
+   pcg->SetPreconditioner(*prec);
+   pcg->Mult(B, X);
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+   /*
+   // 14. Compute and print the L^2 norm of the error.
+   {
+      double err = x.ComputeL2Error(F);
+      if (myid == 0)
+      {
+         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
+      }
+   }
+   */
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete pcg;
+   delete prec;
+   delete hfes;
+   delete hfec;
+   delete a;
+   delete alpha;
+   delete beta;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+/*
+// The exact solution (for non-surface meshes)
+void F_exact(const Vector &p, Vector &F)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0;
+
+   F(0) = cos(kappa*x)*sin(kappa*y);
+   F(1) = cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      F(2) = 0.0;
+   }
+}
+
+// The right hand side
+void f_exact(const Vector &p, Vector &f)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0;
+
+   double temp = 1 + 2*kappa*kappa;
+
+   f(0) = temp*cos(kappa*x)*sin(kappa*y);
+   f(1) = temp*cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      f(2) = 0;
+   }
+}
+*/
+void f_const(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+      f(0) = 1.0;
+      f(1) = 1.0;
+      f(2) = 1.0;
+   }
+   else
+   {
+      f(0) = 1.0;
+      f(1) = 1.0;
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/ex6.cpp

@@ -125,7 +125,6 @@ int main(int argc, char *argv[])
    BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
    a.AddDomainIntegrator(integ);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
 
    // 7. The solution vector x and the associated finite element grid function
    //    will be maintained over the AMR iterations. We initialize it to zero.

examples/ex6p.cpp

@@ -202,7 +202,6 @@ int main(int argc, char *argv[])
    BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
    a.AddDomainIntegrator(integ);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
 
    // 12. The solution vector x and the associated finite element grid function
    //     will be maintained over the AMR iterations. We initialize it to zero.

examples/ex6p_es.cpp

@@ -0,0 +1,325 @@
+//                       MFEM Example 6 - Parallel Version
+//
+// Compile with: make ex6p
+//
+// Sample runs:  mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 1
+//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/square-disc-nurbs.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/star.mesh -o 3
+//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/fichera.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/disc-nurbs.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/ball-nurbs.mesh
+//               mpirun -np 4 ex6p -m ../data/pipe-nurbs.mesh
+//               mpirun -np 4 ex6p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
+//
+// Description:  This is a version of Example 1 with a simple adaptive mesh
+//               refinement loop. The problem being solved is again the Laplace
+//               equation -Delta u = 1 with homogeneous Dirichlet boundary
+//               conditions. The problem is solved on a sequence of meshes which
+//               are locally refined in a conforming (triangles, tetrahedrons)
+//               or non-conforming (quadrilaterals, hexahedra) manner according
+//               to a simple ZZ error estimator.
+//
+//               The example demonstrates MFEM's capability to work with both
+//               conforming and nonconforming refinements, in 2D and 3D, on
+//               linear, curved and surface meshes. Interpolation of functions
+//               from coarse to fine meshes, as well as persistent GLVis
+//               visualization are also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+static int max_dofs = 100000;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "./star-set.mesh";
+   int order = 1;
+   int bt = EntitySets::INVALID;
+   const char *bs = "";
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&max_dofs, "-md", "--max-dofs",
+                  "Maximum number of degrees of freedom.");
+   args.AddOption(&bt, "-bt", "--bc-entity-type",
+                  "");
+   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
+                  "");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution.
+   //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
+   //    sure that the mesh is non-conforming.
+   if (mesh->NURBSext)
+   {
+      mesh->UniformRefinement();
+      mesh->SetCurvature(2);
+   }
+   mesh->EnsureNCMesh();
+   if ( mesh->ent_sets )
+   {
+      cout << "mesh->ent_sets is non NULL" << endl;
+      mesh->ent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "mesh->ent_sets is NULL" << endl;
+   }
+
+   // 5. Define a parallel mesh by partitioning the serial mesh.
+   //    Once the parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   if ( pmesh.pent_sets )
+   {
+      cout << "pmesh->pent_sets is non NULL" << endl;
+      pmesh.pent_sets->PrintSetInfo(cout);
+   }
+   else
+   {
+      cout << "pmesh->pent_sets is NULL" << endl;
+   }
+
+   // 6. Define a finite element space on the mesh. The polynomial order is
+   //    one (linear) by default, but this can be changed on the command line.
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+
+   Array<int> ess_tdof_list;
+   if ( bt == EntitySets::INVALID )
+   {
+      if (pmesh.bdr_attributes.Size())
+      {
+         Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+   }
+   else
+   {
+      fespace.GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
+                                   ess_tdof_list);
+   }
+
+   // 7. As in Example 1p, we set up bilinear and linear forms corresponding to
+   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
+   //    problem yet, this will be done in the main loop.
+   ParBilinearForm a(&fespace);
+   ParLinearForm b(&fespace);
+
+   ConstantCoefficient one(1.0);
+
+   BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
+   a.AddDomainIntegrator(integ);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+
+   // 8. The solution vector x and the associated finite element grid function
+   //    will be maintained over the AMR iterations. We initialize it to zero.
+   ParGridFunction x(&fespace);
+   x = 0;
+
+   // 9. Connect to GLVis.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   socketstream sout;
+   if (visualization)
+   {
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         if (myid == 0)
+         {
+            cout << "Unable to connect to GLVis server at "
+                 << vishost << ':' << visport << endl;
+            cout << "GLVis visualization disabled.\n";
+         }
+         visualization = false;
+      }
+
+      sout.precision(8);
+   }
+
+   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   //     with L2 projection in the smoothing step to better handle hanging
+   //     nodes and parallel partitioning. We need to supply a space for the
+   //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
+   //     used here).
+   L2_FECollection flux_fec(order, dim);
+   ParFiniteElementSpace flux_fes(&pmesh, &flux_fec, sdim);
+   RT_FECollection smooth_flux_fec(order-1, dim);
+   ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec);
+   // Another possible option for the smoothed flux space:
+   // H1_FECollection smooth_flux_fec(order, dim);
+   // ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec, dim);
+   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
+
+   // 11. A refiner selects and refines elements based on a refinement strategy.
+   //     The strategy here is to refine elements with errors larger than a
+   //     fraction of the maximum element error. Other strategies are possible.
+   //     The refiner will call the given error estimator.
+   ThresholdRefiner refiner(estimator);
+   refiner.SetTotalErrorFraction(0.7);
+
+   // 12. The main AMR loop. In each iteration we solve the problem on the
+   //     current mesh, visualize the solution, and refine the mesh.
+   // const int max_dofs = 100000;
+   for (int it = 0; ; it++)
+   {
+      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
+      if (myid == 0)
+      {
+         cout << "\nAMR iteration " << it << endl;
+         cout << "Number of unknowns: " << global_dofs << endl;
+      }
+
+      // 13. Assemble the stiffness matrix and the right-hand side. Note that
+      //     MFEM doesn't care at this point that the mesh is nonconforming
+      //     and parallel. The FE space is considered 'cut' along hanging
+      //     edges/faces, and also across processor boundaries.
+      a.Assemble();
+      b.Assemble();
+
+      // 14. Create the parallel linear system: eliminate boundary conditions,
+      //     constrain hanging nodes and nodes across processor boundaries.
+      //     The system will be solved for true (unconstrained/unique) DOFs only.
+      // Array<int> ess_tdof_list;
+      if ( bt == EntitySets::INVALID )
+      {
+         if (pmesh.bdr_attributes.Size())
+         {
+            Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+            ess_bdr = 1;
+            fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         }
+      }
+      else
+      {
+         fespace.GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
+                                      ess_tdof_list);
+      }
+
+      HypreParMatrix A;
+      Vector B, X;
+      const int copy_interior = 1;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+
+      // 15. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreBoomerAMG amg;
+      amg.SetPrintLevel(0);
+      CGSolver pcg(A.GetComm());
+      pcg.SetPreconditioner(amg);
+      pcg.SetOperator(A);
+      pcg.SetRelTol(1e-6);
+      pcg.SetMaxIter(200);
+      pcg.SetPrintLevel(3); // print the first and the last iterations only
+      pcg.Mult(B, X);
+
+      // 16. Extract the parallel grid function corresponding to the finite element
+      //     approximation X. This is the local solution on each processor.
+      a.RecoverFEMSolution(X, b, x);
+
+      // 17. Send the solution by socket to a GLVis server.
+      if (visualization)
+      {
+         sout << "parallel " << num_procs << " " << myid << "\n";
+         sout << "solution\n" << pmesh << x << flush;
+      }
+
+      if (global_dofs > max_dofs)
+      {
+         if (myid == 0)
+         {
+            cout << "Reached the maximum number of dofs. Stop." << endl;
+         }
+         break;
+      }
+
+      // 18. Call the refiner to modify the mesh. The refiner calls the error
+      //     estimator to obtain element errors, then it selects elements to be
+      //     refined and finally it modifies the mesh. The Stop() method can be
+      //     used to determine if a stopping criterion was met.
+      refiner.Apply(pmesh);
+      if (refiner.Stop())
+      {
+         if (myid == 0)
+         {
+            cout << "Stopping criterion satisfied. Stop." << endl;
+         }
+         break;
+      }
+
+      // 19. Update the finite element space (recalculate the number of DOFs,
+      //     etc.) and create a grid function update matrix. Apply the matrix
+      //     to any GridFunctions over the space. In this case, the update
+      //     matrix is an interpolation matrix so the updated GridFunction will
+      //     still represent the same function as before refinement.
+      fespace.Update();
+      x.Update();
+
+      // 20. Load balance the mesh, and update the space and solution. Currently
+      //     available only for nonconforming meshes.
+      if (pmesh.Nonconforming())
+      {
+         pmesh.Rebalance();
+
+         // Update the space and the GridFunction. This time the update matrix
+         // redistributes the GridFunction among the processors.
+         fespace.Update();
+         x.Update();
+      }
+
+      // 21. Inform also the bilinear and linear forms that the space has
+      //     changed.
+      a.Update();
+      b.Update();
+   }
+
+   MPI_Finalize();
+   return 0;
+}

examples/ex7p.cpp

@@ -282,10 +282,10 @@ int main(int argc, char *argv[])
    delete b;
 
    // 12. Compute and print the L^2 norm of the error.
-   double error = x.ComputeL2Error(sol_coef);
+   double err = x.ComputeL2Error(sol_coef);
    if (myid == 0)
    {
-      cout << "\nL2 norm of error: " << error << endl;
+      cout << "\nL2 norm of error: " << err << endl;
    }
 
    // 13. Save the refined mesh and the solution. This output can be viewed

examples/fichera-mixed-set.mesh

@@ -0,0 +1,162 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+14
+1 4 13 15 21 25
+1 4 12 13 15 21
+1 4 13 21 22 25
+1 4 15 24 21 25
+1 4 13 15 25 16
+1 5  0  1  4  3  9 10 13 12
+1 5  8  9 12 11 17 18 21 20
+1 5  2  3  6  5 11 12 15 14
+1 6  3  4  6 12 13 15
+1 6  4  7  6 13 16 15
+1 6 12 13 21  9 10 18
+1 6 13 22 21 10 19 18
+1 6 11 14 20 12 15 21
+1 6 15 21 24 14 20 23
+
+boundary
+30
+ 1 3  5  6  3  2
+ 2 2  3  6  4
+ 2 2  4  6  7
+ 3 3  3  4  1  0
+ 4 3 11 12  9  8
+ 5 3  2  3 12 11
+ 6 3  0  1 10  9
+ 7 2  9 10 18
+ 7 2 10 19 18
+ 8 3  8  9 18 17
+ 9 3  1  4 13 10
+10 3  4  7 16 13
+11 2 13 16 25
+11 2 13 25 22
+12 3 10 13 22 19
+13 3  7  6 15 16
+14 3  6  5 14 15
+15 3 15 14 23 24
+16 2 16 15 25
+16 2 15 24 25
+17 3  5  2 11 14
+18 3  3  0  9 12
+19 3 11  8 17 20
+20 2 11 20 14
+20 2 14 20 23
+21 3 17 18 21 20
+22 3 18 19 22 21
+23 2 21 22 25
+23 2 21 25 24
+24 3 20 21 24 23
+
+vertices
+26
+3
+0 -1 -1
+1 -1 -1
+-1 0 -1
+0 0 -1
+1 0 -1
+-1 1 -1
+0 1 -1
+1 1 -1
+-1 -1 0
+0 -1 0
+1 -1 0
+-1 0 0
+0 0 0
+1 0 0
+-1 1 0
+0 1 0
+1 1 0
+-1 -1 1
+0 -1 1
+1 -1 1
+-1 0 1
+0 0 1
+1 0 1
+-1 1 1
+0 1 1
+1 1 1
+
+MFEM sets v1.0
+
+vertex_sets
+1
+
+Origin
+1
+12
+
+edge_sets
+2
+
+Axes
+3
+12 13
+12 15
+12 21
+
+Negative Axes
+3
+12 9
+12 11
+12 3
+
+face_sets
+2
+
+Interior Corner
+3
+3 11 12  9 8
+3  2  3 12 11
+3  3  0  9 12
+
+Exterior Corner
+15
+2 13 16 25
+2 13 25 22
+2 16 15 25
+2 15 24 25
+2 21 22 25
+2 21 25 24
+3 10 13 22 19
+3 4 7 16 13
+3 1 4 13 10
+3 7 6 15 16
+3 6 5 14 15
+3 15 14 23 24
+3 20 21 24 23
+3 18 19 22 21
+3 17 18 21 20
+
+element_sets
+3
+
+Interior Corner
+3
+5 6 7
+
+Exterior Corner
+5
+0 1 2 3 4
+
+Steps
+3
+6 8 9

examples/fichera-set.mesh

@@ -0,0 +1,145 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+3
+
+elements
+7
+1 5 0 1 4 3 9 10 13 12
+1 5 3 4 7 6 12 13 16 15
+1 5 2 3 6 5 11 12 15 14
+1 5 8 9 12 11 17 18 21 20
+1 5 9 10 13 12 18 19 22 21
+1 5 12 13 16 15 21 22 25 24
+1 5 11 12 15 14 20 21 24 23
+
+boundary
+24
+1 3 5 6 3 2
+2 3 6 7 4 3
+3 3 3 4 1 0
+4 3 11 12 9 8
+5 3 2 3 12 11
+6 3 0 1 10 9
+7 3 9 10 19 18
+8 3 8 9 18 17
+9 3 1 4 13 10
+10 3 4 7 16 13
+11 3 13 16 25 22
+12 3 10 13 22 19
+13 3 7 6 15 16
+14 3 6 5 14 15
+15 3 15 14 23 24
+16 3 16 15 24 25
+17 3 5 2 11 14
+18 3 3 0 9 12
+19 3 11 8 17 20
+20 3 14 11 20 23
+21 3 17 18 21 20
+22 3 18 19 22 21
+23 3 21 22 25 24
+24 3 20 21 24 23
+
+vertices
+26
+3
+0 -1 -1
+1 -1 -1
+-1 0 -1
+0 0 -1
+1 0 -1
+-1 1 -1
+0 1 -1
+1 1 -1
+-1 -1 0
+0 -1 0
+1 -1 0
+-1 0 0
+0 0 0
+1 0 0
+-1 1 0
+0 1 0
+1 1 0
+-1 -1 1
+0 -1 1
+1 -1 1
+-1 0 1
+0 0 1
+1 0 1
+-1 1 1
+0 1 1
+1 1 1
+
+MFEM sets v1.0
+
+vertex_sets
+1
+
+Origin
+1
+12
+
+edge_sets
+2
+
+Axes
+3
+12 13
+12 15
+12 21
+
+Negative Axes
+3
+12 9
+12 11
+12 3
+
+face_sets
+2
+
+Interior Corner
+3
+3 11 12  9 8
+3  2  3 12 11
+3  3  0  9 12
+
+Exterior Corner
+12
+3 13 16 25 22
+3 16 15 24 25
+3 21 22 25 24
+3 10 13 22 19
+3 4 7 16 13
+3 1 4 13 10
+3 7 6 15 16
+3 6 5 14 15
+3 15 14 23 24
+3 20 21 24 23
+3 18 19 22 21
+3 17 18 21 20
+
+element_sets
+3
+
+Interior Corner
+3
+0 2 3
+
+Exterior Corner
+1
+5
+
+Steps
+2
+1 3

examples/star-mixed-set.mesh

@@ -0,0 +1,158 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+30
+1 3 0 11 26 14
+1 3 0 14 27 17
+1 3 0 17 28 20
+1 3 0 20 29 23
+1 3 0 23 30 11
+1 2 11 1 26
+1 2 1 12 26
+1 3 26 12 3 13
+1 2 26 13 2
+1 2 14 26 2
+1 2 14 2 27
+1 2 2 15 27
+1 3 27 15 5 16
+1 2 27 16 4
+1 2 17 27 4
+1 2 17 4 28
+1 2 4 18 28
+1 3 28 18 7 19
+1 2 28 19 6
+1 2 20 28 6
+1 2 20 6 29
+1 2 6 21 29
+1 3 29 21 9 22
+1 2 29 22 8
+1 2 23 29 8
+1 2 23 8 30
+1 2 8 24 30
+1 3 30 24 10 25
+1 2 30 25 1
+1 2 11 30 1
+
+boundary
+20
+1 1 13 2
+1 1 12 3
+1 1 16 4
+1 1 15 5
+1 1 19 6
+1 1 18 7
+1 1 22 8
+1 1 21 9
+1 1 25 1
+1 1 24 10
+1 1 3 13
+1 1 1 12
+1 1 5 16
+1 1 2 15
+1 1 7 19
+1 1 4 18
+1 1 9 22
+1 1 6 21
+1 1 10 25
+1 1 8 24
+
+vertices
+31
+2
+0 0
+1 0
+0.309017 0.951057
+1.30902 0.951057
+-0.809017 0.587785
+-0.5 1.53884
+-0.809017 -0.587785
+-1.61803 0
+0.309017 -0.951057
+-0.5 -1.53884
+1.30902 -0.951057
+0.5 0
+1.15451 0.475529
+0.809019 0.951057
+0.154508 0.475529
+-0.0954915 1.24495
+-0.654508 1.06331
+-0.404508 0.293893
+-1.21352 0.293893
+-1.21352 -0.293892
+-0.404508 -0.293893
+-0.654508 -1.06331
+-0.0954915 -1.24495
+0.154508 -0.475529
+0.809019 -0.951057
+1.15451 -0.475529
+0.654509 0.475529
+-0.25 0.769421
+-0.809016 0
+-0.25 -0.76942
+0.654509 -0.475529
+
+MFEM sets v1.0
+
+vertex_sets
+3
+
+Origin
+1
+0
+
+Tent
+5
+1 2 4 6 8
+
+Gazebo
+5
+3 5 7 9 10
+
+edge_sets
+2
+
+Columbine
+5
+1 11
+2 14
+4 17
+6 20
+8 23
+
+Lily
+5
+0 11
+0 14
+0 17
+0 20
+0 23
+
+element_sets
+3
+
+Flying Squirrel
+3
+7 17 27
+
+Sea Lion
+4
+12 17 22 27
+
+Pinwheel
+5
+8 13 18 23 28

examples/star-set.mesh

@@ -0,0 +1,143 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+2
+
+elements
+20
+1 3 0 11 26 14
+1 3 0 14 27 17
+1 3 0 17 28 20
+1 3 0 20 29 23
+1 3 0 23 30 11
+1 3 11 1 12 26
+1 3 26 12 3 13
+1 3 14 26 13 2
+1 3 14 2 15 27
+1 3 27 15 5 16
+1 3 17 27 16 4
+1 3 17 4 18 28
+1 3 28 18 7 19
+1 3 20 28 19 6
+1 3 20 6 21 29
+1 3 29 21 9 22
+1 3 23 29 22 8
+1 3 23 8 24 30
+1 3 30 24 10 25
+1 3 11 30 25 1
+
+boundary
+20
+1 1 13 2
+1 1 12 3
+1 1 16 4
+1 1 15 5
+1 1 19 6
+1 1 18 7
+1 1 22 8
+1 1 21 9
+1 1 25 1
+1 1 24 10
+1 1 3 13
+1 1 1 12
+1 1 5 16
+1 1 2 15
+1 1 7 19
+1 1 4 18
+1 1 9 22
+1 1 6 21
+1 1 10 25
+1 1 8 24
+
+vertices
+31
+2
+0 0
+1 0
+0.309017 0.951057
+1.30902 0.951057
+-0.809017 0.587785
+-0.5 1.53884
+-0.809017 -0.587785
+-1.61803 0
+0.309017 -0.951057
+-0.5 -1.53884
+1.30902 -0.951057
+0.5 0
+1.15451 0.475529
+0.809019 0.951057
+0.154508 0.475529
+-0.0954915 1.24495
+-0.654508 1.06331
+-0.404508 0.293893
+-1.21352 0.293893
+-1.21352 -0.293892
+-0.404508 -0.293893
+-0.654508 -1.06331
+-0.0954915 -1.24495
+0.154508 -0.475529
+0.809019 -0.951057
+1.15451 -0.475529
+0.654509 0.475529
+-0.25 0.769421
+-0.809016 0
+-0.25 -0.76942
+0.654509 -0.475529
+
+MFEM sets v1.0
+
+vertex_sets
+3
+
+Origin
+1
+0
+
+Tent
+5
+1 2 4 6 8
+
+Gazebo
+5
+3 5 7 9 10
+
+edge_sets
+2
+
+Columbine
+5
+1 11
+2 14
+4 17
+6 20
+8 23
+
+Lily
+5
+0 11
+0 14
+0 17
+0 20
+0 23
+
+element_sets
+2
+
+Flying Squirrel
+3
+6 12 18
+
+Sea Lion
+4
+9 12 15 18

fem/CMakeLists.txt

@@ -21,8 +21,6 @@ set(SRCS
   bilininteg_dgtrace_ea.cpp
   bilininteg_diffusion_mf.cpp
   bilininteg_diffusion_pa.cpp
-  bilininteg_diffusion_pa_diag_fast.cpp
-  bilininteg_diffusion_pa_fast.cpp
   bilininteg_diffusion_ea.cpp
   bilininteg_divergence.cpp
   bilininteg_hcurl.cpp
@@ -31,11 +29,6 @@ set(SRCS
   bilininteg_gradient.cpp
   bilininteg_mass_mf.cpp
   bilininteg_mass_pa.cpp
-  bilininteg_mass_pa_diag.cpp
-  bilininteg_mass_pa_fast.cpp
-  bilininteg_mass_pa_fast_amd.cpp
-  bilininteg_mass_pa_fast_hip.cpp
-  bilininteg_mass_pa_fast_diag.cpp
   bilininteg_mass_ea.cpp
   bilininteg_transpose_ea.cpp
   bilininteg_vecdiffusion.cpp
@@ -75,16 +68,7 @@ set(SRCS
   ceed/operator.cpp
   ceed/util.cpp
   linearform.cpp
-  linearform_ext.cpp
   lininteg.cpp
-  lininteg_domain.cpp
-  lininteg_domain_grad.cpp
-  lininteg_vector_domain.cpp
-  lininteg_vector_domain_grad.cpp
-  lor.cpp
-  lor_assembly.cpp
-  lor_assembly_ker.cpp
-  lor_assembly_ni3.cpp
   multigrid.cpp
   nonlinearform.cpp
   nonlinearform_ext.cpp
@@ -177,12 +161,7 @@ set(HDRS
   ceed/operator.hpp
   ceed/util.hpp
   linearform.hpp
-  linearform_ext.hpp
   lininteg.hpp
-  lininteg_domain.hpp
-  lininteg_domain_grad.hpp
-  lor.hpp
-  lor_assembly.hpp
   multigrid.hpp
   nonlinearform.hpp
   nonlinearform_ext.hpp

fem/bilinearform_ext.cpp

@@ -318,31 +318,14 @@ void PABilinearFormExtension::Assemble()
 void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integrators_size = integrators.Size();
 
-   // Scan the different action types we need to launch
-   bool E2E_actions = false,
-        L2L_actions = false;
-   for (int i = 0; i < integrators_size; ++i)
-   {
-      const ActionType action_type = integrators[i]->GetActionType();
-      E2E_actions |= action_type == ActionType::E2E;
-      L2L_actions |= action_type == ActionType::L2L;
-   }
-
-   // typically this is a large vector, so store on device
-   y.UseDevice(true);
-
-   // If E2E kernels are present, do their computation on local vectors
-   if (E2E_actions && elem_restrict)
+   const int iSz = integrators.Size();
+   if (elem_restrict && !DeviceCanUseCeed())
    {
       localY = 0.0;
-      for (int i = 0; i < integrators_size; ++i)
+      for (int i = 0; i < iSz; ++i)
       {
-         if (integrators[i]->GetActionType() == ActionType::E2E)
-         {
-            integrators[i]->AssembleDiagonalPA(localY);
-         }
+         integrators[i]->AssembleDiagonalPA(localY);
       }
       const ElementRestriction* H1elem_restrict =
          dynamic_cast<const ElementRestriction*>(elem_restrict);
@@ -357,20 +340,11 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
    }
    else
    {
-      // otherwise, initialize the y output
+      y.UseDevice(true); // typically this is a large vector, so store on device
       y = 0.0;
-   }
-
-   // Continue with the computation of the L2L integrators
-   if (L2L_actions || !elem_restrict)
-   {
-      for (int i = 0; i < integrators_size; ++i)
+      for (int i = 0; i < iSz; ++i)
       {
-         const ActionType action = integrators[i]->GetActionType();
-         if (action == ActionType::L2L || !elem_restrict)
-         {
-            integrators[i]->AssembleDiagonalPA(y);
-         }
+         integrators[i]->AssembleDiagonalPA(y);
       }
    }
 }
@@ -409,51 +383,26 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integrators_size = integrators.Size();
 
-   // Scan the different action types we need to launch
-   bool E2E_actions = false,
-        L2L_actions = false;
-   for (int i = 0; i < integrators_size; ++i)
+   const int iSz = integrators.Size();
+   if (DeviceCanUseCeed() || !elem_restrict)
    {
-      const ActionType action_type = integrators[i]->GetActionType();
-      E2E_actions |= action_type == ActionType::E2E;
-      L2L_actions |= action_type == ActionType::L2L;
-   }
-
-   // typically this is a large vector, so store on device
-   y.UseDevice(true);
-
-   // If E2E kernels are present, do their computation on local vectors
-   if (E2E_actions && elem_restrict)
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < integrators_size; ++i)
+      y.UseDevice(true); // typically this is a large vector, so store on device
+      y = 0.0;
+      for (int i = 0; i < iSz; ++i)
       {
-         if (integrators[i]->GetActionType() == ActionType::E2E)
-         {
-            integrators[i]->AddMultPA(localX, localY);
-         }
+         integrators[i]->AddMultPA(x, y);
       }
-      elem_restrict->MultTranspose(localY, y);
    }
    else
    {
-      // otherwise, initialize the y output
-      y = 0.0;
-   }
-
-   // Continue with the computation of the L2L integrators
-   if (L2L_actions || !elem_restrict)
-   {
-      for (int i = 0; i < integrators_size; ++i)
+      elem_restrict->Mult(x, localX);
+      localY = 0.0;
+      for (int i = 0; i < iSz; ++i)
       {
-         if (integrators[i]->GetActionType() == ActionType::L2L || !elem_restrict)
-         {
-            integrators[i]->AddMultPA(x, y);
-         }
+         integrators[i]->AddMultPA(localX, localY);
       }
+      elem_restrict->MultTranspose(localY, y);
    }
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();

fem/bilininteg.hpp

@@ -244,8 +244,6 @@ public:
                                     Vector &flux, Vector *d_energy = NULL)
    { return 0.0; }
 
-   virtual bool SupportsBatchedLOR() const { return false; }
-
    virtual ~BilinearFormIntegrator() { }
 };
 
@@ -1979,7 +1977,7 @@ private:
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
    int dim, ne, dofs1D, quad1D;
-   Vector pa_data, CoG;
+   Vector pa_data;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
@@ -2060,8 +2058,6 @@ public:
                                          const FiniteElement &test_fe);
 
    bool SupportsCeed() const { return DeviceCanUseCeed(); }
-
-   bool SupportsBatchedLOR() const { return true; }
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */

fem/bilininteg_convection_pa.cpp

@@ -768,8 +768,8 @@ void SmemPAConvectionApply3D(const int ne,
 
 void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : memory_type;
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetFE(0);

fem/bilininteg_diffusion_pa.cpp

@@ -9,45 +9,18 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#define MFEM_DEBUG_COLOR 87
-#include "../general/debug.hpp"
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
 #include "ceed/diffusion.hpp"
-#include <string>
-
-#define MFEM_NVTX_COLOR Olive
-#include "../general/nvtx.hpp"
 
 using namespace std;
 
 namespace mfem
 {
 
-void NDK_PADiffusionAssembleDiagonal(const int dim,
-                                     const int D1D,
-                                     const int Q1D,
-                                     const int NE,
-                                     const bool symm,
-                                     const FiniteElementSpace *fes,
-                                     const DofToQuad *maps,
-                                     const Vector &D,
-                                     Vector &Y);
-
 // PA Diffusion Integrator
 
-void NDK_PADiffusionApply(const int dim,
-                          const int D1D,
-                          const int Q1D,
-                          const int NE,
-                          const Vector &CoG,
-                          const FiniteElementSpace *fes,
-                          const DofToQuad *maps,
-                          const Vector &D,
-                          const Vector &X,
-                          Vector &Y);
-
 // OCCA 2D Assemble kernel
 #ifdef MFEM_USE_OCCA
 static void OccaPADiffusionSetup2D(const int D1D,
@@ -376,115 +349,10 @@ static void PADiffusionSetup(const int dim,
    }
 }
 
-////////////////////////////////////////////////////////////////////////////////
-template<typename T> MFEM_HOST_DEVICE inline
-void HouseholderReflect(T *A, const T *v,
-                        const T b, const int m, const int n,
-                        const int row, const int col)
-{
-   for (int j = 0; j < n; j++)
-   {
-      T w = A[0*row + j*col];
-      for (int i = 1; i < m; i++) { w += v[i] * A[i*row + j*col]; }
-      A[0*row + j*col] -= b * w;
-      for (int i = 1; i < m; i++) { A[i*row + j*col] -= b * w * v[i]; }
-   }
-}
-
-template<int Q1D, typename T> MFEM_HOST_DEVICE inline
-void HouseholderApplyQ(T *A, const T *Q, const T *tau,
-                       const int k, const int row, const int col)
-{
-   T v[Q1D];
-   for (int ii=0; ii<k; ii++)
-   {
-      const int i = k-1-ii;
-      for (int j = i+1; j < Q1D; j++) { v[j] = Q[j*k+i]; }
-      // Apply Householder reflector (I - tau v v^T) coG^T
-      HouseholderReflect(&A[i*row], &v[i], tau[i], Q1D-i, Q1D, row, col);
-   }
-}
-
-template<int D1D, int Q1D, typename T> MFEM_HOST_DEVICE inline
-void QRFactorization(T *mat, T *tau)
-{
-   T v[Q1D];
-   DeviceMatrix B(mat, D1D, Q1D);
-   for (int i = 0; i < D1D; i++)
-   {
-      // Calculate Householder vector, magnitude
-      T sigma = 0.0;
-      v[i] = B(i,i);
-      for (int j = i + 1; j < Q1D; j++)
-      {
-         v[j] = B(i,j);
-         sigma += v[j] * v[j];
-      }
-      T norm = std::sqrt(v[i]*v[i] + sigma); // norm of v[i:m]
-      T Rii = -copysign(norm, v[i]);
-      v[i] -= Rii;
-      // norm of v[i:m] after modification above and scaling below
-      //   norm = sqrt(v[i]*v[i] + sigma) / v[i];
-      //   tau = 2 / (norm*norm)
-      tau[i] = 2 * v[i]*v[i] / (v[i]*v[i] + sigma);
-      for (int j=i+1; j<Q1D; j++) { v[j] /= v[i]; }
-      // Apply Householder reflector to lower right panel
-      HouseholderReflect(&mat[i*D1D+i+1], &v[i], tau[i],
-                         Q1D-i, D1D-i-1, D1D, 1);
-      // Save v
-      B(i,i) = Rii;
-      for (int j=i+1; j<Q1D; j++) { B(i,j) = v[j]; }
-   }
-}
-
-template<int D1D, int Q1D>
-void GetCollocatedGrad(const ConstDeviceMatrix &b,
-                       const ConstDeviceMatrix &g,
-                       const DeviceMatrix &CoG)
-{
-   double tau[Q1D];
-   double B1d[Q1D*D1D];
-   double G1d[Q1D*D1D];
-   DeviceMatrix B(B1d, D1D, Q1D);
-   DeviceMatrix G(G1d, D1D, Q1D);
-
-   for (int d = 0; d < D1D; d++)
-   {
-      for (int q = 0; q < Q1D; q++)
-      {
-         B(d,q) = b(q,d);
-         G(d,q) = g(q,d);
-      }
-   }
-   QRFactorization<D1D,Q1D>(B1d, tau);
-   // Apply Rinv, colograd1d = grad1d Rinv
-   for (int i = 0; i < Q1D; i++)
-   {
-      CoG(0,i) = G(0,i)/B(0,0);
-      for (int j = 1; j < D1D; j++)
-      {
-         CoG(j,i) = G(j,i);
-         for (int k = 0; k < j; k++) { CoG(j,i) -= B(j,k)*CoG(k,i); }
-         CoG(j,i) /= B(j,j);
-      }
-      for (int j = D1D; j < Q1D; j++) { CoG(j,i) = 0.0; }
-   }
-   // Apply Qtranspose, colograd = colograd Qtranspose
-   HouseholderApplyQ<Q1D>((double*)CoG, B1d, tau, D1D, 1, Q1D);
-}
-
-////////////////////////////////////////////////////////////////////////////////
 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : memory_type;
-
-   // If device options allow fast kernels, set the action type to L2L
-   action_type =
-      (Device::FastKernelsEnabled())?
-      ActionType::L2L: // all fast kernel are L2L
-      ActionType::E2E; // default is E2E
-
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
    // Assuming the same element type
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
@@ -497,7 +365,6 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
       MFEM_VERIFY(!VQ && !MQ && !SMQ,
                   "Only scalar coefficient supported for DiffusionIntegrator"
                   " with libCEED");
-      action_type = ActionType::L2L;
       ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
       return;
    }
@@ -623,40 +490,6 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
          }
       }
    }
-
-   if (Device::FastKernelsEnabled())
-   {
-      NVTX("CoG");
-      const int D1D = dofs1D;
-      const int Q1D = quad1D;
-      const int id = (D1D << 4) | Q1D;
-
-      CoG.SetSize(Q1D*Q1D);
-      CoG.UseDevice(true);
-      assert(CoG.UseDevice());
-
-      void (*KoG)(const ConstDeviceMatrix &b,
-                  const ConstDeviceMatrix &g,
-                  const DeviceMatrix &CoG) = nullptr;
-
-      switch (id) // orders 1~8
-      {
-         case 0x23: KoG=GetCollocatedGrad<2,3>; break; // 1
-         case 0x34: KoG=GetCollocatedGrad<3,4>; break; // 2
-         case 0x45: KoG=GetCollocatedGrad<4,5>; break; // 3
-         case 0x56: KoG=GetCollocatedGrad<5,6>; break; // 4
-         case 0x67: KoG=GetCollocatedGrad<6,7>; break; // 5
-         case 0x78: KoG=GetCollocatedGrad<7,8>; break; // 6
-         //case 0x89: KoG=GetCollocatedGrad<8,9>; break; // 7
-         //case 0x9A: KoG=GetCollocatedGrad<9,10>; break; // 8
-         default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
-      }
-
-      KoG(ConstDeviceMatrix(maps->B.HostRead(),Q1D,D1D),
-          ConstDeviceMatrix(maps->G.HostRead(),Q1D,D1D),
-          DeviceMatrix(CoG.HostReadWrite(),Q1D,Q1D));
-   }
-
    pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
    PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
                     geom->J, coeff, pa_data);
@@ -1036,7 +869,6 @@ static void SmemPADiffusionDiagonal3D(const int NE,
                   }
                }
             }
-            MFEM_SYNC_THREAD;
          }
       }
    });
@@ -1093,11 +925,6 @@ void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
    {
       ceedOp->GetDiagonal(diag);
    }
-   else if (Device::FastKernelsEnabled())
-   {
-      NDK_PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
-                                      fespace, maps, pa_data, diag);
-   }
    else
    {
       if (pa_data.Size()==0) { AssemblePA(*fespace); }
@@ -1353,7 +1180,6 @@ static void SmemPADiffusionApply2D(const int NE,
    auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-
    MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
    {
       const int tidz = MFEM_THREAD_ID(z);
@@ -1394,7 +1220,6 @@ static void SmemPADiffusionApply2D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-
       MFEM_FOREACH_THREAD(dy,y,D1D)
       {
          MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -1720,7 +1545,6 @@ static void SmemPADiffusionApply3D(const int NE,
                                    const int d1d = 0,
                                    const int q1d = 0)
 {
-   MFEM_NVTX;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -1771,7 +1595,6 @@ static void SmemPADiffusionApply3D(const int NE,
             }
          }
       }
-      MFEM_SYNC_THREAD;
       if (MFEM_THREAD_ID(z) == 0)
       {
          MFEM_FOREACH_THREAD(dy,y,D1D)
@@ -1968,7 +1791,6 @@ static void SmemPADiffusionApply3D(const int NE,
             }
          }
       }
-      MFEM_SYNC_THREAD;
    });
 }
 
@@ -2008,11 +1830,8 @@ static void PADiffusionApply(const int dim,
       switch (ID)
       {
          case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
-         case 0x23: return SmemPADiffusionApply2D<2,3,16>(NE,symm,B,G,D,X,Y);
          case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
-         case 0x34: return SmemPADiffusionApply2D<3,4,16>(NE,symm,B,G,D,X,Y);
          case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
-         case 0x45: return SmemPADiffusionApply2D<4,5,8>(NE,symm,B,G,D,X,Y);
          case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
          case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
          case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
@@ -2049,12 +1868,6 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
    {
       ceedOp->AddMult(x, y);
    }
-   else if (Device::FastKernelsEnabled())
-   {
-      NDK_PADiffusionApply(dim, dofs1D, quad1D, ne, CoG,
-                           fespace, maps,
-                           pa_data, x, y);
-   }
    else
    {
       PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,

fem/bilininteg_diffusion_pa_diag_fast.cpp

@@ -1,186 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#define MFEM_NVTX_COLOR Pink
-#include "../general/nvtx.hpp"
-
-#define MFEM_DEBUG_COLOR 206
-#include "../general/debug.hpp"
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// Shared memory PA Diffusion Diagonal 3D kernel
-template<int D1D, int Q1D>
-static void NDK_SmemPADiffusionDiag3D(const int ndofs,
-                                      const int NE,
-                                      const bool symmetric,
-                                      const int *map_,
-                                      const double *b_,
-                                      const double *g_,
-                                      const double *d_,
-                                      double *y_)
-{
-   MFEM_NVTX;
-   constexpr int DIM = 3;
-   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
-   auto b = Reshape(b_, Q1D, D1D);
-   auto g = Reshape(g_, Q1D, D1D);
-   auto D = Reshape(d_, Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
-   auto Y = Reshape(y_, D1D, D1D, D1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      MFEM_SHARED double BG[2][Q1D*D1D];
-      double (*B)[D1D] = (double (*)[D1D]) (BG+0);
-      double (*G)[D1D] = (double (*)[D1D]) (BG+1);
-      MFEM_SHARED double QQD[Q1D][Q1D][D1D];
-      MFEM_SHARED double QDD[Q1D][D1D][D1D];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-               G[q][d] = g(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      for (int i = 0; i < DIM; ++i)
-      {
-         for (int j = 0; j < DIM; ++j)
-         {
-            // first tensor contraction, along z direction
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qy,y,Q1D)
-               {
-                  MFEM_FOREACH_THREAD(dz,z,D1D)
-                  {
-                     QQD[qx][qy][dz] = 0.0;
-                     for (int qz = 0; qz < Q1D; ++qz)
-                     {
-                        const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int ksym = j >= i ?
-                                         3 - (3-i)*(2-i)/2 + j:
-                                         3 - (3-j)*(2-j)/2 + i;
-                        const int k = symmetric ? ksym : (i*DIM) + j;
-                        const double O = D(q,k,e);
-                        const double Bz = B[qz][dz];
-                        const double Gz = G[qz][dz];
-                        const double L = i==2 ? Gz : Bz;
-                        const double R = j==2 ? Gz : Bz;
-                        QQD[qx][qy][dz] += L * O * R;
-                     }
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // second tensor contraction, along y direction
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               MFEM_FOREACH_THREAD(dz,z,D1D)
-               {
-                  MFEM_FOREACH_THREAD(dy,y,D1D)
-                  {
-                     QDD[qx][dy][dz] = 0.0;
-                     for (int qy = 0; qy < Q1D; ++qy)
-                     {
-                        const double By = B[qy][dy];
-                        const double Gy = G[qy][dy];
-                        const double L = i==1 ? Gy : By;
-                        const double R = j==1 ? Gy : By;
-                        QDD[qx][dy][dz] += L * QQD[qx][qy][dz] * R;
-                     }
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // third tensor contraction, along x direction
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1D)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1D)
-                  {
-                     for (int qx = 0; qx < Q1D; ++qx)
-                     {
-                        const double Bx = B[qx][dx];
-                        const double Gx = G[qx][dx];
-                        const double L = i==0 ? Gx : Bx;
-                        const double R = j==0 ? Gx : Bx;
-                        const double lvr = L * QDD[qx][dy][dz] * R;
-                        const int gid = MAP(dx,dy,dz,e);
-                        const int idx = gid >= 0 ? gid : -1 - gid;
-                        AtomicAdd(Y[idx], lvr);
-                     }
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   });
-}
-
-void NDK_PADiffusionAssembleDiagonal(const int dim,
-                                     const int D1D,
-                                     const int Q1D,
-                                     const int NE,
-                                     const bool symm,
-                                     const FiniteElementSpace *fes,
-                                     const DofToQuad *maps,
-                                     const Vector &D,
-                                     Vector &Y)
-{
-   MFEM_NVTX;
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
-   assert(ER);
-   const int *m = ER->GatherMap().Read();
-   const double *b = maps->B.Read();
-   const double *g = maps->G.Read();
-   const double *d = D.Read();
-   double *y = Y.ReadWrite();
-
-   if (dim == 2) {  assert(false); }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return NDK_SmemPADiffusionDiag3D<2,2>(ND,NE,symm,m,b,g,d,y);
-         case 0x23: return NDK_SmemPADiffusionDiag3D<2,3>(ND,NE,symm,m,b,g,d,y);
-         case 0x34: return NDK_SmemPADiffusionDiag3D<3,4>(ND,NE,symm,m,b,g,d,y);
-         case 0x45: return NDK_SmemPADiffusionDiag3D<4,5>(ND,NE,symm,m,b,g,d,y);
-         case 0x46: return NDK_SmemPADiffusionDiag3D<4,6>(ND,NE,symm,m,b,g,d,y);
-         case 0x56: return NDK_SmemPADiffusionDiag3D<5,6>(ND,NE,symm,m,b,g,d,y);
-         case 0x67: return NDK_SmemPADiffusionDiag3D<6,7>(ND,NE,symm,m,b,g,d,y);
-         case 0x78: return NDK_SmemPADiffusionDiag3D<7,8>(ND,NE,symm,m,b,g,d,y);
-         //case 0x89: return NDK_SmemPADiffusionDiag3D<8,9>(ND,NE,symm,m,b,g,d,y);
-         //case 0x9A: return NDK_SmemPADiffusionDiag3D<9,10>(ND,NE,symm,m,b,g,d,y);
-         default: MFEM_ABORT("Not implemented!");
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-} // namespace mfem

fem/bilininteg_diffusion_pa_fast.cpp

@@ -1,332 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#define MFEM_NVTX_COLOR Pink
-#include "../general/nvtx.hpp"
-
-#define MFEM_DEBUG_COLOR 206
-#include "../general/debug.hpp"
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-////////////////////////////////////////////////////////////////////////////////
-template<int D1D, int Q1D, int NBZ, int NBK> static
-MFEM_GLOBAL MFEM_LAUNCH_BOUNDS(Q1D*Q1D*NBZ,NBK)
-void NDK_PADiffApply(const int NE,
-                     const int *MAP,
-                     const double *B,
-                     const double *G,
-                     const double *D,
-                     const double *X,
-                     double *Y)
-{
-   double r_qt, r_q[Q1D];
-   MFEM_SHARED double s_B[D1D][Q1D];
-   MFEM_SHARED double s_G[Q1D][Q1D];
-   MFEM_SHARED double s_Iq[NBZ][Q1D][Q1D][Q1D];
-   MFEM_SHARED double s_Gqr[NBZ][Q1D][Q1D];
-   MFEM_SHARED double s_Gqs[NBZ][Q1D][Q1D];
-
-   for (int be = MFEM_BLOCK_ID(x); be < (NE+NBZ-1)/NBZ; be += MFEM_GRID_DIM(x))
-   {
-      const int tz = MFEM_THREAD_ID(z);
-      const int e = be * NBZ + tz;
-      if (e>=NE) { return; }
-
-      // Scatter X
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            if (tz == 0) { s_G[j][i] = G[i+Q1D*j]; } // ok with init
-            if (tz == 0 && j<D1D) { s_B[j][i] = B[i+Q1D*j]; } // ok
-            if (j<D1D && i<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int k = 0; k < D1D; k++)
-               {
-                  const int gid = MAP[e*D1D*D1D*D1D + k*D1D*D1D + j*D1D + i];
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  r_q[k] = X[idx];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Grad1X
-      MFEM_FOREACH_THREAD(b,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(D1D)
-               for (int c=0; c<D1D; ++c) { u += s_B[c][k] * r_q[c]; }
-               s_Iq[tz][k][b][a] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Grad1Y
-      MFEM_FOREACH_THREAD(k,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b) { r_q[b] = s_Iq[tz][k][b][a]; }
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(D1D)
-               for (int b=0; b<D1D; ++b) { u += s_B[b][j] * r_q[b]; }
-               s_Iq[tz][k][j][a] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Grad1Z
-      MFEM_FOREACH_THREAD(k,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(j,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a) { r_q[a] = s_Iq[tz][k][j][a]; }
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(D1D)
-               for (int a=0; a<D1D; ++a) { u += s_B[a][i] * r_q[a]; }
-               s_Iq[tz][k][j][i] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Flush
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int k = 0; k < Q1D; ++k) { r_q[k] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Q-Function
-      MFEM_UNROLL(Q1D)
-      for (int k = 0; k < Q1D; ++k)
-      {
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(j,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(i,x,Q1D)
-            {
-               double qr = 0.0, qs = 0.0, qt = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int m = 0; m < Q1D; ++m)
-               {
-                  const double Dim = s_G[i][m];
-                  const double Djm = s_G[j][m];
-                  const double Dkm = s_G[k][m];
-                  qr += Dim * s_Iq[tz][k][j][m];
-                  qs += Djm * s_Iq[tz][k][m][i];
-                  qt += Dkm * s_Iq[tz][m][j][i];
-               }
-               //(d, Q1D,Q1D,Q1D, 6, NE);
-               const int ebase = e * 6*Q1D*Q1D*Q1D;
-               const int gbase = i + j*Q1D + k*Q1D*Q1D;
-               const double D00 = D[0*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D01 = D[1*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D02 = D[2*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D11 = D[3*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D12 = D[4*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D22 = D[5*Q1D*Q1D*Q1D + gbase + ebase];
-
-               s_Gqr[tz][j][i] = D00*qr + D01*qs + D02*qt;
-               s_Gqs[tz][j][i] = D01*qr + D11*qs + D12*qt;
-               r_qt = D02*qr + D12*qs + D22*qt;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(j,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(i,x,Q1D)
-            {
-               double Aqtmp = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int m = 0; m < Q1D; ++m)
-               {
-                  const double Dmi = s_G[m][i];
-                  const double Dmj = s_G[m][j];
-                  const double Dkm = s_G[k][m];
-                  Aqtmp += Dmi * s_Gqr[tz][j][m];
-                  Aqtmp += Dmj * s_Gqs[tz][m][i];
-                  r_q[m] += Dkm * r_qt;
-               }
-               r_q[k] += Aqtmp;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-
-      // GradZT
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { u += s_B[c][k] * r_q[k]; }
-               s_Iq[tz][c][j][i] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // GradYT
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_q[j] = s_Iq[tz][c][j][i]; }
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { u += s_B[b][j] * r_q[j]; }
-               s_Iq[tz][c][b][i] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // GradXT
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,D1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_q[i] = s_Iq[tz][c][b][i]; }
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { u += s_B[a][i] * r_q[i]; }
-               s_Iq[tz][c][b][a] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Gather
-      MFEM_FOREACH_THREAD(j,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int k = 0; k < D1D; k++)
-            {
-               const int gid = MAP[e*D1D*D1D*D1D + k*D1D*D1D + j*D1D + i];
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               const double output = s_Iq[tz][k][j][i];
-               AtomicAdd(Y[idx], output);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-}
-
-void NDK_PADiffusionApply(const int dim,
-                          const int D1D,
-                          const int Q1D,
-                          const int NE,
-                          const Vector &CoG,
-                          const FiniteElementSpace *fes,
-                          const DofToQuad *maps,
-                          const Vector &D,
-                          const Vector &X,
-                          Vector &Y)
-{
-   //dbg();
-   MFEM_NVTX;
-   assert(dim == 3);
-
-   const double *b = maps->B.Read();
-   const double *d = D.Read();
-   const double *x = X.Read();
-   double *y = Y.ReadWrite();
-
-   const int id = (D1D << 4) | Q1D;
-
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction *ER = dynamic_cast<const ElementRestriction*>(ERop);
-   assert(ER);
-   const int *map = ER->GatherMap().Read();
-
-   const auto dM = Reshape(map, D1D,D1D,D1D, NE);
-   const auto dB = Reshape(b, Q1D,D1D);
-   const auto dG = Reshape(CoG.Read(), Q1D,Q1D);
-   const auto dD = Reshape(d, Q1D,Q1D,Q1D, 6, NE);
-   const auto dX = Reshape(x, ND);
-   auto dY = Reshape(y, ND);
-
-   void (*Ker)(const int NE,
-               const int *MAP,
-               const double *B,
-               const double *G,
-               const double *D,
-               const double *X,
-               double *Y) = nullptr;
-
-   int NBZ = 1;
-
-   switch (id) // orders 1~8
-   {
-      case 0x23: Ker=NDK_PADiffApply<2,3,16,5>; NBZ=16; break; // 1
-      case 0x34: Ker=NDK_PADiffApply<3,4,8,6>; NBZ=8; break; // 2
-      case 0x45: Ker=NDK_PADiffApply<4,5,4,4>; NBZ=4; break; // 3
-      case 0x56: Ker=NDK_PADiffApply<5,6,4,3>; NBZ=4; break; // 4
-      case 0x67: Ker=NDK_PADiffApply<6,7,1,3>; break; // 5
-      case 0x78: Ker=NDK_PADiffApply<7,8,1,0>; break; // 6
-      //case 0x89: Ker=NDK_PADiffApply<8,9,1,0>; break; // 7
-      //case 0x9A: Ker=NDK_PADiffApply<9,10,1,1>; break; // 8
-      default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
-   }
-   MFEM_CONTRACT_VAR(NBZ);
-   MFEM_LAUNCH_KERNEL(Ker,(NE+NBZ-1)/NBZ,dim3(Q1D,Q1D,NBZ),0,NE,dM,dB,dG,dD,dX,dY);
-}
-
-} // namespace mfem

fem/bilininteg_mass_pa.cpp

@@ -19,49 +19,14 @@ using namespace std;
 namespace mfem
 {
 
-// Forward declaration of non-deterministic 'fast' kernels
-void NDK_PAMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const FiniteElementSpace *fes,
-                     const DofToQuad *maps,
-                     const Vector &D,
-                     const Vector &X,
-                     Vector &Y);
-
-void NDK_AMD_PAMassApply(const int dim,
-                         const int D1D,
-                         const int Q1D,
-                         const int NE,
-                         const FiniteElementSpace *fes,
-                         const DofToQuad *maps,
-                         const Vector &D,
-                         const Vector &X,
-                         Vector &Y);
-
-void NDK_HIP_PAMassApply(const int dim,
-                         const int D1D,
-                         const int Q1D,
-                         const int NE,
-                         const FiniteElementSpace *fes,
-                         const DofToQuad *maps,
-                         const Vector &D,
-                         const Vector &X,
-                         Vector &Y);
-
 // PA Mass Integrator
 
 // PA Mass Assemble kernel
 
 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : memory_type;
-
-   // If device options allow fast kernels, set the action type to L2L
-   action_type =
-      Device::FastKernelsEnabled() ? ActionType::L2L : ActionType::E2E;
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
 
    // Assuming the same element type
    fespace = &fes;
@@ -73,7 +38,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q, action_type);
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
       return;
    }
    dim = mesh->Dimension();
@@ -189,6 +154,328 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    }
 }
 
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAMassAssembleDiagonal2D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QD[MQ1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+static void SmemPAMassAssembleDiagonal2D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
+      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               // might need absolute values on next line
+               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAMassAssembleDiagonal3D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QQD[MQ1][MQ1][MD1];
+      double QDD[MQ1][MD1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void SmemPAMassAssembleDiagonal3D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
+      MFEM_SHARED double QDD[MQ1][MD1][MD1];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+static void PAMassAssembleDiagonal(const int dim, const int D1D,
+                                   const int Q1D, const int NE,
+                                   const Array<double> &B,
+                                   const Vector &D,
+                                   Vector &Y)
+{
+   if (dim == 2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
+         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
+         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
+         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
+         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
+         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
+         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
+         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
+         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
+         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
+         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
+         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
+         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
+         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
+         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
+         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
+         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
+         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void MassIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->GetDiagonal(diag);
+   }
+   else
+   {
+      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
+}
+
+
 #ifdef MFEM_USE_OCCA
 // OCCA PA Mass Apply 2D kernel
 static void OccaPAMassApply2D(const int D1D,
@@ -864,159 +1151,6 @@ static void SmemPAMassApply3D(const int NE,
    });
 }
 
-template<int D1D, int Q1D>
-void SmemPAMassApply3D_v1(const int NE,
-                          const Array<double> &b_,
-                          const Array<double> &bt_,
-                          const Vector &d_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   MFEM_CONTRACT_VAR(bt_);
-   MFEM_CONTRACT_VAR(d1d);
-   MFEM_CONTRACT_VAR(q1d);
-   const auto B = Reshape(b_.Read(), Q1D,D1D);
-   const auto D = Reshape(d_.Read(), Q1D,Q1D,Q1D, NE);
-   const auto X = Reshape(x_.Read(), D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D,D1D,D1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      double u[Q1D];
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
-
-      // Load input, B & X interpolation
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            s_B[qx][dy] = B(qx,dy);
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double Bx = B(qx,dx);
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += X(dx,dy,dz,e) * Bx;
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[dz][dy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Y interpolation
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double zyX = s_q[dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { s_q[dz][qy][qx] = u[qy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Z interpolation, Q-function & Zt projection
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            // Z interpolation
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const double zYX = s_q[dz][qy][qx];
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
-            }
-
-            // Q-function
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               s_q[qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
-            }
-
-            // Zt projection
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double ZYX = s_q[qz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[dz][qy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Yt projection
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double zYX = s_q[dz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { s_q[dz][dy][qx] = u[dy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Xt projection & save output
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double zyX = s_q[dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double output = u[dx];
-               Y(dx,dy,dz,e) += output;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
 static void PAMassApply(const int dim,
                         const int D1D,
                         const int Q1D,
@@ -1067,47 +1201,30 @@ static void PAMassApply(const int dim,
    }
    else if (dim == 3)
    {
-      const int ver = Device::KernelsVersion();
-      const int id = (ver << 8) | (D1D << 4) | Q1D;
-
-      static int ini = 0;
-      if (!ini++) { printf("\033[33mkernel #0x%x\033[m\n",id); }
-
       switch (id)
       {
-         case 0x123: return SmemPAMassApply3D_v1<2,3>(NE,B,Bt,D,X,Y);
-         case 0x124: return SmemPAMassApply3D_v1<2,4>(NE,B,Bt,D,X,Y);
-         case 0x134: return SmemPAMassApply3D_v1<3,4>(NE,B,Bt,D,X,Y);
-         case 0x136: return SmemPAMassApply3D_v1<3,6>(NE,B,Bt,D,X,Y);
-         case 0x145: return SmemPAMassApply3D_v1<4,5>(NE,B,Bt,D,X,Y);
-         case 0x148: return SmemPAMassApply3D_v1<4,8>(NE,B,Bt,D,X,Y);
-         case 0x156: return SmemPAMassApply3D_v1<5,6>(NE,B,Bt,D,X,Y);
-         case 0x158: return SmemPAMassApply3D_v1<5,8>(NE,B,Bt,D,X,Y);
-         case 0x167: return SmemPAMassApply3D_v1<6,7>(NE,B,Bt,D,X,Y);
-         case 0x178: return SmemPAMassApply3D_v1<7,8>(NE,B,Bt,D,X,Y);
-
-         case 0x022: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
-         case 0x023: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
-         case 0x024: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
-         case 0x026: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
-         case 0x034: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x035: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
-         case 0x036: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
-         case 0x037: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
-         case 0x045: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x046: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x048: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
-         case 0x056: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x058: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
-         case 0x067: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
-         case 0x078: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
-         case 0x089: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x09A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
-
-         default:   break; //return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
+         case 0x22: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
+         case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
+         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
+         case 0x26: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
+         case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
+         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
+         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
+         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
+         case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
+         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
+         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
+         case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
+         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
+         case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
+         case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
+         case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
+         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
+         default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
       }
    }
-   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Unknown kernel.");
 }
 
 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -1116,35 +1233,6 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    {
       ceedOp->AddMult(x, y);
    }
-   else if (Device::FastKernelsEnabled())
-   {
-      const int version = Device::KernelsVersion();
-      MFEM_VERIFY(version < 4 || version==7, "Unsupported version!");
-      if (version == 3) // AMD
-      {
-         NDK_AMD_PAMassApply(dim, dofs1D, quad1D, ne,
-                             fespace, maps,
-                             pa_data, x, y);
-      }
-      // 4 E-vector
-      // 5 fused
-      // 6 MMA
-      else if (version == 7) // HIP
-      {
-         NDK_HIP_PAMassApply(dim, dofs1D, quad1D, ne,
-                             fespace, maps,
-                             pa_data, x, y);
-      }
-      // 0 legacy
-      // 1 fast
-      // 2 libP
-      else
-      {
-         NDK_PAMassApply(dim, dofs1D, quad1D, ne,
-                         fespace, maps,
-                         pa_data, x, y);
-      }
-   }
    else
    {
       PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);

fem/bilininteg_mass_pa_diag.cpp

@@ -1,358 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/mass.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-void NDK_PAMassAssembleDiagonal(const int dim,
-                                const int D1D,
-                                const int Q1D,
-                                const int NE,
-                                const FiniteElementSpace *fes,
-                                const DofToQuad *maps,
-                                const Vector &D,
-                                Vector &Y);
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal2D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QD[MQ1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassAssembleDiagonal2D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
-      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               // might need absolute values on next line
-               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal3D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QQD[MQ1][MQ1][MD1];
-      double QDD[MQ1][MD1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassAssembleDiagonal3D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
-      MFEM_SHARED double QDD[MQ1][MD1][MD1];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-static void PAMassAssembleDiagonal(const int dim, const int D1D,
-                                   const int Q1D, const int NE,
-                                   const Array<double> &B,
-                                   const Vector &D,
-                                   Vector &Y)
-{
-   if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
-         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
-         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
-         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
-         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
-         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
-         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
-         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
-         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
-         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
-         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
-         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
-         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
-         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
-         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
-         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
-         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
-         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else if (Device::FastKernelsEnabled())
-   {
-      NDK_PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne,
-                                 fespace, maps,
-                                 pa_data, diag);
-   }
-   else
-   {
-      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
-}
-
-} // namespace mfem

fem/bilininteg_mass_pa_fast.cpp

@@ -1,684 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "gridfunc.hpp"
-#include "restriction.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// Fast '0' non-deterministic 3D mass kernel
-template<int D1D, int Q1D>
-void NDK_SmemPAMassApply3D(const int ndofs,
-                           const int NE,
-                           const int *map,
-                           const double *b_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_)
-{
-   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x_, ndofs);
-   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      MFEM_SHARED double sDQ[Q1D*Q1D];
-      double (*B)[D1D] = (double (*)[D1D]) sDQ;
-      double (*Bt)[Q1D] = (double (*)[Q1D]) sDQ;
-      MFEM_SHARED double sm0[Q1D*Q1D*Q1D];
-      MFEM_SHARED double sm1[Q1D*Q1D*Q1D];
-      double (*DDQ)[D1D][Q1D] = (double (*)[D1D][Q1D]) sm1;
-      double (*DQQ)[Q1D][Q1D] = (double (*)[Q1D][Q1D]) sm0;
-      double (*QQQ)[Q1D][Q1D] = (double (*)[Q1D][Q1D]) sm1;
-      double (*QQD)[Q1D][D1D] = (double (*)[Q1D][D1D]) sm0;
-      double (*QDD)[D1D][D1D] = (double (*)[D1D][D1D]) sm1;
-
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            B[qx][dy] = b(qx,dy);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  if (map)
-                  {
-                     const int gid = MAP(dx, dy, dz, e);
-                     const int idx = gid >= 0 ? gid : -1 - gid;
-                     u[dz] += X(idx) * B[qx][dx];
-                  }
-                  else
-                  {
-                     u[dz] += X1(dx,dy,dz,e) * B[qx][dx];
-                  }
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ[dz][dy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; dz++)
-               {
-                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ[dz][qy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; qz++)
-               {
-                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
-               }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               QQQ[qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(d,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            Bt[d][q] = b(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
-               }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD[qz][qy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD[qz][dy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               if (map)
-               {
-                  const int gid = MAP(dx, dy, dz, e);
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  AtomicAdd(Y(idx), u[dz]);
-               }
-               else
-               {
-                  Y1(dx,dy,dz,e) += u[dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-// Fast '1' non-deterministic 3D mass kernel
-// Smem version melded toward registers + BZ-batch
-template<int D1D, int Q1D, int NBZ>
-void NDK_SmRgPAMassApply3D(const int ndofs,
-                           const int NE,
-                           const int *map,
-                           const double *b_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_)
-{
-   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
-   const auto B = Reshape(b_, Q1D, D1D);
-   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x_, ndofs);
-   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
-
-   MFEM_FORALL_3D(be, (NE+NBZ-1)/NBZ, Q1D, Q1D, NBZ,
-   {
-      double u[Q1D];
-      const int tz = MFEM_THREAD_ID(z);
-      const int e = be * MFEM_THREAD_SIZE(z) + tz;
-
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
-
-      // Load input, B & X interpolation
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            if (tz == 0) { s_B[qx][dy] = B(qx,dy); }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double Bx = B(qx,dx);
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const int gid = map ? MAP(dx,dy,dz,e) : 0;
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  u[dz] += (map ? X(idx) : X1(dx,dy,dz,e)) * Bx;
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Y interpolation
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double zyX = s_q[tz][dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Z interpolation, Q-function & Zt projection
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            // Z interpolation
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const double zYX = s_q[tz][dz][qy][qx];
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
-            }
-
-            // Q-function
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               s_q[tz][qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
-            }
-
-            // Zt projection
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double ZYX = s_q[tz][qz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Yt projection
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double zYX = s_q[tz][dz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Xt projection & save output
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double zyX = s_q[tz][dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double output = u[dx];
-               if (map)
-               {
-                  const int gid = MAP(dx,dy,dz,e);
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  AtomicAdd(Y(idx), output);
-               }
-               else
-               {
-                  Y1(dx,dy,dz,e) += output;
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-// Fast '2' non-deterministic 3D mass kernel
-// Smem version with registers
-template<int D1D, int Q1D>
-void NDK_RegsPAMassApply3D(const int ndofs,
-                           const int NE,
-                           const int *map,
-                           const double *b_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_)
-{
-   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
-   const auto B = Reshape(b_, Q1D,D1D);
-   const auto D = Reshape(d_, Q1D,Q1D,Q1D, NE);
-   const auto X = Reshape(x_, ndofs);
-   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      double r_wk[Q1D];
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
-
-      // Load s_B, load X in shared memory
-      MFEM_FOREACH_THREAD(b,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,Q1D)
-         {
-            if (a<D1D) { s_B[b][a] = B(b,a); }
-
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = 0.0; }
-
-            if (a<D1D && b<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int c=0; c<D1D; ++c)
-               {
-                  const int gid = map ? MAP(a,b,c,e) : 0;
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  s_q[c][b][a] = map ? X(idx) : X1(a,b,c,e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Interpolate in X direction
-      MFEM_FOREACH_THREAD(c,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,Q1D)
-         {
-            if (b<D1D && c<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int a=0; a<D1D; ++a)
-               {
-                  const double q_cba = s_q[c][b][a];
-                  MFEM_UNROLL(Q1D)
-                  for (int i=0; i<Q1D; ++i) { r_wk[i] += s_B[i][a]*q_cba; }
-               }
-               // reg => s_mem
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { s_q[c][b][i] = r_wk[i]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Interpolate in Y direction
-      MFEM_FOREACH_THREAD(c,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            if (c<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int b=0; b<D1D; ++b)
-               {
-                  const double q_cbi = s_q[c][b][i];
-                  MFEM_UNROLL(Q1D)
-                  for (int j=0; j<Q1D; ++j) { r_wk[j] += s_B[j][b]*q_cbi; }
-               }
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { s_q[c][j][i] = r_wk[j]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            // Interpolate in Z direction
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cji = s_q[c][j][i];
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { r_wk[k] += s_B[k][c]*q_cji; }
-            }
-
-            // Scale by Jacobian and integration weights
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] *= D(i,j,k,e); }
-
-            // Project back in Z direction
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               double q_cji = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { q_cji += s_B[k][c] * r_wk[k]; }
-               s_q[c][j][i] = q_cji;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Project back in Y direction
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b)
-            {
-               double q_cbi = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { q_cbi += s_B[j][b] * r_wk[j]; }
-               s_q[c][b][i] = q_cbi;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Project back in X direction
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,D1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a)
-            {
-               double q_cba = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { q_cba += s_B[i][a] * r_wk[i]; }
-               s_q[c][b][a] = q_cba;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Save back to memory
-      MFEM_FOREACH_THREAD(b,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cba = s_q[c][b][a];
-               const int gid = map ? MAP(a,b,c,e) : 0;
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               AtomicAdd(map?Y(idx):Y1(a,b,c,e), q_cba);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-void NDK_PAMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const FiniteElementSpace *fes,
-                     const DofToQuad *maps,
-                     const Vector &D,
-                     const Vector &X,
-                     Vector &Y)
-{
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
-   const int *map = ER ? ER->GatherMap().Read() : nullptr;
-   const double *b = maps->B.Read();
-   const double *d = D.Read();
-   const double *x = X.Read();
-   double *y = Y.ReadWrite();
-
-   assert(dim == 3);
-   const int ver = Device::KernelsVersion();
-   const int id = (ver << 8) | (D1D << 4) | Q1D;
-
-   //printf("\033[32mkernel #0x%x\033[m\n",id); fflush(0);
-
-   switch (id) // orders 1~6
-   {
-      // Fast '0': Legacy non-deterministic 3D mass kernel
-      case 0x023: return NDK_SmemPAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
-      case 0x024: return NDK_SmemPAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
-      case 0x034: return NDK_SmemPAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
-      case 0x036: return NDK_SmemPAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
-      case 0x045: return NDK_SmemPAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
-      case 0x046: return NDK_SmemPAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
-      case 0x048: return NDK_SmemPAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
-      case 0x056: return NDK_SmemPAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
-      case 0x058: return NDK_SmemPAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
-      case 0x067: return NDK_SmemPAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
-      case 0x078: return NDK_SmemPAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
-
-      // Fast '1': Legacy & half smem non-deterministic 3D mass kernel + Z-batch
-      case 0x123: return NDK_SmRgPAMassApply3D<2,3,32>(ND,NE,map,b,d,x,y);//1
-      case 0x124: return NDK_SmRgPAMassApply3D<2,4,16>(ND,NE,map,b,d,x,y);
-      case 0x134: return NDK_SmRgPAMassApply3D<3,4,16>(ND,NE,map,b,d,x,y);//2
-      case 0x136: return NDK_SmRgPAMassApply3D<3,6,8>(ND,NE,map,b,d,x,y);
-      case 0x145: return NDK_SmRgPAMassApply3D<4,5,4>(ND,NE,map,b,d,x,y);//3
-      case 0x146: return NDK_SmRgPAMassApply3D<4,6,4>(ND,NE,map,b,d,x,y);
-      case 0x148: return NDK_SmRgPAMassApply3D<4,8,4>(ND,NE,map,b,d,x,y);
-      case 0x156: return NDK_SmRgPAMassApply3D<5,6,4>(ND,NE,map,b,d,x,y);//4
-      case 0x158: return NDK_SmRgPAMassApply3D<5,8,1>(ND,NE,map,b,d,x,y);
-      case 0x167: return NDK_SmRgPAMassApply3D<6,7,1>(ND,NE,map,b,d,x,y);//5
-      case 0x178: return NDK_SmRgPAMassApply3D<7,8,1>(ND,NE,map,b,d,x,y);//6
-
-      // Fast '2': libP non-deterministic 3D mass kernel
-      case 0x223: return NDK_RegsPAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
-      case 0x224: return NDK_RegsPAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
-      case 0x234: return NDK_RegsPAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
-      case 0x236: return NDK_RegsPAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
-      case 0x245: return NDK_RegsPAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
-      case 0x246: return NDK_RegsPAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
-      case 0x248: return NDK_RegsPAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
-      case 0x256: return NDK_RegsPAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
-      case 0x258: return NDK_RegsPAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
-      case 0x267: return NDK_RegsPAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
-      case 0x278: return NDK_RegsPAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
-
-      // Fast '3': libP + AMD specific non-deterministic 3D mass kernel
-      // in bilininteg_mass_pa_fast_amd.cpp
-
-      default: break;
-   }
-
-   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
-}
-
-} // namespace mfem

fem/bilininteg_mass_pa_fast_amd.cpp

@@ -1,250 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "gridfunc.hpp"
-#include "restriction.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// Fast '3' non-deterministic 3D mass kernel
-template<int D1D, int Q1D>
-void NDK_AMD_PAMassApply3D(const int ndofs,
-                           const int NE,
-                           const int *map,
-                           const double *b_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_)
-{
-   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
-   const auto B = Reshape(b_, Q1D,D1D);
-   const auto D = Reshape(d_, Q1D,Q1D,Q1D, NE);
-   const auto X = Reshape(x_, ndofs);
-   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      double r_wk[Q1D];
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
-
-      // Load s_B, load X in shared memory
-      MFEM_FOREACH_THREAD(b,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,Q1D)
-         {
-            if (a<D1D) { s_B[b][a] = B(b,a); }
-
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = 0.0; }
-
-            if (a<D1D && b<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int c=0; c<D1D; ++c)
-               {
-                  const int gid = map ? MAP(a,b,c,e) : 0;
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  s_q[c][b][a] = map ? X(idx) : X1(a,b,c,e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Interpolate in X direction
-      MFEM_FOREACH_THREAD(c,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,Q1D)
-         {
-            if (b<D1D && c<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int a=0; a<D1D; ++a)
-               {
-                  const double q_cba = s_q[c][b][a];
-                  MFEM_UNROLL(Q1D)
-                  for (int i=0; i<Q1D; ++i) { r_wk[i] += s_B[i][a]*q_cba; }
-               }
-               // reg => s_mem
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { s_q[c][b][i] = r_wk[i]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Interpolate in Y direction
-      MFEM_FOREACH_THREAD(c,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            if (c<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int b=0; b<D1D; ++b)
-               {
-                  const double q_cbi = s_q[c][b][i];
-                  MFEM_UNROLL(Q1D)
-                  for (int j=0; j<Q1D; ++j) { r_wk[j] += s_B[j][b]*q_cbi; }
-               }
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { s_q[c][j][i] = r_wk[j]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            // Interpolate in Z direction
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cji = s_q[c][j][i];
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { r_wk[k] += s_B[k][c]*q_cji; }
-            }
-
-            // Scale by Jacobian and integration weights
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] *= D(i,j,k,e); }
-
-            // Project back in Z direction
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               double q_cji = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { q_cji += s_B[k][c] * r_wk[k]; }
-               s_q[c][j][i] = q_cji;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Project back in Y direction
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b)
-            {
-               double q_cbi = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { q_cbi += s_B[j][b] * r_wk[j]; }
-               s_q[c][b][i] = q_cbi;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Project back in X direction
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,D1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a)
-            {
-               double q_cba = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { q_cba += s_B[i][a] * r_wk[i]; }
-               s_q[c][b][a] = q_cba;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Save back to memory
-      MFEM_FOREACH_THREAD(b,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cba = s_q[c][b][a];
-               const int gid = map ? MAP(a,b,c,e) : 0;
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               AtomicAdd(map?Y(idx):Y1(a,b,c,e), q_cba);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-void NDK_AMD_PAMassApply(const int dim,
-                         const int D1D,
-                         const int Q1D,
-                         const int NE,
-                         const FiniteElementSpace *fes,
-                         const DofToQuad *maps,
-                         const Vector &D,
-                         const Vector &X,
-                         Vector &Y)
-{
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
-   const int *map = ER ? ER->GatherMap().Read() : nullptr;
-   const double *b = maps->B.Read();
-   const double *d = D.Read();
-   const double *x = X.Read();
-   double *y = Y.ReadWrite();
-
-   assert(dim == 3);
-   const int ver = Device::KernelsVersion();
-   const int id = (ver << 8) | (D1D << 4) | Q1D;
-
-   //printf("\033[32mkernel #0x%x\033[m\n",id); fflush(0);
-
-   switch (id) // orders 1~6
-   {
-      // Fast '3': libP + AMD specific non-deterministic 3D mass kernel
-      case 0x323: return NDK_AMD_PAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
-      case 0x324: return NDK_AMD_PAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
-      case 0x334: return NDK_AMD_PAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
-      case 0x336: return NDK_AMD_PAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
-      case 0x345: return NDK_AMD_PAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
-      case 0x346: return NDK_AMD_PAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
-      case 0x348: return NDK_AMD_PAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
-      case 0x356: return NDK_AMD_PAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
-      case 0x358: return NDK_AMD_PAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
-      case 0x367: return NDK_AMD_PAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
-      case 0x378: return NDK_AMD_PAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
-
-      default: break;
-   }
-
-   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
-}
-
-} // namespace mfem

fem/bilininteg_mass_pa_fast_diag.cpp

@@ -1,269 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "gridfunc.hpp"
-#include "restriction.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// Fast '0' non-deterministic 3D mass diag kernel
-template<int D1D, int Q1D>
-static void NDK_SmemPAMassDiag3D(const int ndofs,
-                                 const int NE,
-                                 const int *map_,
-                                 const double *b_,
-                                 const double *d_,
-                                 double *y_)
-{
-   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      MFEM_SHARED double B[Q1D][D1D];
-      MFEM_SHARED double QQD[Q1D][Q1D][D1D];
-      MFEM_SHARED double QDD[Q1D][D1D][D1D];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
-               }
-               const int gid = MAP(dx, dy, dz, e);
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               AtomicAdd(Y(idx), t);
-            }
-         }
-      }
-   });
-}
-
-// Fast non-deterministic 3D mass diag kernel
-// Less smem version with registers
-template<int D1D, int Q1D>
-static void NDK_RegsPAMassDiag3D(const int ndofs,
-                                 const int NE,
-                                 const int *map_,
-                                 const double *b_,
-                                 const double *d_,
-                                 double *y_)
-{
-   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
-   const auto B = Reshape(b_, Q1D, D1D);
-   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_, ndofs);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      double r_wk[Q1D];
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
-
-
-      MFEM_FOREACH_THREAD(d,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            if (d<D1D) { s_B[q][d] = B(q,d); }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] = D(i,j,k,e); }
-            for (int c=0; c<D1D; ++c)
-            {
-               double q_cji = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k)
-               {
-                  const double Bkc = s_B[k][c];
-                  q_cji += Bkc * Bkc * r_wk[k];
-               }
-               s_q[c][j][i] = q_cji;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b)
-            {
-               double q_cbi = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j)
-               {
-                  const double Bjb = s_B[j][b];
-                  q_cbi += Bjb * Bjb * r_wk[j];
-               }
-               s_q[c][b][i] = q_cbi;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,D1D)
-         {
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a)
-            {
-               double q_cba = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i)
-               {
-                  const double Bia = s_B[i][a];
-                  q_cba += Bia * Bia * r_wk[i];
-               }
-               s_q[c][b][a] = q_cba;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(b,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cba = s_q[c][b][a];
-               const int gid = MAP(a,b,c,e);
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               AtomicAdd(Y(idx), q_cba);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-void NDK_PAMassAssembleDiagonal(const int dim,
-                                const int D1D,
-                                const int Q1D,
-                                const int NE,
-                                const FiniteElementSpace *fes,
-                                const DofToQuad *maps,
-                                const Vector &D,
-                                Vector &Y)
-{
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
-   assert(ER);
-   const int *map = ER->GatherMap().Read();
-   const double *b = maps->B.Read();
-   const double *d = D.Read();
-   double *y = Y.ReadWrite();
-
-   assert(dim == 3);
-   const int ver = Device::KernelsVersion();
-   const int vid = (ver << 8) | (D1D << 4) | Q1D;
-
-   // Fast '0' non-deterministic 3D mass diag kernel
-   switch (vid) // orders 1~6
-   {
-      case 0x023: return NDK_SmemPAMassDiag3D<2,3>(ND,NE,map,b,d,y);
-      case 0x024: return NDK_SmemPAMassDiag3D<2,4>(ND,NE,map,b,d,y);
-      case 0x034: return NDK_SmemPAMassDiag3D<3,4>(ND,NE,map,b,d,y);
-      case 0x045: return NDK_SmemPAMassDiag3D<4,5>(ND,NE,map,b,d,y);
-      case 0x046: return NDK_SmemPAMassDiag3D<4,6>(ND,NE,map,b,d,y);
-      case 0x056: return NDK_SmemPAMassDiag3D<5,6>(ND,NE,map,b,d,y);
-      case 0x058: return NDK_SmemPAMassDiag3D<5,8>(ND,NE,map,b,d,y);
-      case 0x067: return NDK_SmemPAMassDiag3D<6,7>(ND,NE,map,b,d,y);
-      case 0x078: return NDK_SmemPAMassDiag3D<7,8>(ND,NE,map,b,d,y);
-      default: break;
-   }
-
-   // all other versions should fold back through these kernels
-   switch ((D1D << 4) | Q1D)
-   {
-      case 0x23: return NDK_RegsPAMassDiag3D<2,3>(ND,NE,map,b,d,y);
-      case 0x24: return NDK_RegsPAMassDiag3D<2,4>(ND,NE,map,b,d,y);
-      case 0x34: return NDK_RegsPAMassDiag3D<3,4>(ND,NE,map,b,d,y);
-      case 0x45: return NDK_RegsPAMassDiag3D<4,5>(ND,NE,map,b,d,y);
-      case 0x46: return NDK_RegsPAMassDiag3D<4,6>(ND,NE,map,b,d,y);
-      case 0x56: return NDK_RegsPAMassDiag3D<5,6>(ND,NE,map,b,d,y);
-      case 0x58: return NDK_RegsPAMassDiag3D<5,8>(ND,NE,map,b,d,y);
-      case 0x67: return NDK_RegsPAMassDiag3D<6,7>(ND,NE,map,b,d,y);
-      case 0x78: return NDK_RegsPAMassDiag3D<7,8>(ND,NE,map,b,d,y);
-      default: break;
-   }
-
-   MFEM_ABORT("Unknown kernel 0x" << std::hex << vid);
-}
-
-} // namespace mfem

fem/bilininteg_mass_pa_fast_hip.cpp

@@ -1,224 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "gridfunc.hpp"
-#include "restriction.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// Fast '7' HIP
-template<int D1D, int Q1D, int NBZ=1, int NBK=1> MFEM_GLOBAL static
-//MFEM_LAUNCH_BOUNDS(Q1D*Q1D*NBZ,NBK)
-void HIP_PAMassApply(const int NE,
-                     const int* MAP,
-                     const double* B,
-                     const double* D,
-                     const double* X,
-                     double* Y)
-{
-   double u[Q1D];
-   const int tz = MFEM_THREAD_ID(z);
-   MFEM_SHARED double s_B[Q1D][D1D];
-   MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
-
-   for (int be = MFEM_BLOCK_ID(x); be < (NE+NBZ-1)/NBZ; be += MFEM_GRID_DIM(x))
-   {
-      const int e = be * NBZ + tz;
-      if (e>=NE) { return; }
-
-      // Load input, B & X interpolation
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            if (tz == 0) { s_B[qx][dy] = B[qx+Q1D*dy]; }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double Bx = B[qx+Q1D*dx];
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const int gid = MAP[e*D1D*D1D*D1D+dz*D1D*D1D+dy*D1D+dx];
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  u[dz] = fma(X[idx], Bx, u[dz]);
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Y interpolation
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double zyX = s_q[tz][dz][dy][qx];
-               MFEM_UNROLL(Q1D)
-               for (int qy = 0; qy < Q1D; ++qy) { u[qy] = fma(zyX,s_B[qy][dy],u[qy]); }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Z interpolation, Q-function & Zt projection
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            // Z interpolation
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const double zYX = s_q[tz][dz][qy][qx];
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = fma(zYX,s_B[qz][dz],u[qz]); }
-            }
-
-            // Q-function
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const int idx = e*Q1D*Q1D*Q1D + qx + qy*Q1D + qz*Q1D*Q1D;
-               s_q[tz][qz][qy][qx] = u[qz] * D[idx];
-            }
-
-            // Zt projection
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double ZYX = s_q[tz][qz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz) { u[dz] = fma(ZYX,s_B[qz][dz],u[dz]); }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Yt projection
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double zYX = s_q[tz][dz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dy = 0; dy < D1D; ++dy) { u[dy] = fma(zYX,s_B[qy][dy],u[dy]); }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Xt projection & save output
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double zyX = s_q[tz][dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dx = 0; dx < D1D; ++dx) { u[dx] = fma(zyX,s_B[qx][dx],u[dx]); }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double output = u[dx];
-               const int gid = MAP[e*D1D*D1D*D1D+dz*D1D*D1D+dy*D1D+dx];
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               AtomicAdd(Y[idx], output);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-}
-
-
-void NDK_HIP_PAMassApply(const int dim,
-                         const int D1D,
-                         const int Q1D,
-                         const int NE,
-                         const FiniteElementSpace *fes,
-                         const DofToQuad *maps,
-                         const Vector &d,
-                         const Vector &x,
-                         Vector &y)
-{
-   assert(dim == 3);
-
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction *ER = dynamic_cast<const ElementRestriction*>(ERop);
-   assert(ER);
-
-   const auto M = ER->GatherMap().Read();
-   const double *B = maps->B.Read();
-   const double *D = d.Read();
-   const double *X = x.Read();
-   double *Y = y.ReadWrite();
-
-   void (*Ker)(const int NE,
-               const int *M,
-               const double *B,
-               const double *D,
-               const double *X,
-               double *Y) = nullptr;
-
-   const int ver = Device::KernelsVersion();
-   const int id = (ver << 8) | (D1D << 4) | Q1D;
-
-   switch (id) // orders 1~8
-   {
-      case 0x723: Ker=HIP_PAMassApply<2,3>; break; // 1
-      case 0x734: Ker=HIP_PAMassApply<3,4>; break; // 2
-      case 0x745: Ker=HIP_PAMassApply<4,5>; break; // 3
-      case 0x756: Ker=HIP_PAMassApply<5,6>; break; // 4
-      case 0x767: Ker=HIP_PAMassApply<6,7>; break; // 5
-      case 0x778: Ker=HIP_PAMassApply<7,8>; break; // 6
-      case 0x789: Ker=HIP_PAMassApply<8,9>; break; // 7
-      case 0x79A: Ker=HIP_PAMassApply<9,10>; break; // 8
-      default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
-   }
-   MFEM_LAUNCH_KERNEL(Ker,NE,dim3(Q1D,Q1D,1),0,NE,M,B,D,X,Y);
-}
-
-} // namespace mfem

fem/bilininteg_vecmass.cpp

@@ -34,7 +34,7 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q, action_type);
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
       return;
    }
    dim = mesh->Dimension();

fem/ceed/mass.cpp

@@ -48,8 +48,7 @@ struct MassOperatorInfo : public OperatorInfo
 
 PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
                                    const mfem::IntegrationRule &irm,
-                                   mfem::Coefficient *Q,
-                                   mfem::ActionType &action_type)
+                                   mfem::Coefficient *Q)
    : PAIntegrator()
 {
 #ifdef MFEM_USE_CEED
@@ -58,7 +57,6 @@ PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
 #else
    MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
 #endif
-   action_type = mfem::ActionType::L2L;
 }
 
 MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,

fem/ceed/mass.hpp

@@ -27,8 +27,7 @@ class PAMassIntegrator : public PAIntegrator
 public:
    PAMassIntegrator(const mfem::FiniteElementSpace &fes,
                     const mfem::IntegrationRule &irm,
-                    mfem::Coefficient *Q,
-                    mfem::ActionType &action_type);
+                    mfem::Coefficient *Q);
 };
 
 /// Represent a MassIntegrator with AssemblyLevel::None using libCEED.

fem/ceed/util.cpp

@@ -65,8 +65,6 @@ void RemoveBasisAndRestriction(const mfem::FiniteElementSpace *fes)
          itr++;
       }
    }
-#else
-   MFEM_CONTRACT_VAR(fes);
 #endif
 }
 

fem/coefficient.cpp

@@ -21,33 +21,6 @@ namespace mfem
 
 using namespace std;
 
-// Given an ElementTransformation and IntegrationPoint in a refined mesh,
-// return the ElementTransformation of the parent coarse element, and set
-// coarse_ip to the location of the original ip within the coarse element.
-ElementTransformation *RefinedToCoarse(
-   Mesh &coarse_mesh, const ElementTransformation &T,
-   const IntegrationPoint &ip, IntegrationPoint &coarse_ip)
-{
-   Mesh &fine_mesh = *T.mesh;
-   // Get the element transformation of the coarse element containing the
-   // fine element.
-   int fine_element = T.ElementNo;
-   const CoarseFineTransformations &cf = fine_mesh.GetRefinementTransforms();
-   int coarse_element = cf.embeddings[fine_element].parent;
-   ElementTransformation *coarse_T = coarse_mesh.GetElementTransformation(
-                                        coarse_element);
-   // Transform the integration point from fine element coordinates to coarse
-   // element coordinates.
-   Geometry::Type geom = T.GetGeometryType();
-   IntegrationPointTransformation fine_to_coarse;
-   IsoparametricTransformation &emb_tr = fine_to_coarse.Transf;
-   emb_tr.SetIdentityTransformation(geom);
-   emb_tr.SetPointMat(cf.point_matrices[geom](cf.embeddings[fine_element].matrix));
-   fine_to_coarse.Transform(ip, coarse_ip);
-   coarse_T->SetIntPoint(&coarse_ip);
-   return coarse_T;
-}
-
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -122,17 +95,7 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
 double GridFunctionCoefficient::Eval (ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
-   Mesh *gf_mesh = GridF->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
-   {
-      return GridF->GetValue(T, ip, Component);
-   }
-   else
-   {
-      IntegrationPoint coarse_ip;
-      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
-      return GridF->GetValue(*coarse_T, coarse_ip, Component);
-   }
+   return GridF -> GetValue (T, ip, Component);
 }
 
 void TransformedCoefficient::SetTime(double t)
@@ -342,30 +305,13 @@ void VectorGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
-   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
-   {
-      GridFunc->GetVectorValue(T, ip, V);
-   }
-   else
-   {
-      IntegrationPoint coarse_ip;
-      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
-      GridFunc->GetVectorValue(*coarse_T, coarse_ip, V);
-   }
+   GridFunc->GetVectorValue(T, ip, V);
 }
 
 void VectorGridFunctionCoefficient::Eval(
    DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir)
 {
-   if (T.mesh == GridFunc->FESpace()->GetMesh())
-   {
-      GridFunc->GetVectorValues(T, ir, M);
-   }
-   else
-   {
-      VectorCoefficient::Eval(M, T, ir);
-   }
+   GridFunc->GetVectorValues(T, ir, M);
 }
 
 GradientGridFunctionCoefficient::GradientGridFunctionCoefficient (
@@ -385,30 +331,13 @@ void GradientGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void GradientGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                            const IntegrationPoint &ip)
 {
-   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
-   {
-      GridFunc->GetGradient(T, V);
-   }
-   else
-   {
-      IntegrationPoint coarse_ip;
-      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
-      GridFunc->GetGradient(*coarse_T, V);
-   }
+   GridFunc->GetGradient(T, V);
 }
 
 void GradientGridFunctionCoefficient::Eval(
    DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir)
 {
-   if (T.mesh == GridFunc->FESpace()->GetMesh())
-   {
-      GridFunc->GetGradients(T, ir, M);
-   }
-   else
-   {
-      VectorCoefficient::Eval(M, T, ir);
-   }
+   GridFunc->GetGradients(T, ir, M);
 }
 
 CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
@@ -434,17 +363,7 @@ void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                        const IntegrationPoint &ip)
 {
-   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
-   {
-      GridFunc->GetCurl(T, V);
-   }
-   else
-   {
-      IntegrationPoint coarse_ip;
-      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
-      GridFunc->GetCurl(*coarse_T, V);
-   }
+   GridFunc->GetCurl(T, V);
 }
 
 DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
@@ -456,17 +375,7 @@ DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
 double DivergenceGridFunctionCoefficient::Eval(ElementTransformation &T,
                                                const IntegrationPoint &ip)
 {
-   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
-   {
-      return GridFunc->GetDivergence(T);
-   }
-   else
-   {
-      IntegrationPoint coarse_ip;
-      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
-      return GridFunc->GetDivergence(*coarse_T);
-   }
+   return GridFunc->GetDivergence(T);
 }
 
 void VectorDeltaCoefficient::SetTime(double t)

fem/coefficient.hpp

@@ -574,7 +574,6 @@ public:
    /// Evaluate the coefficient.
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
-   using VectorCoefficient::Eval;
 };
 
 /// A general vector function coefficient

fem/conduitdatacollection.cpp

@@ -330,7 +330,7 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
          }
          else
          {
-            Node &n_bndry_conn_conv =
+            Node &(n_bndry_conn_conv) =
                n_conv["topologies"][bndry_topo_name]["elements/connectivity"];
             n_bndry_conn.to_int_array(n_bndry_conn_conv);
             bndry_indices = (n_bndry_conn_conv).value();

fem/doftrans.hpp

@@ -243,7 +243,6 @@ public:
    void TransformDual(double *v) const;
 
    void InvTransformDual(double *v) const;
-   using DofTransformation::InvTransformDual;
 };
 
 /// DoF transformation implementation for the Nedelec basis on tetrahedra

fem/eltrans.cpp

@@ -21,8 +21,7 @@ ElementTransformation::ElementTransformation()
      EvalState(0),
      geom(Geometry::INVALID),
      Attribute(-1),
-     ElementNo(-1),
-     mesh(nullptr)
+     ElementNo(-1)
 { }
 
 double ElementTransformation::EvalWeight()

fem/eltrans.hpp

@@ -75,12 +75,6 @@ public:
 
    int Attribute, ElementNo, ElementType;
 
-   /// The Mesh object containing the element.
-   /** If the element transformation belongs to a mesh, this will point to the
-       containing Mesh object. ElementNo will be the number of the element in
-       this Mesh. This will be NULL if the element does not belong to a mesh. */
-   class Mesh *mesh;
-
    ElementTransformation();
 
    /** @brief Force the reevaluation of the Jacobian in the next call. */

fem/fe/fe_base.hpp

@@ -749,12 +749,11 @@ public:
                            DenseMatrix &div) const;
 
    /** @brief Get an Array<int> that maps lexicographically ordered indices to
-       the indices of the respective nodes/dofs/basis functions.
-
-       Lexicographic ordering of nodes is defined in terms of reference-space
-       coordinates (x,y,z). Lexicographically ordered nodes are listed first in
-       order of increasing x-coordinate, and then in order of increasing
-       y-coordinate, and finally in order of increasing z-coordinate.
+       the indices of the respective nodes/dofs/basis functions. Lexicographic
+       ordering of nodes is defined in terms of reference-space coordinates
+       (x,y,z). Lexicographically ordered nodes are listed first in order of
+       increasing x-coordinate, and then in order of increasing y-coordinate,
+       and finally in order of increasing z-coordinate.
 
        For example, the six nodes of a quadratic triangle are lexicographically
        ordered as follows:

fem/fespace.cpp

@@ -561,6 +561,155 @@ void FiniteElementSpace::GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
    }
 }
 
+void FiniteElementSpace::GetEssentialVDofs(EntitySets::EntityType type,
+                                           int set_index,
+                                           Array<int> &ess_vdofs,
+                                           int component) const
+{
+   Array<int> vdofs, dofs;
+
+   ess_vdofs.SetSize(GetVSize());
+   ess_vdofs = 0;
+
+   MFEM_VERIFY(mesh->ent_sets != NULL, "Mesh object contains no "
+               "entity set information");
+   if (!mesh->ent_sets->SetExists(type, set_index))
+   {
+      ostringstream oss; oss << "Entity set of type \""
+                             << EntitySets::GetTypeName(type)
+                             << "\" and index " << set_index
+                             << " was not found.";
+
+      MFEM_VERIFY(false, oss.str().c_str());
+   }
+
+   set<int>::iterator it;
+   for (it=(*mesh->ent_sets)(type, set_index).begin();
+        it!=(*mesh->ent_sets)(type, set_index).end(); it++)
+   {
+      int ent_index = *it;
+      cout << "collecting vdofs for entity " << ent_index << "->";
+      if (component < 0)
+      {
+         switch (type)
+         {
+            case EntitySets::VERTEX:
+               GetVertexVDofs(ent_index, vdofs);
+               break;
+            case EntitySets::EDGE:
+               GetEdgeVDofs(ent_index, vdofs);
+               break;
+            case EntitySets::FACE:
+               GetFaceVDofs(ent_index, vdofs);
+               break;
+            case EntitySets::ELEMENT:
+               GetElementVDofs(ent_index, vdofs);
+               break;
+            default:
+               mfem_error("GetEssentialVDofs: Invalid entity type");
+         }
+         vdofs.Print(cout);
+         mark_dofs(vdofs, ess_vdofs);
+      }
+      else
+      {
+         switch (type)
+         {
+            case EntitySets::VERTEX:
+               GetVertexDofs(ent_index, dofs);
+               break;
+            case EntitySets::EDGE:
+               GetEdgeDofs(ent_index, dofs);
+               break;
+            case EntitySets::FACE:
+               GetFaceDofs(ent_index, dofs);
+               break;
+            case EntitySets::ELEMENT:
+               GetElementDofs(ent_index, dofs);
+               break;
+            default:
+               mfem_error("GetEssentialDofs: Invalid entity type");
+         }
+         for (int d = 0; d < dofs.Size(); d++)
+         { dofs[d] = DofToVDof(dofs[d], component); }
+         mark_dofs(dofs, ess_vdofs);
+      }
+   }
+
+   if (mesh->ncmesh)
+   {
+      Array<int> es_verts, es_edges, es_faces;
+      mesh->ncmesh->GetEntitySetClosure(type, set_index,
+                                        es_verts, es_edges, es_faces);
+      cout << "returned from get closure" << endl;
+      for (int i = 0; i < es_verts.Size(); i++)
+      {
+         if (es_verts[i] < GetNV())
+         {
+            if (component < 0)
+            {
+               GetVertexVDofs(es_verts[i], vdofs);
+               mark_dofs(vdofs, ess_vdofs);
+            }
+            else
+            {
+               GetVertexDofs(es_verts[i], dofs);
+               for (int d = 0; d < dofs.Size(); d++)
+               { dofs[d] = DofToVDof(dofs[d], component); }
+               mark_dofs(dofs, ess_vdofs);
+            }
+         }
+      }
+      for (int i = 0; i < es_edges.Size(); i++)
+      {
+         if (es_edges[i] < GetMesh()->GetNEdges())
+         {
+            if (component < 0)
+            {
+               GetEdgeVDofs(es_edges[i], vdofs);
+               mark_dofs(vdofs, ess_vdofs);
+            }
+            else
+            {
+               GetEdgeDofs(es_edges[i], dofs);
+               for (int d = 0; d < dofs.Size(); d++)
+               { dofs[d] = DofToVDof(dofs[d], component); }
+               mark_dofs(dofs, ess_vdofs);
+            }
+         }
+      }
+      for (int i = 0; i < es_faces.Size(); i++)
+      {
+         if (es_faces[i] < GetMesh()->GetNFaces())
+         {
+            if (component < 0)
+            {
+               GetFaceVDofs(es_faces[i], vdofs);
+               mark_dofs(vdofs, ess_vdofs);
+            }
+            else
+            {
+               GetFaceDofs(es_faces[i], dofs);
+               for (int d = 0; d < dofs.Size(); d++)
+               { dofs[d] = DofToVDof(dofs[d], component); }
+               mark_dofs(dofs, ess_vdofs);
+            }
+         }
+      }
+   }
+}
+
+void FiniteElementSpace::GetEssentialVDofs(EntitySets::EntityType type,
+                                           const string & set_name,
+                                           Array<int> &ess_vdofs,
+                                           int component) const
+{
+   MFEM_VERIFY(mesh->ent_sets != NULL, "Mesh object contains no "
+               "entity set information");
+   GetEssentialVDofs(type, mesh->ent_sets->GetSetIndex(type, set_name),
+                     ess_vdofs, component);
+}
+
 void FiniteElementSpace::GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
                                               Array<int> &ess_tdof_list,
                                               int component)
@@ -579,6 +728,36 @@ void FiniteElementSpace::GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
    MarkerToList(ess_tdofs, ess_tdof_list);
 }
 
+void FiniteElementSpace::GetEssentialTrueDofs(EntitySets::EntityType type,
+                                              int set_index,
+                                              Array<int> &ess_tdof_list,
+                                              int component)
+{
+   Array<int> ess_vdofs, ess_tdofs;
+   GetEssentialVDofs(type, set_index, ess_vdofs, component);
+   const SparseMatrix *R = GetConformingRestriction();
+   if (!R)
+   {
+      ess_tdofs.MakeRef(ess_vdofs);
+   }
+   else
+   {
+      R->BooleanMult(ess_vdofs, ess_tdofs);
+   }
+   MarkerToList(ess_tdofs, ess_tdof_list);
+}
+
+void FiniteElementSpace::GetEssentialTrueDofs(EntitySets::EntityType type,
+                                              const string & set_name,
+                                              Array<int> &ess_tdof_list,
+                                              int component)
+{
+   MFEM_VERIFY(mesh->ent_sets != NULL, "Mesh object contains no "
+               "entity set information");
+   GetEssentialTrueDofs(type, mesh->ent_sets->GetSetIndex(type, set_name),
+                        ess_tdof_list, component);
+}
+
 void FiniteElementSpace::GetBoundaryTrueDofs(Array<int> &boundary_dofs,
                                              int component)
 {

fem/fespace.hpp

@@ -778,6 +778,19 @@ public:
                                   Array<int> &ess_vdofs,
                                   int component = -1) const;
 
+   /** Mark degrees of freedom associated with the entity set with the
+       specified entity type and set index. */
+   virtual void GetEssentialVDofs(EntitySets::EntityType type, int set_index,
+                                  Array<int> &ess_vdofs,
+                                  int component = -1) const;
+
+   /** Mark degrees of freedom associated with the entity set with the
+       specified entity type and set index. */
+   virtual void GetEssentialVDofs(EntitySets::EntityType type,
+                                  const std::string & set_name,
+                                  Array<int> &ess_vdofs,
+                                  int component = -1) const;
+
    /** @brief Get a list of essential true dofs, ess_tdof_list, corresponding to the
        boundary attributes marked in the array bdr_attr_is_ess.
        For spaces with 'vdim' > 1, the 'component' parameter can be used
@@ -786,6 +799,19 @@ public:
                                      Array<int> &ess_tdof_list,
                                      int component = -1);
 
+   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
+       entity set specified by the given entity type and set index. */
+   virtual void GetEssentialTrueDofs(EntitySets::EntityType type, int set_index,
+                                     Array<int> &ess_tdof_list,
+                                     int component = -1);
+
+   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
+       entity set specified by the given entity type and set name. */
+   virtual void GetEssentialTrueDofs(EntitySets::EntityType type,
+                                     const std::string & set_name,
+                                     Array<int> &ess_tdof_list,
+                                     int component = -1);
+
    /** @brief Get a list of all boundary true dofs, @a boundary_dofs. For spaces
        with 'vdim' > 1, the 'component' parameter can be used to restricts the
        marked tDOFs to the specified component. Equivalent to

fem/fmsconvert.cpp

@@ -112,7 +112,7 @@ FmsFieldToGridFunction(FmsMesh fms_mesh, FmsField f, Mesh *mesh,
    // NOTE: transplanted from the FmsMeshToMesh function
    //       We should do this work once and save it.
    //--------------------------------------------------
-   FmsInt dim, n_elem, space_dim;
+   FmsInt dim, n_vert, n_elem, space_dim;
 
    // Find the first component that has coordinates - that will be the new mfem
    // mesh.
@@ -144,6 +144,7 @@ FmsFieldToGridFunction(FmsMesh fms_mesh, FmsField f, Mesh *mesh,
          n_ents[et] += num_ents;
       }
    }
+   n_vert = n_ents[FMS_VERTEX];
    //--------------------------------------------------
 
    // Interrogate the field.

fem/kernels.hpp

@@ -48,26 +48,6 @@ MFEM_HOST_DEVICE inline void LoadB(const int D1D, const int Q1D,
    MFEM_SYNC_THREAD;
 }
 
-/// Load B1d matrice into shared memory
-MFEM_HOST_DEVICE inline void LoadB(const int D1D, const int Q1D,
-                                   const ConstDeviceMatrix &b,
-                                   const DeviceMatrix &B)
-{
-   const int tidz = MFEM_THREAD_ID(z);
-
-   if (tidz == 0)
-   {
-      MFEM_FOREACH_THREAD(d,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            B(q,d) = b(q,d);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
 /// Load Bt1d matrices into shared memory
 template<int MD1, int MQ1>
 MFEM_HOST_DEVICE inline void LoadBt(const int D1D, const int Q1D,
@@ -140,29 +120,6 @@ MFEM_HOST_DEVICE inline void LoadBGt(const int D1D, const int Q1D,
    MFEM_SYNC_THREAD;
 }
 
-/// Load Bt1d & Gt1d matrices into shared memory w/o MAX
-MFEM_HOST_DEVICE inline void LoadBGt(const int D1D, const int Q1D,
-                                     const ConstDeviceMatrix &b,
-                                     const ConstDeviceMatrix &g,
-                                     const DeviceMatrix &Bt,
-                                     const DeviceMatrix &Gt)
-{
-   const int tidz = MFEM_THREAD_ID(z);
-
-   if (tidz == 0)
-   {
-      MFEM_FOREACH_THREAD(d,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            Bt(d,q) = b(q,d);
-            Gt(d,q) = g(q,d);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
 /// Load 2D input scalar into shared memory
 template<int MD1, int NBZ>
 MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
@@ -671,141 +628,6 @@ MFEM_HOST_DEVICE inline void GradXt(const int D1D, const int Q1D,
    MFEM_SYNC_THREAD;
 }
 
-/// Atomic 2D Transposed Evaluation, 1/2
-MFEM_HOST_DEVICE inline void Atomic2DEvalYt(const int D1D, const int Q1D,
-                                            const DeviceMatrix &B,
-                                            const DeviceMatrix &QQ,
-                                            const DeviceMatrix &QD)
-{
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u = 0.0;
-         for (int qx = 0; qx < Q1D; ++qx) { u += QQ(qy,qx) * B(qx,dx); }
-         QD(qy,dx) = u;
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-/// Atomic 2D Transposed Evaluation, 2/2
-MFEM_HOST_DEVICE inline void Atomic2DEvalXt(const int D1D, const int Q1D,
-                                            const DeviceMatrix &B,
-                                            const DeviceMatrix &Q,
-                                            const DeviceTensor<3,const int> &I,
-                                            const DeviceMatrix &Y,
-                                            const int c,
-                                            const int e,
-                                            const bool byVDIM)
-{
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u = 0.0;
-         for (int qy = 0; qy < Q1D; ++qy) { u += Q(qy,dx) * B(qy,dy); }
-         const int gid = I(dx,dy,e);
-         const int idx = gid >= 0 ? gid : -1 - gid;
-         if (byVDIM) { AtomicAdd(Y(c,idx), u); }
-         else { AtomicAdd(Y(idx,c), u); }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-MFEM_HOST_DEVICE inline
-void Atomic2DEvalTranspose(const int D1D,
-                           const int Q1D,
-                           const DeviceMatrix &B,
-                           const DeviceMatrix &QQ,
-                           const DeviceMatrix &QD,
-                           const DeviceTensor<3,const int> &I,
-                           const DeviceMatrix &Y,
-                           const int c,
-                           const int e,
-                           const bool byVDIM)
-{
-   kernels::internal::Atomic2DEvalYt(D1D,Q1D,B,QQ,QD);
-   kernels::internal::Atomic2DEvalXt(D1D,Q1D,B,QD,I,Y,c,e,byVDIM);
-}
-
-/// Atomic 2D Transposed Gradient, 1/2
-MFEM_HOST_DEVICE inline void AtomicGradYt(const int D1D, const int Q1D,
-                                          const DeviceMatrix &Bt,
-                                          const DeviceMatrix &Gt,
-                                          const DeviceMatrix &QQ0,
-                                          const DeviceMatrix &QQ1,
-                                          const DeviceMatrix &DQ0,
-                                          const DeviceMatrix &DQ1)
-{
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u = 0.0, v = 0.0;
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            u += Gt(dx,qx) * QQ0(qy,qx);
-            v += Bt(dx,qx) * QQ1(qy,qx);
-         }
-         DQ0(dx,qy) = u;
-         DQ1(dx,qy) = v;
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-/// Atomic 2D Transposed Gradient, 2/2
-MFEM_HOST_DEVICE inline void AtomicGradXt(const int D1D, const int Q1D,
-                                          const DeviceMatrix &Bt,
-                                          const DeviceMatrix &Gt,
-                                          const DeviceMatrix &DQ0,
-                                          const DeviceMatrix &DQ1,
-                                          const DeviceTensor<3,const int> &I,
-                                          const DeviceMatrix &Y,
-                                          const int c,
-                                          const int e,
-                                          const bool byVDIM)
-{
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u = 0.0, v = 0.0;
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            u += DQ0(dx,qy) * Bt(dy,qy);
-            v += DQ1(dx,qy) * Gt(dy,qy);
-         }
-         const double sum = u + v;
-         const int gid = I(dx,dy,e);
-         const int idx = gid >= 0 ? gid : -1-gid;
-         if (byVDIM) { AtomicAdd(Y(c,idx), sum); }
-         else { AtomicAdd(Y(idx,c), sum); }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-MFEM_HOST_DEVICE inline
-void Atomic2DGradTranspose(const int D1D, const int Q1D,
-                           const DeviceMatrix &Bt,
-                           const DeviceMatrix &Gt,
-                           const DeviceMatrix &QQ0,
-                           const DeviceMatrix &QQ1,
-                           const DeviceMatrix &DQ0,
-                           const DeviceMatrix &DQ1,
-                           const DeviceTensor<3,const int> &I,
-                           const DeviceMatrix &Y,
-                           const int c,
-                           const int e,
-                           const bool byVDIM)
-{
-   kernels::internal::AtomicGradYt(D1D,Q1D,Bt,Gt,QQ0,QQ1,DQ0,DQ1);
-   kernels::internal::AtomicGradXt(D1D,Q1D,Bt,Gt,DQ0,DQ1,I,Y,c,e,byVDIM);
-}
-
 /// Load 3D scalar input vector into shared memory
 MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
                                    const DeviceTensor<4, const double> &x,
@@ -1721,269 +1543,6 @@ MFEM_HOST_DEVICE inline void GradXt(const int D1D, const int Q1D,
    }
 }
 
-/// Atomic 3D Transposed Evaluation, 1/3
-MFEM_HOST_DEVICE inline void Atomic3DEvalZt(const int D1D, const int Q1D,
-                                            double *u,
-                                            const DeviceMatrix &B,
-                                            const DeviceCube &Q)
-{
-   MFEM_FOREACH_THREAD(qx,x,Q1D)
-   {
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const double ZYX = Q(qz,qy,qx);
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * B(qz,dz); }
-         }
-         for (int dz = 0; dz < D1D; ++dz) { Q(dz,qy,qx) = u[dz]; }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-/// Atomic 3D Transposed Evaluation, 2/3
-MFEM_HOST_DEVICE inline void Atomic3DEvalYt(const int D1D, const int Q1D,
-                                            double *u,
-                                            const DeviceMatrix &B,
-                                            const DeviceCube &Q)
-{
-   MFEM_FOREACH_THREAD(dz,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double zYX = Q(dz,qy,qx);
-            for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * B(qy,dy); }
-         }
-         for (int dy = 0; dy < D1D; ++dy) { Q(dz,dy,qx) = u[dy]; }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-/// Atomic 3D Transposed Evaluation, 3/3
-MFEM_HOST_DEVICE inline void Atomic3DEvalXt(const int D1D, const int Q1D,
-                                            double *u,
-                                            const DeviceMatrix &B,
-                                            const DeviceCube &Q,
-                                            const DeviceTensor<4,const int> &I,
-                                            const DeviceMatrix &Y,
-                                            const int c,
-                                            const int e,
-                                            const bool byVDIM)
-{
-   MFEM_FOREACH_THREAD(dz,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dy,x,D1D)
-      {
-         for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double zyX = Q(dz,dy,qx);
-            for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * B(qx,dx); }
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const double val = u[dx];
-            const int gid = I(dx,dy,dz,e);
-            const int idx = gid >= 0 ? gid : -1 - gid;
-            if (byVDIM) { AtomicAdd(Y(c,idx), val); }
-            else { AtomicAdd(Y(idx,c), val); }
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-MFEM_HOST_DEVICE inline
-void Atomic3DEvalTranspose(const int D1D,
-                           const int Q1D,
-                           double *u,
-                           const DeviceMatrix &B,
-                           const DeviceCube &Q,
-                           const DeviceTensor<4,const int> &I,
-                           const DeviceMatrix &Y,
-                           const int c,
-                           const int e,
-                           const bool byVDIM)
-{
-   kernels::internal::Atomic3DEvalZt(D1D,Q1D,u,B,Q);
-   kernels::internal::Atomic3DEvalYt(D1D,Q1D,u,B,Q);
-   kernels::internal::Atomic3DEvalXt(D1D,Q1D,u,B,Q,I,Y,c,e,byVDIM);
-}
-
-// Half of B and G are stored in shared to get B, Bt, G and Gt.
-// Indices computation for SmemPADiffusionApply3D.
-static MFEM_HOST_DEVICE inline int qi(const int q, const int d, const int Q)
-{
-   return (q<=d) ? q : Q-1-q;
-}
-
-static MFEM_HOST_DEVICE inline int dj(const int q, const int d, const int D)
-{
-   return (q<=d) ? d : D-1-d;
-}
-
-static MFEM_HOST_DEVICE inline int qk(const int q, const int d, const int Q)
-{
-   return (q<=d) ? Q-1-q : q;
-}
-
-static MFEM_HOST_DEVICE inline int dl(const int q, const int d, const int D)
-{
-   return (q<=d) ? D-1-d : d;
-}
-
-static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
-{
-   return (q<=d) ? -1.0 : 1.0;
-}
-
-/// Atomic 3D Transposed Gradient, 1/3
-MFEM_HOST_DEVICE inline void AtomicGradZt(const int D1D, const int Q1D,
-                                          const DeviceMatrix &Bt,
-                                          const DeviceMatrix &Gt,
-                                          const DeviceCube &QQQ0,
-                                          const DeviceCube &QQQ1,
-                                          const DeviceCube &QQQ2,
-                                          const DeviceCube &QQD0,
-                                          const DeviceCube &QQD1,
-                                          const DeviceCube &QQD2)
-{
-   MFEM_FOREACH_THREAD(qz,z,Q1D)
-   {
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u = 0.0, v = 0.0, w = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
-               u += QQQ0(qz,qy,qx) * Gt(l,k) * s;
-               v += QQQ1(qz,qy,qx) * Bt(j,i);
-               w += QQQ2(qz,qy,qx) * Bt(j,i);
-            }
-            QQD0(qz,qy,dx) = u;
-            QQD1(qz,qy,dx) = v;
-            QQD2(qz,qy,dx) = w;
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-/// Atomic 3D Transposed Gradient, 2/3
-MFEM_HOST_DEVICE inline void AtomicGradYt(const int D1D, const int Q1D,
-                                          const DeviceMatrix &Bt,
-                                          const DeviceMatrix &Gt,
-                                          const DeviceCube &QQD0,
-                                          const DeviceCube &QQD1,
-                                          const DeviceCube &QQD2,
-                                          const DeviceCube &QDD0,
-                                          const DeviceCube &QDD1,
-                                          const DeviceCube &QDD2)
-{
-   MFEM_FOREACH_THREAD(qz,z,Q1D)
-   {
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u = 0.0, v = 0.0, w = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               u += QQD0(qz,qy,dx) * Bt(j,i);
-               v += QQD1(qz,qy,dx) * Gt(l,k) * s;
-               w += QQD2(qz,qy,dx) * Bt(j,i);
-            }
-            QDD0(qz,dy,dx) = u;
-            QDD1(qz,dy,dx) = v;
-            QDD2(qz,dy,dx) = w;
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-/// Atomic 3D Transposed Gradient, 3/3
-MFEM_HOST_DEVICE inline void AtomicGradXt(const int D1D, const int Q1D,
-                                          const DeviceMatrix &Bt,
-                                          const DeviceMatrix &Gt,
-                                          const DeviceCube &QDD0,
-                                          const DeviceCube &QDD1,
-                                          const DeviceCube &QDD2,
-                                          const DeviceTensor<4,const int> &I,
-                                          const DeviceMatrix &Y,
-                                          const int c,
-                                          const int e,
-                                          const bool byVDIM)
-{
-   MFEM_FOREACH_THREAD(dz,z,D1D)
-   {
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u = 0.0, v = 0.0, w = 0.0;
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const int i = qi(qz,dz,Q1D);
-               const int j = dj(qz,dz,D1D);
-               const int k = qk(qz,dz,Q1D);
-               const int l = dl(qz,dz,D1D);
-               const double s = sign(qz,dz);
-               u += QDD0(qz,dy,dx) * Bt(j,i);
-               v += QDD1(qz,dy,dx) * Bt(j,i);
-               w += QDD2(qz,dy,dx) * Gt(l,k) * s;
-            }
-            const double sum = u + v + w;
-            const int gid = I(dx,dy,dz,e);
-            const int idx = gid >= 0 ? gid : -1-gid;
-            if (byVDIM) { AtomicAdd(Y(c,idx), sum); }
-            else { AtomicAdd(Y(idx,c), sum); }
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-MFEM_HOST_DEVICE inline void Atomic3DGrad(const int D1D, const int Q1D,
-                                          const DeviceMatrix &Bt,
-                                          const DeviceMatrix &Gt,
-                                          const DeviceCube &QQ0,
-                                          const DeviceCube &QQ1,
-                                          const DeviceCube &QQ2,
-                                          const DeviceCube &QD0,
-                                          const DeviceCube &QD1,
-                                          const DeviceCube &QD2,
-                                          const DeviceCube &DD0,
-                                          const DeviceCube &DD1,
-                                          const DeviceCube &DD2,
-                                          const DeviceTensor<4,const int> &I,
-                                          const DeviceMatrix &Y,
-                                          const int c,
-                                          const int e,
-                                          const bool byVDIM)
-{
-   kernels::internal::AtomicGradZt(D1D,Q1D,Bt,Gt,QQ0,QQ1,QQ2,QD0,QD1,QD2);
-   kernels::internal::AtomicGradYt(D1D,Q1D,Bt,Gt,QD0,QD1,QD2,DD0,DD1,DD2);
-   kernels::internal::AtomicGradXt(D1D,Q1D,Bt,Gt,DD0,DD1,DD2,I,Y,c,e,byVDIM);
-}
-
 } // namespace kernels::internal
 
 } // namespace kernels

fem/linearform.cpp

@@ -23,7 +23,6 @@ LinearForm::LinearForm(FiniteElementSpace *f, LinearForm *lf)
    UseDevice(true);
 
    fes = f;
-   ext = nullptr;
    extern_lfs = 1;
 
    // Copy the pointers to the integrators
@@ -70,14 +69,14 @@ void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi,
 
 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi)
 {
-   boundary_integs.Append(lfi);
+   boundary_integs.Append (lfi);
    boundary_integs_marker.Append(NULL); // NULL -> all attributes are active
 }
 
 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi,
                                         Array<int> &bdr_attr_marker)
 {
-   boundary_integs.Append(lfi);
+   boundary_integs.Append (lfi);
    boundary_integs_marker.Append(&bdr_attr_marker);
 }
 
@@ -100,35 +99,8 @@ void LinearForm::AddInteriorFaceIntegrator(LinearFormIntegrator *lfi)
    interior_face_integs.Append(lfi);
 }
 
-void LinearForm::SetAssemblyLevel(LinearAssemblyLevel assembly_level)
-{
-   if (ext)
-   {
-      MFEM_ABORT("the assembly level has already been set!");
-   }
-   assembly = assembly_level;
-   switch (assembly)
-   {
-      case LinearAssemblyLevel::LEGACY:
-         break;
-      case LinearAssemblyLevel::FULL:
-         ext = new FullLinearFormExtension(this);
-         break;
-      default:
-         mfem_error("Unknown assembly level");
-   }
-}
-
 void LinearForm::Assemble()
 {
-   Vector::operator=(0.0);
-
-   // The above operation is executed on device because of UseDevice().
-   // The first use of AddElementVector() below will move it back to host
-   // because both 'vdofs' and 'elemvect' are on host.
-
-   if (ext) { return ext->Assemble(); }
-
    Array<int> vdofs;
    ElementTransformation *eltrans;
    DofTransformation *doftrans;
@@ -136,6 +108,12 @@ void LinearForm::Assemble()
 
    int i;
 
+   Vector::operator=(0.0);
+
+   // The above operation is executed on device because of UseDevice().
+   // The first use of AddElementVector() below will move it back to host
+   // because both 'vdofs' and 'elemvect' are on host.
+
    if (domain_integs.Size())
    {
       for (int k = 0; k < domain_integs.Size(); k++)
@@ -165,7 +143,7 @@ void LinearForm::Assemble()
                {
                   doftrans->TransformDual(elemvect);
                }
-               AddElementVector(vdofs, elemvect);
+               AddElementVector (vdofs, elemvect);
             }
          }
       }
@@ -296,18 +274,6 @@ void LinearForm::Assemble()
    }
 }
 
-void LinearForm::Update()
-{
-   SetSize(fes->GetVSize()); ResetDeltaLocations();
-   if (ext) { ext->Update(); }
-}
-
-void LinearForm::Update(FiniteElementSpace *f)
-{
-   fes = f;
-   Update();
-}
-
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
    MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
@@ -315,7 +281,6 @@ void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
    v.UseDevice(true);
    this->Vector::MakeRef(v, v_offset, fes->GetVSize());
    ResetDeltaLocations();
-   if (ext) { ext->Update(); }
 }
 
 void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
@@ -327,8 +292,6 @@ void LinearForm::AssembleDelta()
 {
    if (domain_delta_integs.Size() == 0) { return; }
 
-   if (ext) { return ext->AssembleDelta(); }
-
    if (!HaveDeltaLocations())
    {
       int sdim = fes->GetMesh()->SpaceDimension();
@@ -393,8 +356,6 @@ LinearForm::~LinearForm()
       for (k=0; k < interior_face_integs.Size(); k++)
       { delete interior_face_integs[k]; }
    }
-
-   delete ext;
 }
 
 }

fem/linearform.hpp

@@ -14,39 +14,18 @@
 
 #include "../config/config.hpp"
 #include "lininteg.hpp"
-#include "linearform_ext.hpp"
 #include "gridfunc.hpp"
 
 namespace mfem
 {
 
-/** @brief Enumeration defining the assembly level for linear form classes
- *  derived from Vector. */
-enum class LinearAssemblyLevel
-{
-   /// Legacy assembled form, i.e. a global vector in MFEM format.
-   /// This assembly level is ALWAYS performed on the host.
-   LEGACY = 0,
-   /// Fully assembled form, i.e. a global vector in MFEM format.
-   /// This assembly is compatible with device execution.
-   FULL
-};
-
 /// Vector with associated FE space and LinearFormIntegrators.
 class LinearForm : public Vector
 {
-   friend LinearFormExtension;
-
 protected:
    /// FE space on which the LinearForm lives. Not owned.
    FiniteElementSpace *fes;
 
-   /// The assembly level of the form (legacy or full)
-   LinearAssemblyLevel assembly;
-
-   /** @brief Extension for supporting different assembly levels. */
-   LinearFormExtension *ext;
-
    /** @brief Indicates the LinearFormIntegrator%s stored in #domain_integs,
        #domain_delta_integs, #boundary_integs, and #boundary_face_integs are
        owned by another LinearForm. */
@@ -96,7 +75,7 @@ public:
    /// Creates linear form associated with FE space @a *f.
    /** The pointer @a f is not owned by the newly constructed object. */
    LinearForm(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; ext = nullptr; extern_lfs = 0; UseDevice(true); }
+   { fes = f; extern_lfs = 0; UseDevice(true); }
 
    /** @brief Create a LinearForm on the FiniteElementSpace @a f, using the
        same integrators as the LinearForm @a lf.
@@ -111,7 +90,7 @@ public:
    /** The associated FiniteElementSpace can be set later using one of the
        methods: Update(FiniteElementSpace *) or
        Update(FiniteElementSpace *, Vector &, int). */
-   LinearForm() { fes = NULL; ext = nullptr; extern_lfs = 0; UseDevice(true); }
+   LinearForm() { fes = NULL; extern_lfs = 0; UseDevice(true); }
 
    /// Construct a LinearForm using previously allocated array @a data.
    /** The LinearForm does not assume ownership of @a data which is assumed to
@@ -119,7 +98,7 @@ public:
        for externally allocated array, the pointer @a data can be NULL. The data
        array can be replaced later using the method SetData(). */
    LinearForm(FiniteElementSpace *f, double *data) : Vector(data, f->GetVSize())
-   { fes = f; ext = nullptr; extern_lfs = 0; }
+   { fes = f; extern_lfs = 0; }
 
    /// Copy assignment. Only the data of the base class Vector is copied.
    /** It is assumed that this object and @a rhs use FiniteElementSpace%s that
@@ -176,9 +155,6 @@ public:
        coefficients. */
    Array<LinearFormIntegrator*> *GetDLFI() { return &domain_integs; }
 
-   /// Access the element attribute marker (of length mesh->attributes)
-   Array<Array<int>*> *GetDLFIM() { return &domain_integs_marker; }
-
    /** @brief Access all integrators added with AddDomainIntegrator() which are
        DeltaLFIntegrator%s with delta coefficients. */
    Array<DeltaLFIntegrator*> *GetDLFI_Delta() { return &domain_delta_integs; }
@@ -197,10 +173,6 @@ public:
        corresponding pointer (to Array<int>) will be NULL. */
    Array<Array<int>*> *GetFLFI_Marker() { return &boundary_face_integs_marker; }
 
-   /// Set the desired assembly level, default is LinearAssemblyLevel::LEGACY.
-   /** This method must be called before assembly. */
-   void SetAssemblyLevel(LinearAssemblyLevel);
-
    /// Assembles the linear form i.e. sums over all domain/bdr integrators.
    void Assemble();
 
@@ -212,10 +184,11 @@ public:
        updated, e.g. after its associated Mesh object has been refined.
 
        @note This method does not perform assembly. */
-   void Update();
+   void Update() { SetSize(fes->GetVSize()); ResetDeltaLocations(); }
 
    /// Associate a new FE space, @a *f, with this object and Update() it. */
-   void Update(FiniteElementSpace *f);
+   void Update(FiniteElementSpace *f)
+   { fes = f; SetSize(f->GetVSize()); ResetDeltaLocations(); }
 
    /** @brief Associate a new FE space, @a *f, with this object and use the data
        of @a v, offset by @a v_offset, to initialize this object's Vector::data.

fem/linearform_ext.cpp

@@ -1,104 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "linearform.hpp"
-#include "../general/forall.hpp"
-
-// Implementations of FullLinearFormExtension.
-
-namespace mfem
-{
-
-FullLinearFormExtension::FullLinearFormExtension(LinearForm *lf):
-   LinearFormExtension(lf)
-{
-   Update();
-}
-
-void FullLinearFormExtension::Assemble()
-{
-   MFEM_VERIFY(lf->Size() == lf->FESpace()->GetVSize(), "");
-
-   // Filter out the unsupported integrators
-   MFEM_VERIFY(lf->GetBLFI()->Size() == 0,
-               "Integrators added with AddBoundaryIntegrator() "
-               "are not supported!");
-
-   MFEM_VERIFY(lf->GetDLFI_Delta()->Size() == 0, ""
-               "Integrators added with AddDomainIntegrator() which are "
-               "DeltaLFIntegrators with delta coefficients "
-               "are not supported!");
-
-   MFEM_VERIFY(lf->GetIFLFI()->Size() == 0,
-               "Integrators added with AddInteriorFaceIntegrator() "
-               "are not supported!");
-
-   MFEM_VERIFY(lf->GetFLFI()->Size() == 0,
-               "Integrators added with AddBdrFaceIntegrator() "
-               " are not supported!");
-
-   const FiniteElementSpace &fes = *lf->FESpace();
-   const Array<Array<int>*> &domain_integs_marker = *lf->GetDLFIM();
-   const int mesh_attributes_size = fes.GetMesh()->attributes.Size();
-   const Array<LinearFormIntegrator*> &domain_integs = *lf->GetDLFI();
-
-   for (int k = 0; k < domain_integs.Size(); ++k)
-   {
-      // Get the markers for this integrator
-      const Array<int> *domain_integs_marker_k = domain_integs_marker[k];
-
-      // check if there are markers for this integrator
-      const bool has_markers_k = domain_integs_marker_k != nullptr;
-
-      if (has_markers_k)
-      {
-         // Element attribute marker should be of length mesh->attributes
-         MFEM_VERIFY(mesh_attributes_size == domain_integs_marker_k->Size(),
-                     "invalid element marker for domain linear form "
-                     "integrator #" << k << ", counting from zero");
-      }
-
-      const int NE = fes.GetNE();
-      auto markers_w = markers.Write();
-
-      // if there are no markers, just use the whole linear form (1)
-      if (!has_markers_k)
-      {
-         // done this way as operator= for array are still done on the host
-         MFEM_FORALL(e, NE, markers_w[e] = 1;);
-      }
-      else
-      {
-         // otherwise, scan the attributes to set the markers to 0 or 1
-         const auto attr = attributes.Read();
-         const auto dimk = domain_integs_marker_k->Read();
-         MFEM_FORALL(e, NE, markers_w[e] = dimk[attr[e]-1] == 1;);
-      }
-
-      domain_integs[k]->AssembleFull(fes, markers, *lf);
-   }
-}
-
-void FullLinearFormExtension::Update()
-{
-   MFEM_VERIFY(lf->Size() == lf->FESpace()->GetVSize(), "");
-
-   const int ne = lf->FESpace()->GetNE();
-   const Mesh &mesh = *lf->FESpace()->GetMesh();
-
-   markers.SetSize(ne);
-
-   // Gather the attributes on the host from all the elements
-   attributes.SetSize(ne);
-   for (int i = 0; i < ne; ++i) { attributes[i] = mesh.GetAttribute(i); }
-}
-
-} // namespace mfem

fem/linearform_ext.hpp

@@ -1,71 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_LINEARFORM_EXT
-#define MFEM_LINEARFORM_EXT
-
-#include "../general/array.hpp"
-
-namespace mfem
-{
-
-class LinearForm;
-class FullLinearFormExtension;
-
-/// Class extending the LinearForm class to support different AssemblyLevels.
-class LinearFormExtension
-{
-protected:
-   /// Linear form from which this extension depends. Not owned.
-   LinearForm *lf;
-
-public:
-   LinearFormExtension(LinearForm *lf): lf(lf) { }
-
-   virtual ~LinearFormExtension() { }
-
-   /// Updates the linear form extension
-   virtual void Update() = 0;
-
-   /// Assemble at the level given for the linear form extension
-   virtual void Assemble() = 0;
-
-   /// Assembles delta functions of the linear form extension
-   virtual void AssembleDelta() = 0;
-};
-
-/// Data and methods for fully-assembled linear forms
-class FullLinearFormExtension : public LinearFormExtension
-{
-private:
-   /// Attributes of all mesh elements.
-   Array<int> attributes;
-
-   /// Temporary markers for device kernels.
-   Array<int> markers;
-
-public:
-   FullLinearFormExtension(LinearForm *lf);
-
-   /// Fully assembles the linear form, compatible with device execution.
-   /// Only integrators added with AddDomainIntegrator are supported.
-   void Assemble() override;
-
-   /// Fully assembles the delta functions of the linear form.
-   /// Not yet supported.
-   void AssembleDelta() override { MFEM_ABORT("Not yet supported!"); }
-
-   void Update() override;
-};
-
-} // namespace mfem
-
-#endif // MFEM_LINEARFORM_EXT

fem/lininteg.cpp

@@ -9,21 +9,13 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+
 #include "fem.hpp"
 #include <cmath>
 
 namespace mfem
 {
 
-void LinearFormIntegrator::AssembleFull(const FiniteElementSpace&,
-                                        const Array<int> &markers,
-                                        Vector &b)
-{
-   MFEM_CONTRACT_VAR(b);
-   MFEM_CONTRACT_VAR(markers);
-   mfem_error("LinearFormIntegrator::AssembleFull(...)");
-}
-
 void LinearFormIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect)
 {
@@ -288,11 +280,12 @@ void VectorDomainLFIntegrator::AssembleRHSElementVect(
       val = Tr.Weight();
 
       el.CalcShape(ip, shape);
-      Q.Eval(Qvec, Tr, ip);
+      Q.Eval (Qvec, Tr, ip);
 
       for (int k = 0; k < vdim; k++)
       {
          cf = val * Qvec(k);
+
          for (int s = 0; s < dof; s++)
          {
             elvect(dof*k+s) += ip.weight * cf * shape(s);
@@ -318,54 +311,6 @@ void VectorDomainLFIntegrator::AssembleDeltaElementVect(
    MultVWt(shape, Qvec, elvec_as_mat);
 }
 
-void VectorDomainLFGradIntegrator::AssembleRHSElementVect(
-   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
-{
-   const int dim = el.GetDim();
-   const int dof = el.GetDof();
-   const int vdim = Q.GetVDim();
-   const int sdim = Tr.GetSpaceDim();
-
-   dshape.SetSize(dof,sdim);
-
-   elvect.SetSize(dof*(vdim/sdim));
-   elvect = 0.0;
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      int intorder = 2 * el.GetOrder();
-      ir = &IntRules.Get(el.GetGeomType(), intorder);
-   }
-
-   Vector pelvect(dof);
-   Vector part_x(dim);
-
-   for (int q = 0; q < ir->GetNPoints(); q++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(q);
-
-      Tr.SetIntPoint(&ip);
-      el.CalcPhysDShape(Tr, dshape);
-
-      Q.Eval(Qvec, Tr, ip);
-      Qvec *= ip.weight * Tr.Weight();
-
-      for (int k = 0; k < vdim/sdim; k++)
-      {
-         for (int d=0; d < sdim; ++d) { part_x(d) = Qvec(k*sdim+d); }
-         dshape.Mult(part_x, pelvect);
-         for (int s = 0; s < dof; ++s) { elvect(s+k*dof) += pelvect(s); }
-      }
-   }
-}
-
-void VectorDomainLFGradIntegrator::AssembleDeltaElementVect(
-   const FiniteElement&, ElementTransformation&, Vector&)
-{
-   MFEM_ABORT("Not implemented!");
-}
-
 void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {

fem/lininteg.hpp

@@ -27,11 +27,6 @@ protected:
    LinearFormIntegrator(const IntegrationRule *ir = NULL) { IntRule = ir; }
 
 public:
-   /// Method defining full assembly
-   virtual void AssembleFull(const FiniteElementSpace &fes,
-                             const Array<int> &markers,
-                             Vector &b);
-
    /** Given a particular Finite Element and a transformation (Tr)
        computes the element vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -115,10 +110,6 @@ public:
    DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir)
       : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1) { }
 
-   virtual void AssembleFull(const FiniteElementSpace &fes,
-                             const Array<int> &markers,
-                             Vector &b);
-
    /** Given a particular Finite Element and a transformation (Tr)
        computes the element right hand side element vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -145,10 +136,6 @@ public:
    DomainLFGradIntegrator(VectorCoefficient &QF)
       : DeltaLFIntegrator(QF), Q(QF) { }
 
-   virtual void AssembleFull(const FiniteElementSpace &fes,
-                             const Array<int> &markers,
-                             Vector &b);
-
    /** Given a particular Finite Element and a transformation (Tr)
        computes the element right hand side element vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -236,10 +223,6 @@ public:
    VectorDomainLFIntegrator(VectorCoefficient &QF)
       : DeltaLFIntegrator(QF), Q(QF) { }
 
-   virtual void AssembleFull(const FiniteElementSpace &fes,
-                             const Array<int> &markers,
-                             Vector &b);
-
    /** Given a particular Finite Element and a transformation (Tr)
        computes the element right hand side element vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -253,37 +236,6 @@ public:
    using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
-/** Class for domain integrator L(v) := (f, grad v), where
-    f=(f1x,f1y,f1z,...,fnx,fny,fnz) and v=(v1,...,vn). */
-class VectorDomainLFGradIntegrator : public DeltaLFIntegrator
-{
-private:
-   Vector shape, Qvec;
-   VectorCoefficient &Q;
-   DenseMatrix dshape;
-
-public:
-   /// Constructs the domain integrator (Q, grad v)
-   VectorDomainLFGradIntegrator(VectorCoefficient &QF)
-      : DeltaLFIntegrator(QF), Q(QF) { }
-
-   virtual void AssembleFull(const FiniteElementSpace &fes,
-                             const Array<int> &markers,
-                             Vector &b) override;
-
-   /** Given a particular Finite Element and a transformation (Tr)
-       computes the element right hand side element vector, elvect. */
-   virtual void AssembleRHSElementVect(const FiniteElement &el,
-                                       ElementTransformation &Tr,
-                                       Vector &elvect) override;
-
-   virtual void AssembleDeltaElementVect(const FiniteElement &fe,
-                                         ElementTransformation &Trans,
-                                         Vector &elvect) override;
-
-   using LinearFormIntegrator::AssembleRHSElementVect;
-};
-
 /** Class for boundary integration of L(v) := (g, v), where
     f=(f1,...,fn) and v=(v1,...,vn). */
 class VectorBoundaryLFIntegrator : public LinearFormIntegrator

fem/lininteg_domain.cpp

@@ -1,104 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-#include "lininteg_domain.hpp"
-
-namespace mfem
-{
-
-using namespace internal::linearform_extension;
-
-void DomainLFIntegrator::AssembleFull(const FiniteElementSpace &fes,
-                                      const Array<int> &markers,
-                                      Vector &y)
-{
-   const int vdim = fes.GetVDim();
-   MFEM_VERIFY(vdim == 1, "vdim should be equal to 1!");
-   GetOrder_f qorder_fct = [&](int el_order) { return oa * el_order + ob; };
-   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, qorder_fct);
-
-   Vector coeff;
-   const int NQ = ir->GetNPoints();
-   const int NE = fes.GetMesh()->GetNE();
-
-   if (ConstantCoefficient *cQ =
-          dynamic_cast<ConstantCoefficient*>(&Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient *cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qfun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qfun.Size() == vdim*NE*NQ,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different.\n");
-      qfun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
-   }
-   else
-   {
-      coeff.SetSize(NQ * NE);
-      auto C = Reshape(coeff.HostWrite(), NQ, NE);
-      for (int e = 0; e < NE; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < NQ; ++q)
-         {
-            C(q,e) = Q.Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
-
-   const int id = GetKernelId(fes,ir);
-   const int dim = fes.GetMesh()->Dimension();
-
-   LinearFormExtensionKernel_f ker = nullptr;
-   if (dim==2) { ker=VectorDomainLFIntegratorAssemble2D; }
-   if (dim==3) { ker=VectorDomainLFIntegratorAssemble3D; }
-
-   switch (id)
-   {
-      // 2D kernels, q=p+1
-      case 0x222: ker=VectorDomainLFIntegratorAssemble2D<2,2>; break;
-      case 0x233: ker=VectorDomainLFIntegratorAssemble2D<3,3>; break;
-      case 0x244: ker=VectorDomainLFIntegratorAssemble2D<4,4>; break;
-      case 0x255: ker=VectorDomainLFIntegratorAssemble2D<5,5>; break;
-
-      // 2D kernels, q=p+2
-      case 0x223: ker=VectorDomainLFIntegratorAssemble2D<2,3>; break;
-      case 0x234: ker=VectorDomainLFIntegratorAssemble2D<3,4>; break;
-      case 0x245: ker=VectorDomainLFIntegratorAssemble2D<4,5>; break;
-      case 0x256: ker=VectorDomainLFIntegratorAssemble2D<5,6>; break;
-
-      // 3D kernels, q=p+1, BENCH_FULL_DomainLF_3D_GLL
-      case 0x322: ker=VectorDomainLFIntegratorAssemble3D<2,2>; break;
-      case 0x333: ker=VectorDomainLFIntegratorAssemble3D<3,3>; break;
-      case 0x344: ker=VectorDomainLFIntegratorAssemble3D<4,4>; break;
-      case 0x355: ker=VectorDomainLFIntegratorAssemble3D<5,5>; break;
-      case 0x366: ker=VectorDomainLFIntegratorAssemble3D<6,6>; break;
-
-      // 3D kernels, q=p+2, BENCH_FULL_DomainLF_3D_GL
-      case 0x323: ker=VectorDomainLFIntegratorAssemble3D<2,3>; break;
-      case 0x334: ker=VectorDomainLFIntegratorAssemble3D<3,4>; break;
-      case 0x345: ker=VectorDomainLFIntegratorAssemble3D<4,5>; break;
-      case 0x356: ker=VectorDomainLFIntegratorAssemble3D<5,6>; break;
-      case 0x367: ker=VectorDomainLFIntegratorAssemble3D<6,7>; break;
-   }
-   MFEM_VERIFY(ker, "Unexpected kernel error!");
-   Launch(ker,fes,ir,coeff,markers,y);
-}
-
-} // namespace mfem

fem/lininteg_domain.hpp

@@ -1,263 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-// Internal header, included only by .cpp files
-
-#include "fem.hpp"
-
-#include "../fem/kernels.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-namespace internal
-{
-
-namespace linearform_extension
-{
-
-/// Signature of the kernels used for linear form extension
-using LinearFormExtensionKernel_f = void (*)(const int vdim,
-                                             const bool byVDIM,
-                                             const int ND,
-                                             const int NE,
-                                             const int d,
-                                             const int q,
-                                             const int *markers,
-                                             const double *b,
-                                             const double *g,
-                                             const int *idx,
-                                             const double *J,
-                                             const double *detJ,
-                                             const double *weights,
-                                             const Vector &coeff,
-                                             double *output);
-
-/// Signature of the function used to compute the quadrature order
-using GetOrder_f = std::function<int(int)>;
-
-/// Internal helper function to get the integration rule
-inline const IntegrationRule *GetIntRuleFromOrder(const FiniteElementSpace &fes,
-                                                  const IntegrationRule *IntRule,
-                                                  const GetOrder_f &qorder_fct)
-{
-   const FiniteElement &fe = *fes.GetFE(0);
-   const int qorder = qorder_fct(fe.GetOrder());
-   const Geometry::Type geom_type = fe.GetGeomType();
-   return IntRule ? IntRule : &IntRules.Get(geom_type, qorder);
-}
-
-/// Internal helper function to encode the ID of a LinearFormExtension kernel
-/// The ID is equal to: (dim << 8) |
-///                     (1D number of degrees of freedom << 4) |
-///                     (1D number of quadrature points)
-inline int GetKernelId(const FiniteElementSpace &fes,
-                       const IntegrationRule *ir)
-{
-   Mesh *mesh = fes.GetMesh();
-   const int dim = mesh->Dimension();
-   const FiniteElement &el = *fes.GetFE(0);
-   const DofToQuad &maps = el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   const int d = maps.ndof;
-   const int q = maps.nqpt;
-   return (dim << 8) | (d << 4) | q;
-}
-
-/// Internal helper function to launch the LinearFormExtension kernel
-inline void Launch(const LinearFormExtensionKernel_f &kernel,
-                   const FiniteElementSpace &fes,
-                   const IntegrationRule *ir,
-                   const Vector &coeff,
-                   const Array<int> &markers,
-                   Vector &y)
-{
-   Mesh *mesh = fes.GetMesh();
-   const int vdim = fes.GetVDim();
-   const bool byVDIM = fes.GetOrdering() == Ordering::byVDIM;
-
-   const FiniteElement &el = *fes.GetFE(0);
-   constexpr int flags = GeometricFactors::JACOBIANS |
-                         GeometricFactors::DETERMINANTS;
-   const MemoryType mt = Device::GetDeviceMemoryType();
-   const GeometricFactors *geom = mesh->GetGeometricFactors(*ir, flags, mt);
-   const DofToQuad &maps = el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes.GetElementRestriction(ordering);
-   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
-   MFEM_ASSERT(ER, "Not supported!");
-
-   const int *M = markers.Read();
-   const double *B = maps.B.Read();
-   const double *G = maps.G.Read();
-   const double *J = geom->J.Read();
-   const double *detJ = geom->detJ.Read();
-   const int *I = ER->GatherMap().Read();
-   const double *W = ir->GetWeights().Read();
-   double *Y = y.ReadWrite();
-
-   const int ND = fes.GetNDofs();
-   const int NE = fes.GetMesh()->GetNE();
-
-   const int d = maps.ndof;
-   const int q = maps.nqpt;
-
-   kernel(vdim, byVDIM, ND, NE, d, q, M, B, G, I, J, detJ, W, coeff, Y);
-}
-
-/// Internal assembly kernel for the 2D (Vector)DomainLFIntegrator
-template<int D=0, int Q=0> static
-void VectorDomainLFIntegratorAssemble2D(const int vdim,
-                                        const bool byVDIM,
-                                        const int ND,
-                                        const int NE,
-                                        const int d,
-                                        const int q,
-                                        const int *markers,
-                                        const double *b,
-                                        const double *, // g
-                                        const int *idx,
-                                        const double *, // jacobians
-                                        const double *detJ,
-                                        const double *weights,
-                                        const Vector &coeff,
-                                        double *y)
-{
-   constexpr bool USE_SMEM = D > 0 && Q > 0;
-
-   const bool cst_coeff = coeff.Size() == vdim;
-
-   const auto F = coeff.Read();
-   const auto M = Reshape(markers, NE);
-   const auto B = Reshape(b, q,d);
-   const auto DetJ = Reshape(detJ, q,q, NE);
-   const auto W = Reshape(weights, q,q);
-   const auto I = Reshape(idx, d,d, NE);
-   const auto C = cst_coeff ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,NE);
-
-   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
-
-   const int sm_size = 2*q*(d+q);
-   constexpr int GRID = USE_SMEM ? 0 : 128;
-   double *gmem = ScratchMem<GRID>(sm_size);
-
-   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
-   {
-      if (M(e) == 0) { /* ignore */ return; }
-
-      const int bid = MFEM_BLOCK_ID(x);
-      constexpr int SM_SIZE = 2*Q*(D+Q);
-      constexpr bool USE_SMEM = D > 0 && Q > 0;
-      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
-      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
-      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), q,d);
-      const DeviceMatrix QQ(DeviceMemAlloc(sm,q*q), q,q);
-      const DeviceMatrix QD(DeviceMemAlloc(sm,q*d), q,d);
-
-      kernels::internal::LoadB(d,q,B,Bt);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const double cst_val = C(c,0,0,0);
-         MFEM_FOREACH_THREAD(x,x,q)
-         {
-            MFEM_FOREACH_THREAD(y,y,q)
-            {
-               const double detJ = DetJ(x,y,e);
-               const double coeff_val = cst_coeff ? cst_val : C(c,x,y,e);
-               QQ(y,x) = W(x,y) * coeff_val * detJ;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         kernels::internal::Atomic2DEvalTranspose(d,q,Bt,QQ,QD,I,Y,c,e,byVDIM);
-      }
-   });
-}
-
-/// Internal assembly kernel for the 2D (Vector)DomainLFIntegrator
-template<int D=0, int Q=0> static
-void VectorDomainLFIntegratorAssemble3D(const int vdim,
-                                        const bool byVDIM,
-                                        const int ND,
-                                        const int NE,
-                                        const int d,
-                                        const int q,
-                                        const int *markers,
-                                        const double *b,
-                                        const double *, // g
-                                        const int *idx,
-                                        const double *, // jacobians
-                                        const double *detJ,
-                                        const double *weights,
-                                        const Vector &coeff,
-                                        double *y)
-{
-   constexpr bool USE_SMEM = D > 0 && Q > 0;
-
-   const bool cst_coeff = coeff.Size() == vdim;
-
-   const auto F = coeff.Read();
-   const auto M = Reshape(markers, NE);
-   const auto B = Reshape(b, q,d);
-   const auto DetJ = Reshape(detJ, q,q,q, NE);
-   const auto W = Reshape(weights, q,q,q);
-   const auto I = Reshape(idx, d,d,d, NE);
-   const auto C = cst_coeff ? Reshape(F,vdim,1,1,1,1):Reshape(F,vdim,q,q,q,NE);
-
-   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
-
-   const int sm_size = q*d + q*q*q;
-   const int GRID = USE_SMEM ? 0 : 128;
-   double *gmem = ScratchMem<GRID>(sm_size);
-   MFEM_VERIFY(q < 32, "Unsupported quadrature order!");
-
-   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
-   {
-      if (M(e) == 0) { /* ignore */ return; }
-
-      double u[Q>0?Q:32];
-
-      const int bid = MFEM_BLOCK_ID(x);
-      constexpr int SM_SIZE = Q*D + Q*Q*Q;
-      constexpr bool USE_SMEM = D > 0 && Q > 0;
-      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
-      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
-      const DeviceCube QQQ(DeviceMemAlloc(sm,q*q*q), q,q,q);
-      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), q,d);
-      kernels::internal::LoadB(d,q,B,Bt);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const double cst_val = C(c,0,0,0,0);
-         MFEM_FOREACH_THREAD(x,x,q)
-         {
-            MFEM_FOREACH_THREAD(y,y,q)
-            {
-               for (int z = 0; z < q; ++z)
-               {
-                  const double detJ = DetJ(x,y,z,e);
-                  const double coeff_val = cst_coeff ? cst_val : C(c,x,y,z,e);
-                  QQQ(z,y,x) = W(x,y,z) * coeff_val * detJ;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         kernels::internal::Atomic3DEvalTranspose(d,q,u,Bt,QQQ,I,Y,c,e,byVDIM);
-      }
-   });
-}
-
-} // namespace linearform_extension
-
-} // namespace internal
-
-} // namespace mfem

fem/lininteg_domain_grad.cpp

@@ -1,107 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-#include "lininteg_domain.hpp"
-#include "lininteg_domain_grad.hpp"
-
-namespace mfem
-{
-
-using namespace internal::linearform_extension;
-
-void DomainLFGradIntegrator::AssembleFull(const FiniteElementSpace &fes,
-                                          const Array<int> &markers,
-                                          Vector &y)
-{
-   MFEM_VERIFY(fes.GetVDim()==1, "vdim != 1");
-   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
-   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
-
-   Vector coeff;
-   const int NQ = ir->GetNPoints();
-   const int NE = fes.GetMesh()->GetNE();
-
-   if (VectorConstantCoefficient *vcQ =
-          dynamic_cast<VectorConstantCoefficient*>(&Q))
-   {
-      coeff = vcQ->GetVec();
-   }
-   else if (VectorQuadratureFunctionCoefficient *vqfQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qfun = vqfQ->GetQuadFunction();
-      MFEM_VERIFY(qfun.Size() == NE*NQ,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different.\n");
-      qfun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
-   }
-   else
-   {
-      const int qvdim = Q.GetVDim();
-      Vector Qvec(qvdim);
-      coeff.SetSize(qvdim * NQ * NE);
-      auto C = Reshape(coeff.HostWrite(), qvdim, NQ, NE);
-      for (int e = 0; e < NE; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < NQ; ++q)
-         {
-            Q.Eval(Qvec, T, ir->IntPoint(q));
-            for (int c=0; c<qvdim; ++c)
-            {
-               C(c,q,e) = Qvec[c];
-            }
-         }
-      }
-   }
-
-   const int id = GetKernelId(fes,ir);
-   const int dim = fes.GetMesh()->Dimension();
-
-   LinearFormExtensionKernel_f ker = nullptr;
-   if (dim==2) { ker=VectorDomainLFGradIntegratorAssemble2D; }
-   if (dim==3) { ker=VectorDomainLFGradIntegratorAssemble3D; }
-
-   switch (id)
-   {
-      // 2D kernels, q=p+1
-      case 0x222: ker=VectorDomainLFGradIntegratorAssemble2D<2,2>; break;
-      case 0x233: ker=VectorDomainLFGradIntegratorAssemble2D<3,3>; break;
-      case 0x244: ker=VectorDomainLFGradIntegratorAssemble2D<4,4>; break;
-      case 0x255: ker=VectorDomainLFGradIntegratorAssemble2D<5,5>; break;
-
-      // 2D kernels, q=p+2
-      case 0x223: ker=VectorDomainLFGradIntegratorAssemble2D<2,3>; break;
-      case 0x234: ker=VectorDomainLFGradIntegratorAssemble2D<3,4>; break;
-      case 0x245: ker=VectorDomainLFGradIntegratorAssemble2D<4,5>; break;
-      case 0x256: ker=VectorDomainLFGradIntegratorAssemble2D<5,6>; break;
-
-      // 3D kernels, q=p+1
-      case 0x322: ker=VectorDomainLFGradIntegratorAssemble3D<2,2>; break;
-      case 0x333: ker=VectorDomainLFGradIntegratorAssemble3D<3,3>; break;
-      case 0x344: ker=VectorDomainLFGradIntegratorAssemble3D<4,4>; break;
-      case 0x355: ker=VectorDomainLFGradIntegratorAssemble3D<5,5>; break;
-
-      // 3D kernels, q=p+2
-      case 0x323: ker=VectorDomainLFGradIntegratorAssemble3D<2,3>; break;
-      case 0x334: ker=VectorDomainLFGradIntegratorAssemble3D<3,4>; break;
-      case 0x345: ker=VectorDomainLFGradIntegratorAssemble3D<4,5>; break;
-      case 0x356: ker=VectorDomainLFGradIntegratorAssemble3D<5,6>; break;
-   }
-   MFEM_VERIFY(ker, "Unexpected kernel error!");
-   Launch(ker,fes,ir,coeff,markers,y);
-}
-
-} // namespace mfem

fem/lininteg_domain_grad.hpp

@@ -1,239 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-// Internal header, included only by .cpp files
-
-#include "fem.hpp"
-
-#include "../general/forall.hpp"
-#include "../fem/kernels.hpp"
-#include "../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-namespace internal
-{
-
-namespace linearform_extension
-{
-
-////////////////////////////////////////////////////////////////////////////////
-template<int D=0, int Q=0> static
-void VectorDomainLFGradIntegratorAssemble2D(const int vdim,
-                                            const bool byVDIM,
-                                            const int ND,
-                                            const int NE,
-                                            const int d,
-                                            const int q,
-                                            const int *markers,
-                                            const double *b,
-                                            const double *g,
-                                            const int *idx,
-                                            const double *jacobians,
-                                            const double *detJ,
-                                            const double *weights,
-                                            const Vector &coeff,
-                                            double *y)
-{
-   constexpr int DIM = 2;
-   constexpr bool USE_SMEM = D > 0 && Q > 0;
-
-   const bool cst_coeff = coeff.Size() == vdim*DIM;
-
-   const auto F = coeff.Read();
-   const auto M = Reshape(markers, NE);
-   const auto B = Reshape(b, q,d);
-   const auto G = Reshape(g, q,d);
-   const auto J = Reshape(jacobians, q,q, DIM,DIM, NE);
-   const auto DetJ = Reshape(detJ, q,q, NE);
-   const auto W = Reshape(weights, q,q);
-   const auto I = Reshape(idx, d,d, NE);
-   const auto C = cst_coeff ?
-                  Reshape(F,DIM,vdim,1,1,1):
-                  Reshape(F,DIM,vdim,q,q,NE);
-
-   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
-
-   const int sm_size = 2*d*q + 4*q*q;
-   const int GRID = USE_SMEM ? 0 : 128;
-   double *gmem = ScratchMem<GRID>(sm_size);
-
-   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
-   {
-      if (M(e) == 0) { /* ignore */ return; }
-
-      const int bid = MFEM_BLOCK_ID(x);
-      constexpr int SM_SIZE = 2*D*Q + 4*Q*Q;
-      constexpr bool USE_SMEM = D > 0 && Q > 0;
-      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
-      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
-
-      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), d,q);
-      const DeviceMatrix Gt(DeviceMemAlloc(sm,q*d), d,q);
-      kernels::internal::LoadBGt(d,q,B,G,Bt,Gt);
-
-      const DeviceMatrix QQ0(DeviceMemAlloc(sm,q*q), q,q);
-      const DeviceMatrix QQ1(DeviceMemAlloc(sm,q*q), q,q);
-
-      const DeviceMatrix DQ0(DeviceMemAlloc(sm,d*q), d,q);
-      const DeviceMatrix DQ1(DeviceMemAlloc(sm,d*q), d,q);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const double cst_val0 = C(0,c,0,0,0);
-         const double cst_val1 = C(1,c,0,0,0);
-
-         MFEM_FOREACH_THREAD(x,x,q)
-         {
-            MFEM_FOREACH_THREAD(y,y,q)
-            {
-               double Jloc[4], Jinv[4];
-               Jloc[0] = J(x,y,0,0,e);
-               Jloc[1] = J(x,y,1,0,e);
-               Jloc[2] = J(x,y,0,1,e);
-               Jloc[3] = J(x,y,1,1,e);
-               const double detJ = DetJ(x,y,e);
-               kernels::CalcInverse<2>(Jloc, Jinv);
-               const double weight = W(x,y);
-               const double u = cst_coeff ? cst_val0 : C(0,c,x,y,e);
-               const double v = cst_coeff ? cst_val1 : C(1,c,x,y,e);
-               QQ0(y,x) = Jinv[0]*u + Jinv[2]*v;
-               QQ1(y,x) = Jinv[1]*u + Jinv[3]*v;
-               QQ0(y,x) *= weight * detJ;
-               QQ1(y,x) *= weight * detJ;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         kernels::internal::Atomic2DGradTranspose(d,q,Bt,Gt,
-                                                  QQ0,QQ1,DQ0,DQ1,
-                                                  I,Y,c,e,byVDIM);
-      }
-   });
-}
-
-template<int D=0, int Q=0> static
-void VectorDomainLFGradIntegratorAssemble3D(const int vdim,
-                                            const bool byVDIM,
-                                            const int ND,
-                                            const int NE,
-                                            const int d,
-                                            const int q,
-                                            const int *markers,
-                                            const double *b,
-                                            const double *g,
-                                            const int *idx,
-                                            const double *jacobians,
-                                            const double *detJ,
-                                            const double *weights,
-                                            const Vector &coeff,
-                                            double *y)
-{
-   constexpr int DIM = 3;
-   constexpr bool USE_SMEM = D > 0 && Q > 0;
-
-   const bool cst_coeff = coeff.Size() == vdim*DIM;
-
-   const auto F = coeff.Read();
-   const auto M = Reshape(markers, NE);
-   const auto B = Reshape(b, q,d);
-   const auto G = Reshape(g, q,d);
-   const auto J = Reshape(jacobians, q,q,q, DIM,DIM, NE);
-   const auto DetJ = Reshape(detJ, q,q,q, NE);
-   const auto W = Reshape(weights, q,q,q);
-   const auto I = Reshape(idx, d,d,d, NE);
-   const auto C = cst_coeff ?
-                  Reshape(F,DIM,vdim,1,1,1,1):
-                  Reshape(F,DIM,vdim,q,q,q,NE);
-
-   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
-
-   const int sm_size = 2*q*d + 6*q*q*q;
-
-   const int GRID = USE_SMEM ? 0 : 128;
-   double *gmem = ScratchMem<GRID>(sm_size);
-
-   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
-   {
-      if (M(e) == 0) { /* ignore */ return; }
-
-      const int bid = MFEM_BLOCK_ID(x);
-      constexpr int SM_SIZE = 2*Q*D + 6*Q*Q*Q;
-      constexpr bool USE_SMEM = D > 0 && Q > 0;
-      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
-      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
-
-      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), d,q);
-      const DeviceMatrix Gt(DeviceMemAlloc(sm,q*d), d,q);
-      kernels::internal::LoadBGt(d,q,B,G,Bt,Gt);
-
-      const DeviceCube QQ0(DeviceMemAlloc(sm,q*q*q), q,q,q);
-      const DeviceCube QQ1(DeviceMemAlloc(sm,q*q*q), q,q,q);
-      const DeviceCube QQ2(DeviceMemAlloc(sm,q*q*q), q,q,q);
-
-      const DeviceCube QD0(DeviceMemAlloc(sm,q*q*q), q,q,d);
-      const DeviceCube QD1(DeviceMemAlloc(sm,q*q*q), q,q,d);
-      const DeviceCube QD2(DeviceMemAlloc(sm,q*q*q), q,q,d);
-
-      const DeviceCube DD0(QQ0,q,d,d);
-      const DeviceCube DD1(QQ1,q,d,d);
-      const DeviceCube DD2(QQ2,q,d,d);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const double cst_val_0 = C(0,c,0,0,0,0);
-         const double cst_val_1 = C(1,c,0,0,0,0);
-         const double cst_val_2 = C(2,c,0,0,0,0);
-
-         MFEM_FOREACH_THREAD(x,x,q)
-         {
-            MFEM_FOREACH_THREAD(y,y,q)
-            {
-               for (int z = 0; z < q; ++z)
-               {
-                  double Jloc[9], Jinv[9];
-                  for (int j = 0; j < 3; j++)
-                  {
-                     for (int i = 0; i < 3; i++)
-                     {
-                        Jloc[i+3*j] = J(x,y,z,i,j,e);
-                     }
-                  }
-                  const double detJ = DetJ(x,y,z,e);
-                  kernels::CalcInverse<3>(Jloc, Jinv);
-                  const double weight = W(x,y,z);
-                  const double u = cst_coeff ? cst_val_0 : C(0,c,x,y,z,e);
-                  const double v = cst_coeff ? cst_val_1 : C(1,c,x,y,z,e);
-                  const double w = cst_coeff ? cst_val_2 : C(2,c,x,y,z,e);
-                  QQ0(z,y,x) = Jinv[0]*u + Jinv[3]*v + Jinv[6]*w;
-                  QQ1(z,y,x) = Jinv[1]*u + Jinv[4]*v + Jinv[7]*w;
-                  QQ2(z,y,x) = Jinv[2]*u + Jinv[5]*v + Jinv[8]*w;
-                  QQ0(z,y,x) *= weight * detJ;
-                  QQ1(z,y,x) *= weight * detJ;
-                  QQ2(z,y,x) *= weight * detJ;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         kernels::internal::Atomic3DGrad(d,q,Bt,Gt,
-                                         QQ0,QQ1,QQ2,
-                                         QD0,QD1,QD2,
-                                         DD0,DD1,DD2,
-                                         I,Y,c,e,byVDIM);
-      }
-   });
-}
-
-} // namespace linearform_extension
-
-} // namespace internal
-
-} // namespace mfem

fem/lininteg_vector_domain.cpp

@@ -1,102 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-#include "lininteg_domain.hpp"
-
-namespace mfem
-{
-
-using namespace internal::linearform_extension;
-
-void VectorDomainLFIntegrator::AssembleFull(const FiniteElementSpace &fes,
-                                            const Array<int> &markers,
-                                            Vector &y)
-{
-   const int vdim = fes.GetVDim();
-   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
-   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
-
-   Vector coeff;
-   const int NQ = ir->GetNPoints();
-   const int NE = fes.GetMesh()->GetNE();
-
-   if (VectorConstantCoefficient *vcQ =
-          dynamic_cast<VectorConstantCoefficient*>(&Q))
-   {
-      coeff = vcQ->GetVec();
-   }
-   else if (VectorQuadratureFunctionCoefficient *vQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qfun = vQ->GetQuadFunction();
-      MFEM_VERIFY(qfun.Size() == vdim*NE*NQ,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different.\n");
-      qfun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
-   }
-   else
-   {
-      Vector Qvec(vdim);
-      coeff.SetSize(vdim * NQ * NE);
-      auto C = Reshape(coeff.HostWrite(), vdim, NQ, NE);
-      for (int e = 0; e < NE; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < NQ; ++q)
-         {
-            Q.Eval(Qvec, T, ir->IntPoint(q));
-            for (int c=0; c<vdim; ++c) { C(c,q,e) = Qvec[c]; }
-         }
-      }
-   }
-
-   LinearFormExtensionKernel_f ker = nullptr;
-   const int id = GetKernelId(fes,ir);
-   const int dim = fes.GetMesh()->Dimension();
-
-   if (dim==2) { ker = VectorDomainLFIntegratorAssemble2D<>; }
-   if (dim==3) { ker = VectorDomainLFIntegratorAssemble3D<>; }
-
-   switch (id)
-   {
-      // 2D kernels, q=p+1
-      case 0x222: ker=VectorDomainLFIntegratorAssemble2D<2,2>; break;
-      case 0x233: ker=VectorDomainLFIntegratorAssemble2D<3,3>; break;
-      case 0x244: ker=VectorDomainLFIntegratorAssemble2D<4,4>; break;
-      case 0x255: ker=VectorDomainLFIntegratorAssemble2D<5,5>; break;
-
-      // 2D kernels, q=p+2
-      case 0x223: ker=VectorDomainLFIntegratorAssemble2D<2,3>; break;
-      case 0x234: ker=VectorDomainLFIntegratorAssemble2D<3,4>; break;
-      case 0x245: ker=VectorDomainLFIntegratorAssemble2D<4,5>; break;
-      case 0x256: ker=VectorDomainLFIntegratorAssemble2D<5,6>; break;
-
-      // 3D kernels, q=p+1
-      case 0x322: ker=VectorDomainLFIntegratorAssemble3D<2,2>; break;
-      case 0x333: ker=VectorDomainLFIntegratorAssemble3D<3,3>; break;
-      case 0x344: ker=VectorDomainLFIntegratorAssemble3D<4,4>; break;
-      case 0x355: ker=VectorDomainLFIntegratorAssemble3D<5,5>; break;
-
-      // 3D kernels, q=p+2
-      case 0x323: ker=VectorDomainLFIntegratorAssemble3D<2,3>; break;
-      case 0x334: ker=VectorDomainLFIntegratorAssemble3D<3,4>; break;
-      case 0x345: ker=VectorDomainLFIntegratorAssemble3D<4,5>; break;
-      case 0x356: ker=VectorDomainLFIntegratorAssemble3D<5,6>; break;
-   }
-   MFEM_VERIFY(ker, "Unexpected kernel error!");
-   Launch(ker,fes,ir,coeff,markers,y);
-}
-
-} // namespace mfem

fem/lininteg_vector_domain_grad.cpp

@@ -1,116 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-#include "lininteg_domain.hpp"
-#include "lininteg_domain_grad.hpp"
-
-namespace mfem
-{
-
-using namespace internal::linearform_extension;
-
-void VectorDomainLFGradIntegrator::AssembleFull(const FiniteElementSpace &fes,
-                                                const Array<int> &markers,
-                                                Vector &y)
-{
-   const int vdim = fes.GetVDim();
-   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
-   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
-
-   Vector coeff;
-   const int NQ = ir->GetNPoints();
-   const int NE = fes.GetMesh()->GetNE();
-   const int NS = fes.GetMesh()->SpaceDimension();
-
-   if (VectorConstantCoefficient *vcQ =
-          dynamic_cast<VectorConstantCoefficient*>(&Q))
-   {
-      coeff = vcQ->GetVec();
-   }
-   else if (QuadratureFunctionCoefficient *qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qfun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qfun.Size() == NE*NQ,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different.\n");
-      qfun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
-   }
-   else if (VectorQuadratureFunctionCoefficient* vqfQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == vdim * NS * NQ * NE,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      Vector Qvec(vdim);
-      coeff.SetSize(vdim * NQ * NE);
-      auto C = Reshape(coeff.HostWrite(), vdim, NQ, NE);
-      for (int e = 0; e < NE; ++e)
-      {
-         ElementTransformation &Tr = *fes.GetElementTransformation(e);
-         for (int q = 0; q < NQ; ++q)
-         {
-            Q.Eval(Qvec, Tr, ir->IntPoint(q));
-            for (int c = 0; c<vdim; ++c) { C(c,q,e) = Qvec[c]; }
-         }
-      }
-   }
-
-   LinearFormExtensionKernel_f ker = nullptr;
-   const int id = GetKernelId(fes,ir);
-   const int dim = fes.GetMesh()->Dimension();
-
-   if (dim==2) { ker=VectorDomainLFGradIntegratorAssemble2D; }
-   if (dim==3) { ker=VectorDomainLFGradIntegratorAssemble3D; }
-
-   switch (id)
-   {
-      // 2D kernels, q=p+1
-      case 0x222: ker=VectorDomainLFGradIntegratorAssemble2D<2,2>; break;
-      case 0x233: ker=VectorDomainLFGradIntegratorAssemble2D<3,3>; break;
-      case 0x244: ker=VectorDomainLFGradIntegratorAssemble2D<4,4>; break;
-      case 0x255: ker=VectorDomainLFGradIntegratorAssemble2D<5,5>; break;
-
-      // 2D kernels, q=p+2
-      case 0x223: ker=VectorDomainLFGradIntegratorAssemble2D<2,3>; break;
-      case 0x234: ker=VectorDomainLFGradIntegratorAssemble2D<3,4>; break;
-      case 0x245: ker=VectorDomainLFGradIntegratorAssemble2D<4,5>; break;
-      case 0x256: ker=VectorDomainLFGradIntegratorAssemble2D<5,6>; break;
-
-      // 3D kernels, q=p+1
-      case 0x322: ker=VectorDomainLFGradIntegratorAssemble3D<2,2>; break;
-      case 0x333: ker=VectorDomainLFGradIntegratorAssemble3D<3,3>; break;
-      case 0x344: ker=VectorDomainLFGradIntegratorAssemble3D<4,4>; break;
-      case 0x355: ker=VectorDomainLFGradIntegratorAssemble3D<5,5>; break;
-
-      // 3D kernels, q=p+2
-      case 0x323: ker=VectorDomainLFGradIntegratorAssemble3D<2,3>; break;
-      case 0x334: ker=VectorDomainLFGradIntegratorAssemble3D<3,4>; break;
-      case 0x345: ker=VectorDomainLFGradIntegratorAssemble3D<4,5>; break;
-      case 0x356: ker=VectorDomainLFGradIntegratorAssemble3D<5,6>; break;
-   }
-   MFEM_VERIFY(ker, "Unexpected kernel error!");
-   Launch(ker,fes,ir,coeff,markers,y);
-}
-
-} // namespace mfem

fem/lor.cpp

@@ -10,21 +10,8 @@
 // CONTRIBUTING.md for details.
 
 #include "lor.hpp"
-#include "lor_assembly.hpp"
-#include "restriction.hpp"
 #include "pbilinearform.hpp"
 
-#include "../mfem-performance.hpp"
-
-#include "../general/forall.hpp"
-
-#define MFEM_DEBUG_COLOR 220
-#include "../general/debug.hpp"
-
-#define MFEM_NVTX_COLOR Turquoise
-#include "../general/nvtx.hpp"
-#include <limits.h>
-
 namespace mfem
 {
 
@@ -34,15 +21,12 @@ void LORBase::AddIntegrators(BilinearForm &a_from,
                              AddIntegratorFn add_integrator,
                              const IntegrationRule *ir)
 {
-   MFEM_NVTX;
    Array<BilinearFormIntegrator*> *integrators = (a_from.*get_integrators)();
    for (int i=0; i<integrators->Size(); ++i)
    {
-      BilinearFormIntegrator *integrator = (*integrators)[i];
-      if (!integrator->SupportsBatchedLOR()) { supports_batched_assembly = false; }
-      (a_to.*add_integrator)(integrator);
-      ir_map[integrator] = integrator->GetIntegrationRule();
-      if (ir) { integrator->SetIntegrationRule(*ir); }
+      (a_to.*add_integrator)((*integrators)[i]);
+      ir_map[(*integrators)[i]] = ((*integrators)[i])->GetIntegrationRule();
+      if (ir) { ((*integrators)[i])->SetIntegrationRule(*ir); }
    }
 }
 
@@ -54,24 +38,21 @@ void LORBase::AddIntegratorsAndMarkers(BilinearForm &a_from,
                                        AddIntegratorFn add_integrator,
                                        const IntegrationRule *ir)
 {
-   MFEM_NVTX;
    Array<BilinearFormIntegrator*> *integrators = (a_from.*get_integrators)();
    Array<Array<int>*> *markers = (a_from.*get_markers)();
 
    for (int i=0; i<integrators->Size(); ++i)
    {
-      BilinearFormIntegrator *integrator = (*integrators)[i];
       if (*markers[i])
       {
-         (a_to.*add_integrator_marker)(integrator, *(*markers[i]));
+         (a_to.*add_integrator_marker)((*integrators)[i], *(*markers[i]));
       }
       else
       {
-         (a_to.*add_integrator)(integrator);
+         (a_to.*add_integrator)((*integrators)[i]);
       }
-      if (!integrator->SupportsBatchedLOR()) { supports_batched_assembly = false; }
-      ir_map[integrator] = integrator->GetIntegrationRule();
-      if (ir) { integrator->SetIntegrationRule(*ir); }
+      ir_map[(*integrators)[i]] = ((*integrators)[i])->GetIntegrationRule();
+      if (ir) { ((*integrators)[i])->SetIntegrationRule(*ir); }
    }
 }
 
@@ -103,7 +84,6 @@ int LORBase::GetLOROrder() const
 
 void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
 {
-   MFEM_NVTX;
    FESpaceType type = GetFESpaceType();
    MFEM_VERIFY(type != H1 && type != L2, "");
 
@@ -223,7 +203,6 @@ void LORBase::ConstructLocalDofPermutation(Array<int> &perm_) const
 
 void LORBase::ConstructDofPermutation() const
 {
-   MFEM_NVTX;
    FESpaceType type = GetFESpaceType();
    if (type == H1 || type == L2)
    {
@@ -282,24 +261,8 @@ const OperatorHandle &LORBase::GetAssembledSystem() const
    return A;
 }
 
-const LORRestriction *LORBase::GetLORRestriction() const
-{
-   if (R_lor == NULL)
-   {
-      R_lor = new LORRestriction(fes_ho);
-   }
-   return R_lor;
-}
-
 void LORBase::AssembleSystem_(BilinearForm &a_ho, const Array<int> &ess_dofs)
 {
-   dbg();
-   MFEM_NVTX;
-   // By default, we want to use "batched assembly", however this is only
-   // supported for certain integrators. We set it to true here, and then when
-   // we loop through the integrators, if we encounter unsupported integrators,
-   // we set it to false.
-   supports_batched_assembly = true;
    a->UseExternalIntegrators();
    AddIntegrators(a_ho, *a, &BilinearForm::GetDBFI,
                   &BilinearForm::AddDomainIntegrator, ir_el);
@@ -313,35 +276,8 @@ void LORBase::AssembleSystem_(BilinearForm &a_ho, const Array<int> &ess_dofs)
                             &BilinearForm::GetBFBFI_Marker,
                             &BilinearForm::AddBdrFaceIntegrator,
                             &BilinearForm::AddBdrFaceIntegrator, ir_face);
-
-   if (supports_batched_assembly)
-   {
-      dbg("supports_batched_assembly");
-      fes_ho.GetMesh()->EnsureNodes();
-#ifdef MFEM_USE_MPI
-      ParFiniteElementSpace *pfes_ho =
-         dynamic_cast<ParFiniteElementSpace*>(&fes_ho);
-      if (pfes_ho)
-      {
-         dbg("=> PARALLEL AssembleBatchedLOR");
-         ParAssembleBatchedLOR(*this, *a, fes_ho, ess_dofs, A);
-      }
-      else
-      {
-         dbg("=> SEQUENTIAL AssembleBatchedLOR");
-         AssembleBatchedLOR(*this, *a, fes_ho, ess_dofs, A);
-      }
-#else
-      AssembleBatchedLOR(*this, *a, fes_ho, ess_dofs, A);
-#endif
-   }
-   else
-   {
-      dbg("NOT supports_batched_assembly");
-      a->Assemble();
-      a->FormSystemMatrix(ess_dofs, A);
-   }
-
+   a->Assemble();
+   a->FormSystemMatrix(ess_dofs, A);
    ResetIntegrationRules(&BilinearForm::GetDBFI);
    ResetIntegrationRules(&BilinearForm::GetFBFI);
    ResetIntegrationRules(&BilinearForm::GetBBFI);
@@ -350,7 +286,6 @@ void LORBase::AssembleSystem_(BilinearForm &a_ho, const Array<int> &ess_dofs)
 
 void LORBase::SetupProlongationAndRestriction()
 {
-   MFEM_NVTX;
    if (!HasSameDofNumbering())
    {
       Array<int> p;
@@ -413,7 +348,6 @@ void CheckBasisType(const FiniteElementSpace &fes)
 LORBase::LORBase(FiniteElementSpace &fes_ho_)
    : irs(0, Quadrature1D::GaussLobatto), fes_ho(fes_ho_)
 {
-   MFEM_NVTX;
    Mesh &mesh_ = *fes_ho_.GetMesh();
    int dim = mesh_.Dimension();
    Array<Geometry::Type> geoms;
@@ -429,8 +363,6 @@ LORBase::LORBase(FiniteElementSpace &fes_ho_)
       ir_face = NULL;
    }
    a = NULL;
-   R_lor = NULL;
-   supports_batched_assembly = true;
 }
 
 LORBase::~LORBase()
@@ -446,17 +378,14 @@ LORDiscretization::LORDiscretization(BilinearForm &a_ho_,
                                      int ref_type)
    : LORDiscretization(*a_ho_.FESpace(), ref_type)
 {
-   MFEM_NVTX;
    AssembleSystem(a_ho_, ess_tdof_list);
 }
 
 LORDiscretization::LORDiscretization(FiniteElementSpace &fes_ho,
                                      int ref_type) : LORBase(fes_ho)
 {
-   MFEM_NVTX;
    CheckBasisType(fes_ho);
 
-#if 0
    Mesh &mesh_ho = *fes_ho.GetMesh();
    // For H1, ND and RT spaces, use refinement = element order, for DG spaces,
    // use refinement = element order + 1 (since LOR is p = 0 in this case).
@@ -471,29 +400,21 @@ LORDiscretization::LORDiscretization(FiniteElementSpace &fes_ho,
    fec = fes_ho.FEColl()->Clone(GetLOROrder());
    fes = new FiniteElementSpace(mesh, fec);
    SetupProlongationAndRestriction();
-#else
-   mesh = NULL;
-   fec = NULL;
-   fes = NULL;
-#endif
+
    A.SetType(Operator::MFEM_SPARSEMAT);
 }
 
 void LORDiscretization::AssembleSystem(BilinearForm &a_ho,
                                        const Array<int> &ess_dofs)
 {
-   dbg();
-   MFEM_NVTX;
    delete a;
-   // a = new BilinearForm(&GetFESpace());
-   a = new BilinearForm(&fes_ho);
+   a = new BilinearForm(&GetFESpace());
    AssembleSystem_(a_ho, ess_dofs);
 }
 
 SparseMatrix &LORDiscretization::GetAssembledMatrix() const
 {
-   dbg();
-   MFEM_VERIFY(a != nullptr && A.Ptr() != nullptr, "No LOR system assembled");
+   MFEM_VERIFY(a != NULL && A.Ptr() != NULL, "No LOR system assembled");
    return *A.As<SparseMatrix>();
 }
 
@@ -504,26 +425,20 @@ ParLORDiscretization::ParLORDiscretization(ParBilinearForm &a_ho_,
                                            int ref_type)
    : ParLORDiscretization(*a_ho_.ParFESpace(), ref_type)
 {
-   dbg();
-   MFEM_NVTX;
    AssembleSystem(a_ho_, ess_tdof_list);
 }
 
 ParLORDiscretization::ParLORDiscretization(ParFiniteElementSpace &fes_ho,
                                            int ref_type) : LORBase(fes_ho)
 {
-   dbg();
-   MFEM_NVTX;
    if (fes_ho.GetMyRank() == 0) { CheckBasisType(fes_ho); }
    // TODO: support variable-order spaces in parallel
    MFEM_VERIFY(!fes_ho.IsVariableOrder(),
                "Cannot construct LOR operators on variable-order spaces");
 
-#if 0
    int order = fes_ho.GetMaxElementOrder();
    if (GetFESpaceType() == L2) { ++order; }
 
-   NVTX("ParMesh");
    ParMesh &mesh_ho = *fes_ho.GetParMesh();
    ParMesh *pmesh = new ParMesh(ParMesh::MakeRefined(mesh_ho, order, ref_type));
    mesh = pmesh;
@@ -532,11 +447,6 @@ ParLORDiscretization::ParLORDiscretization(ParFiniteElementSpace &fes_ho,
    ParFiniteElementSpace *pfes = new ParFiniteElementSpace(pmesh, fec);
    fes = pfes;
    SetupProlongationAndRestriction();
-#else
-   mesh = NULL;
-   fec = NULL;
-   fes = NULL;
-#endif
 
    A.SetType(Operator::Hypre_ParCSR);
 }
@@ -544,19 +454,14 @@ ParLORDiscretization::ParLORDiscretization(ParFiniteElementSpace &fes_ho,
 void ParLORDiscretization::AssembleSystem(ParBilinearForm &a_ho,
                                           const Array<int> &ess_dofs)
 {
-   dbg();
-   MFEM_NVTX;
    delete a;
-   // a = new ParBilinearForm(&GetParFESpace());
-   a = new ParBilinearForm(&dynamic_cast<ParFiniteElementSpace&>(fes_ho));
+   a = new ParBilinearForm(&GetParFESpace());
    AssembleSystem_(a_ho, ess_dofs);
 }
 
 HypreParMatrix &ParLORDiscretization::GetAssembledMatrix() const
 {
-   dbg();
-   MFEM_NVTX;
-   MFEM_VERIFY(a != nullptr && A.Ptr() != nullptr, "No LOR system assembled");
+   MFEM_VERIFY(a != NULL && A.Ptr() != NULL, "No LOR system assembled");
    return *A.As<HypreParMatrix>();
 }
 
@@ -565,368 +470,6 @@ ParFiniteElementSpace &ParLORDiscretization::GetParFESpace() const
    return static_cast<ParFiniteElementSpace&>(*fes);
 }
 
-#endif // MFEM_USE_MPI
-
-int LORRestriction::GetNRefinedElements(const FiniteElementSpace &fes)
-{
-   int ref = fes.GetMaxElementOrder();
-   int dim = fes.GetMesh()->Dimension();
-   return pow(ref, dim);
-}
-
-FiniteElementCollection *LORRestriction::GetLowOrderFEC(
-   const FiniteElementSpace &fes)
-{
-   return fes.FEColl()->Clone(1);
-}
-
-LORRestriction::LORRestriction(const FiniteElementSpace &fes_ho)
-   : fes_ho(fes_ho),
-     fec_lo(GetLowOrderFEC(fes_ho)),
-     geom(fes_ho.GetMesh()->GetElementGeometry(0)),
-     ne_ref(GetNRefinedElements(fes_ho)),
-     ne(fes_ho.GetNE()*ne_ref),
-     vdim(fes_ho.GetVDim()),
-     byvdim(fes_ho.GetOrdering() == Ordering::byVDIM),
-     ndofs(fes_ho.GetNDofs()),
-     dof(fec_lo->GetFE(geom, 1)->GetDof()),
-
-     offsets(ndofs+1),
-     indices(ne*dof),
-     gatherMap(ne*dof),
-
-     dof_glob2loc(),
-     dof_glob2loc_offsets(),
-     el_dof_lex()
-{
-   MFEM_NVTX;
-   SetupLocalToElement();
-   SetupGlobalToLocal();
-
-   NVTX("EnsureNodes");
-}
-
-void LORRestriction::SetupLocalToElement()
-{
-   MFEM_NVTX;
-   MFEM_VERIFY(ne>0, "ne==0 not supported");
-   const FiniteElement *fe = fec_lo->GetFE(geom, 1);
-   const TensorBasisElement* el =
-      dynamic_cast<const TensorBasisElement*>(fe);
-   MFEM_VERIFY(el, "!TensorBasisElement");
-
-   const Array<int> &fe_dof_map = el->GetDofMap();
-   MFEM_VERIFY(fe_dof_map.Size() > 0, "invalid dof map");
-
-   const FiniteElement *fe_ho = fes_ho.GetFE(0);
-   const TensorBasisElement* tel_ho =
-      dynamic_cast<const TensorBasisElement*>(fe_ho);
-   MFEM_VERIFY(tel_ho, "!TensorBasisElement");
-   const Array<int> &fe_dof_map_ho = tel_ho->GetDofMap();
-
-   int order = fes_ho.GetMaxElementOrder();
-   RefinedGeometry &RG = *GlobGeometryRefiner.Refine(geom, order);
-   Array<int> local_dof_map(dof*ne_ref);
-   for (int ie_lo = 0; ie_lo < ne_ref; ++ie_lo)
-   {
-      for (int i = 0; i < dof; ++i)
-      {
-         int cart_idx = RG.RefGeoms[i + dof*ie_lo]; // local Cartesian index
-         local_dof_map[i + dof*ie_lo] = fe_dof_map_ho[cart_idx];
-      }
-   }
-
-   const Table& e2dTable_ho = fes_ho.GetElementToDofTable();
-
-   auto d_offsets = offsets.Write();
-   const int NDOFS = ndofs;
-   dbg();
-   MFEM_FORALL(i, NDOFS+1, d_offsets[i] = 0;);
-
-   const Memory<int> &J = e2dTable_ho.GetJMemory();
-   const MemoryClass mc = Device::GetDeviceMemoryClass();
-   const int *d_elementMap = J.Read(mc, J.Capacity());
-   const int *d_local_dof_map = local_dof_map.Read();
-   const int DOF = dof;
-   const int DOF_ho = fe_ho->GetDof();
-   const int NE = ne;
-   const int NR_REF = ne_ref;
-
-   dbg();
-   MFEM_FORALL(e, NE,
-   {
-      const int e_ho = e/NR_REF;
-      const int i_ref = e%NR_REF;
-      for (int d = 0; d < DOF; ++d)
-      {
-         const int d_ho = d_local_dof_map[d + i_ref*DOF];
-         const int sgid = d_elementMap[DOF_ho*e_ho + d_ho];  // signed
-         const int gid = (sgid >= 0) ? sgid : -1 - sgid;
-         AtomicAdd(d_offsets[gid+1], 1);
-      }
-   });
-
-   // Aggregate to find offsets for each global dof
-   offsets.HostReadWrite();
-   for (int i = 1; i <= ndofs; ++i) { offsets[i] += offsets[i - 1]; }
-
-   // For each global dof, fill in all local nodes that point to it
-   auto d_gather = gatherMap.Write();
-   auto d_indices = indices.Write();
-   auto drw_offsets = offsets.ReadWrite();
-   const auto dof_map_mem = fe_dof_map.GetMemory();
-   const auto d_dof_map = fe_dof_map.GetMemory().Read(mc,dof_map_mem.Capacity());
-
-   dbg();
-   MFEM_FORALL(e, NE,
-   {
-      const int e_ho = e/NR_REF;
-      const int i_ref = e%NR_REF;
-      for (int d = 0; d < DOF; ++d)
-      {
-         int d_ho = d_local_dof_map[d + i_ref*DOF];
-         const int sdid = d_dof_map[d];  // signed
-         // const int did = d;
-         const int sgid = d_elementMap[DOF_ho*e_ho + d_ho];  // signed
-         const int gid = (sgid >= 0) ? sgid : -1-sgid;
-         const int lid = DOF*e + d;
-         const bool plus = (sgid >= 0 && sdid >= 0) || (sgid < 0 && sdid < 0);
-         d_gather[lid] = plus ? gid : -1-gid;
-         d_indices[AtomicAdd(drw_offsets[gid], 1)] = plus ? lid : -1-lid;
-      }
-   });
-
-   dbg();
-   offsets.HostReadWrite();
-   for (int i = ndofs; i > 0; --i) { offsets[i] = offsets[i - 1]; }
-   offsets[0] = 0;
-}
-
-void LORRestriction::SetupGlobalToLocal()
-{
-   MFEM_NVTX;
-   const int ndof = fes_ho.GetVSize();
-   const int nel_ho = fes_ho.GetMesh()->GetNE();
-   const int order = fes_ho.GetMaxElementOrder();
-   const int dim = fes_ho.GetMesh()->Dimension();
-   MFEM_VERIFY(dim==3, "Not supported");
-   const int nd1d = order + 1;
-   const int ndof_per_el = nd1d*nd1d*nd1d;
-
-   dof_glob2loc.SetSize(2*ndof_per_el*nel_ho);
-   dof_glob2loc_offsets.SetSize(ndof+1);
-   el_dof_lex.SetSize(ndof_per_el*nel_ho);
-
-   Array<int> dofs;
-
-   const Array<int> &lex_map =
-      dynamic_cast<const NodalFiniteElement&>
-      (*fes_ho.GetFE(0)).GetLexicographicOrdering();
-
-   dof_glob2loc_offsets = 0;
-   const Memory<int> &I = fes_ho.GetElementToDofTable().GetIMemory();
-   const Memory<int> &J = fes_ho.GetElementToDofTable().GetJMemory();
-   I.Read(MemoryClass::HOST, I.Capacity());
-   J.Read(MemoryClass::HOST, J.Capacity());
-
-   for (int iel_ho=0; iel_ho<nel_ho; ++iel_ho)
-   {
-      fes_ho.GetElementDofs(iel_ho, dofs);
-      for (int i=0; i<ndof_per_el; ++i)
-      {
-         const int dof = dofs[lex_map[i]];
-         el_dof_lex[i + iel_ho*ndof_per_el] = dof;
-         dof_glob2loc_offsets[dof+1] += 2;
-      }
-   }
-
-   dof_glob2loc_offsets.PartialSum();
-
-   // Sanity check
-   MFEM_VERIFY(dof_glob2loc_offsets[ndof] == dof_glob2loc.Size(), "");
-
-   Array<int> dof_ptr(ndof);
-
-   for (int i=0; i<ndof; ++i) { dof_ptr[i] = dof_glob2loc_offsets[i]; }
-
-   for (int iel_ho=0; iel_ho<nel_ho; ++iel_ho)
-   {
-      fes_ho.GetElementDofs(iel_ho, dofs);
-      for (int i=0; i<ndof_per_el; ++i)
-      {
-         const int dof = dofs[lex_map[i]];
-         dof_glob2loc[dof_ptr[dof]++] = iel_ho;
-         dof_glob2loc[dof_ptr[dof]++] = i;
-      }
-   }
-}
-
-static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
-                                      const int *nbr_elts, const int nbrNbElts)
-{
-   // Find the minimal element index found in both my_elts[] and nbr_elts[]
-   int min_el = INT_MAX;
-   for (int i = 0; i < nbElts; i++)
-   {
-      const int e_i = my_elts[i];
-      if (e_i >= min_el) { continue; }
-      for (int j = 0; j < nbrNbElts; j++)
-      {
-         if (e_i==nbr_elts[j])
-         {
-            min_el = e_i; // we already know e_i < min_el
-            break;
-         }
-      }
-   }
-   return min_el;
-}
-
-int LORRestriction::FillI(SparseMatrix &mat) const
-{
-   MFEM_NVTX;
-   static constexpr int Max = 16;
-   const int all_dofs = ndofs;
-   const int vd = vdim;
-   const int elt_dofs = dof;
-   auto I = mat.ReadWriteI();
-   auto d_offsets = offsets.Read();
-   auto d_indices = indices.Read();
-   auto d_gatherMap = gatherMap.Read();
-   MFEM_FORALL(i_L, vd*all_dofs+1, { I[i_L] = 0; });
-   MFEM_FORALL(e, ne,
-   {
-      for (int i = 0; i < elt_dofs; i++)
-      {
-         int i_elts[Max];
-         const int i_E = e*elt_dofs + i;
-         const int i_L = d_gatherMap[i_E];
-         const int i_offset = d_offsets[i_L];
-         const int i_nextOffset = d_offsets[i_L+1];
-         const int i_nbElts = i_nextOffset - i_offset;
-         for (int e_i = 0; e_i < i_nbElts; ++e_i)
-         {
-            const int i_E = d_indices[i_offset+e_i];
-            i_elts[e_i] = i_E/elt_dofs;
-         }
-         for (int j = 0; j < elt_dofs; j++)
-         {
-            const int j_E = e*elt_dofs + j;
-            const int j_L = d_gatherMap[j_E];
-            const int j_offset = d_offsets[j_L];
-            const int j_nextOffset = d_offsets[j_L+1];
-            const int j_nbElts = j_nextOffset - j_offset;
-            if (i_nbElts == 1 || j_nbElts == 1) // no assembly required
-            {
-               AtomicAdd(I[i_L],1);
-            }
-            else // assembly required
-            {
-               int j_elts[Max];
-               for (int e_j = 0; e_j < j_nbElts; ++e_j)
-               {
-                  const int j_E = d_indices[j_offset+e_j];
-                  const int elt = j_E/elt_dofs;
-                  j_elts[e_j] = elt;
-               }
-               const int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
-               if (e == min_e) // add the nnz only once
-               {
-                  AtomicAdd(I[i_L],1);
-               }
-            }
-         }
-      }
-   });
-   // We need to sum the entries of I, we do it on CPU as it is very sequential.
-   auto h_I = mat.HostReadWriteI();
-   const int nTdofs = vd*all_dofs;
-   int sum = 0;
-   for (int i = 0; i < nTdofs; i++)
-   {
-      const int nnz = h_I[i];
-      h_I[i] = sum;
-      sum+=nnz;
-   }
-   h_I[nTdofs] = sum;
-   // We return the number of nnz
-   return h_I[nTdofs];
-}
-
-void LORRestriction::FillJAndZeroData(SparseMatrix &mat) const
-{
-   MFEM_NVTX;
-   static constexpr int Max = 8;
-   const int all_dofs = ndofs;
-   const int vd = vdim;
-   const int elt_dofs = dof;
-   auto I = mat.ReadWriteI();
-   auto J = mat.WriteJ();
-   auto Data = mat.WriteData();
-   const int NE = ne;
-   auto d_offsets = offsets.Read();
-   auto d_indices = indices.Read();
-   auto d_gatherMap = gatherMap.Read();
-
-   MFEM_FORALL(e, NE,
-   {
-      for (int i = 0; i < elt_dofs; i++)
-      {
-         int i_elts[Max];
-         const int i_E = e*elt_dofs + i;
-         const int i_L = d_gatherMap[i_E];
-         const int i_offset = d_offsets[i_L];
-         const int i_nextOffset = d_offsets[i_L+1];
-         const int i_nbElts = i_nextOffset - i_offset;
-         for (int e_i = 0; e_i < i_nbElts; ++e_i)
-         {
-            const int i_E = d_indices[i_offset+e_i];
-            i_elts[e_i] = i_E/elt_dofs;
-         }
-         for (int j = 0; j < elt_dofs; j++)
-         {
-            const int j_E = e*elt_dofs + j;
-            const int j_L = d_gatherMap[j_E];
-            const int j_offset = d_offsets[j_L];
-            const int j_nextOffset = d_offsets[j_L+1];
-            const int j_nbElts = j_nextOffset - j_offset;
-            if (i_nbElts == 1 || j_nbElts == 1) // no assembly required
-            {
-               const int nnz = AtomicAdd(I[i_L],1);
-               J[nnz] = j_L;
-               Data[nnz] = 0.0;
-            }
-            else // assembly required
-            {
-               int j_elts[Max];
-               for (int e_j = 0; e_j < j_nbElts; ++e_j)
-               {
-                  const int j_E = d_indices[j_offset+e_j];
-                  const int elt = j_E/elt_dofs;
-                  j_elts[e_j] = elt;
-               }
-               const int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
-               if (e == min_e) // add the nnz only once
-               {
-                  const int nnz = AtomicAdd(I[i_L],1);
-                  J[nnz] = j_L;
-                  Data[nnz] = 0.0;
-               }
-            }
-         }
-      }
-   });
-   // We need to shift again the entries of I, we do it on CPU as it is very
-   // sequential.
-   auto h_I = mat.HostReadWriteI();
-   const int size = vd*all_dofs;
-   for (int i = 0; i < size; i++) { h_I[size-i] = h_I[size-(i+1)]; }
-   h_I[0] = 0;
-}
-
-LORRestriction::~LORRestriction()
-{
-   delete fec_lo;
-}
+#endif
 
 } // namespace mfem

fem/lor.hpp

@@ -69,10 +69,6 @@ protected:
    BilinearForm *a;
    OperatorHandle A;
    mutable Array<int> perm;
-   bool supports_batched_assembly;
-
-   /// The LOR element restriction operator.
-   mutable class LORRestriction *R_lor;
 
    /// Constructs the local DOF (ldof) permutation. In parallel this is used as
    /// an intermediate step in computing the DOF permutation (see
@@ -124,8 +120,6 @@ public:
    /// Returns the low-order refined finite element space.
    FiniteElementSpace &GetFESpace() const { return *fes; }
 
-   /// Returns the low-order restriction.
-   const LORRestriction *GetLORRestriction() const;
    ~LORBase();
 };
 
@@ -265,51 +259,6 @@ public:
    ~LORSolver() { if (own_lor) { delete lor; } }
 };
 
-
-/// Create a low-order refined version of a Restriction.
-/// Only used here for the FillI and FillJAndZeroData methods.
-class LORRestriction
-{
-   const FiniteElementSpace &fes_ho;
-   FiniteElementCollection *fec_lo;
-   const Geometry::Type geom;
-   const int ne_ref;
-   const int ne;
-   const int vdim;
-   const bool byvdim;
-   const int ndofs;
-   const int dof;
-
-   Array<int> offsets;
-   Array<int> indices;
-   Array<int> gatherMap;
-
-   Array<int> dof_glob2loc;
-   Array<int> dof_glob2loc_offsets;
-   Array<int> el_dof_lex;
-
-protected:
-   static int GetNRefinedElements(const FiniteElementSpace &fes);
-   static FiniteElementCollection *GetLowOrderFEC(const FiniteElementSpace &fes);
-
-public:
-   LORRestriction(const FiniteElementSpace &fes_ho);
-
-   int FillI(SparseMatrix &mat) const;
-   void FillJAndZeroData(SparseMatrix &mat) const;
-
-   const Array<int> &GatherMap() const { return el_dof_lex; }
-   const Array<int> &Indices() const { return dof_glob2loc; }
-   const Array<int> &Offsets() const { return dof_glob2loc_offsets; }
-
-   ~LORRestriction();
-
-   // Device lambda cannot have private or protected access
-public:
-   void SetupLocalToElement();
-   void SetupGlobalToLocal();
-};
-
 } // namespace mfem
 
 #endif

fem/lor_assembly.cpp

@@ -1,412 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "lor.hpp"
-#include "lor_assembly.hpp"
-#include "../linalg/dtensor.hpp"
-#include "../general/forall.hpp"
-
-#define MFEM_DEBUG_COLOR 226
-#include "../general/debug.hpp"
-
-#define MFEM_NVTX_COLOR DarkOrchid
-#include "../general/nvtx.hpp"
-
-#include <thrust/execution_policy.h>
-#include <thrust/scan.h>
-
-namespace mfem
-{
-
-// Defined in lor_assembly_ker.cpp
-template <int order, bool use_smem = true>
-void Assemble3DBatchedLOR(const Array<int> &dof_glob2loc,
-                          const Array<int> &dof_glob2loc_offsets,
-                          const Array<int> &el_dof_lex,
-                          Mesh &mesh_ho,
-                          SparseMatrix &A_mat);
-
-static void AssembleBatchedLORWithoutBC(LORBase &lor_disc,
-                                        BilinearForm &form_lor,
-                                        FiniteElementSpace &fes_ho,
-                                        OperatorHandle &Ah)
-{
-   MFEM_NVTX;
-   Mesh &mesh_ho = *fes_ho.GetMesh();
-   const int dim = mesh_ho.Dimension();
-   const int order = fes_ho.GetMaxElementOrder();
-   dbg("dim:%d order:%d",dim,order);
-
-   const bool has_to_init = Ah.Ptr() == nullptr;
-   dbg("has_to_init: %s", has_to_init?"yes":"no");
-   SparseMatrix *A = has_to_init ? nullptr : Ah.As<SparseMatrix>();
-
-   dbg("GetLORRestriction");
-   const LORRestriction *R = lor_disc.GetLORRestriction();
-   MFEM_VERIFY(R,"LOR Restriction error!");
-
-   if (has_to_init)
-   {
-      MFEM_VERIFY(UsesTensorBasis(fes_ho),
-                  "Batched LOR assembly requires tensor basis");
-      if (Device::IsEnabled()||true)
-      {
-         dbg("Device::IsEnabled()");
-#ifdef MFEM_USE_MPI
-         ParFiniteElementSpace *pfes_ho =
-            dynamic_cast<ParFiniteElementSpace*>(&fes_ho);
-         if (pfes_ho && pfes_ho->GetNRanks() > 1)
-         {
-            dbg("Device::IsEnabled() and multiple ranks!");
-            const int width = pfes_ho->GetVSize();
-            const int height = pfes_ho->GetVSize();
-            dbg("HxW: %dx%d",height,width);
-            A = new SparseMatrix(height, width, 0);
-         }
-         else
-#endif
-         {
-            dbg("Device::IsEnabled() but one rank!");
-            const int width = fes_ho.GetVSize();
-            const int height = fes_ho.GetVSize();
-            dbg("HxW: %dx%d",height,width);
-            A = new SparseMatrix(height, width, 0);
-         }
-         A->GetMemoryI().New(A->Height()+1, A->GetMemoryI().GetMemoryType());
-         const int nnz = R->FillI(*A);
-         A->GetMemoryJ().New(nnz, A->GetMemoryJ().GetMemoryType());
-         A->GetMemoryData().New(nnz, A->GetMemoryData().GetMemoryType());
-         R->FillJAndZeroData(*A); // J, A = 0.0
-      }
-      else
-      {
-         MFEM_ABORT("");
-         dbg("NOT Device::IsEnabled()");
-         // the sparsity pattern is defined from the map: element->dof
-         const int ndofs = fes_ho.GetVSize();
-         dbg("ndofs:%d",ndofs);
-         const Table &elem_dof = form_lor.FESpace()->GetElementToDofTable();
-         Table dof_dof, dof_elem;
-         Transpose(elem_dof, dof_elem, ndofs);
-         mfem::Mult(dof_elem, elem_dof, dof_dof);
-         dof_dof.SortRows();
-         int *I = dof_dof.GetI();
-         int *J = dof_dof.GetJ();
-         double *data = Memory<double>(I[ndofs]);
-         A = new SparseMatrix(I,J,data,ndofs,ndofs,true,true,true);
-         dof_dof.LoseData();
-         *A = 0.0;
-         dbg("done");
-      }
-   }
-
-   void (*Kernel)(const Array<int> &dof_glob2loc,
-                  const Array<int> &dof_glob2loc_offsets,
-                  const Array<int> &el_dof_lex,
-                  Mesh &mesh_ho,
-                  SparseMatrix &A_mat) = nullptr;
-
-   if (dim == 2) { MFEM_ABORT("Unsuported!"); }
-   else if (dim == 3)
-   {
-      switch (order)
-      {
-         case 1: Kernel = Assemble3DBatchedLOR<1>; break;
-         case 2: Kernel = Assemble3DBatchedLOR<2>; break;
-         case 3: Kernel = Assemble3DBatchedLOR<3>; break;
-         case 4: Kernel = Assemble3DBatchedLOR<4>; break;
-         case 5: Kernel = Assemble3DBatchedLOR<5>; break;
-         case 6: Kernel = Assemble3DBatchedLOR<6,false>; break;/*
-         case 7: Kernel = Assemble3DBatchedLOR<7,false>; break;
-         case 8: Kernel = Assemble3DBatchedLOR<8,false>; break;
-         case 9: Kernel = Assemble3DBatchedLOR<9,false>; break;
-         case 10: Kernel = Assemble3DBatchedLOR<10,false>; break;
-         case 11: Kernel = Assemble3DBatchedLOR<11,false>; break;
-         case 12: Kernel = Assemble3DBatchedLOR<12,false>; break;
-         case 13: Kernel = Assemble3DBatchedLOR<13,false>; break;
-         case 14: Kernel = Assemble3DBatchedLOR<14,false>; break;
-         case 15: Kernel = Assemble3DBatchedLOR<15,false>; break;
-         case 16: Kernel = Assemble3DBatchedLOR<16,false>; break;*/
-         default: MFEM_ABORT("Kernel not ready!");
-      }
-   }
-
-   Kernel(R->Indices(),
-          R->Offsets(),
-          R->GatherMap(),
-          mesh_ho, *A);
-
-   A->Finalize();
-
-   if (has_to_init) { Ah.Reset(A); } // A now owns A_mat
-}
-
-
-void AssembleBatchedLOR(LORBase &lor_disc,
-                        BilinearForm &form_lor,
-                        FiniteElementSpace &fes_ho,
-                        const Array<int> &ess_dofs,
-                        OperatorHandle &Ah)
-{
-   MFEM_NVTX;
-   AssembleBatchedLORWithoutBC(lor_disc, form_lor, fes_ho, Ah);
-
-   // Set essential dofs to 0.0
-   const int n_ess_dofs = ess_dofs.Size();
-   const auto ess_dofs_d = ess_dofs.Read();
-
-   const auto I = Ah.As<SparseMatrix>()->ReadI();
-   const auto J = Ah.As<SparseMatrix>()->ReadJ();
-   auto dA = Ah.As<SparseMatrix>()->ReadWriteData();
-
-   MFEM_FORALL(i, n_ess_dofs,
-   {
-      const int idof = ess_dofs_d[i];
-      for (int j=I[idof]; j<I[idof+1]; ++j)
-      {
-         const int jdof = J[j];
-         if (jdof != idof)
-         {
-            dA[j] = 0.0;
-            for (int k=I[jdof]; k<I[jdof+1]; ++k)
-            {
-               if (J[k] == idof)
-               {
-                  dA[k] = 0.0;
-                  break;
-               }
-            }
-         }
-      }
-   });
-}
-
-
-#ifdef MFEM_USE_MPI
-
-void ParAssembleBatchedLOR(LORBase &lor_disc,
-                           BilinearForm &form_lor,
-                           FiniteElementSpace &fes_ho,
-                           const Array<int> &ess_dofs,
-                           OperatorHandle &Ah)
-{
-   dbg();
-   MFEM_NVTX;
-   ParFiniteElementSpace *pfes_ho =
-      dynamic_cast<ParFiniteElementSpace*>(&fes_ho);
-   assert(pfes_ho);
-
-   OperatorHandle A_local(Operator::MFEM_SPARSEMAT);
-   AssembleBatchedLORWithoutBC(lor_disc, form_lor, fes_ho, A_local);
-   MFEM_VERIFY(A_local.As<SparseMatrix>()->Finalized(),
-               "the local matrix must be finalized");
-
-   NVTX("Parallel");
-   OperatorHandle dA(Operator::Hypre_ParCSR),
-                  Ph(Operator::Hypre_ParCSR);
-   {
-      NVTX("MakeSquareBlockDiag");
-      dA.MakeSquareBlockDiag(pfes_ho->GetComm(),
-                             pfes_ho->GlobalVSize(),
-                             pfes_ho->GetDofOffsets(),
-                             A_local.As<SparseMatrix>());
-   }
-   Ph.ConvertFrom(pfes_ho->Dof_TrueDof_Matrix());
-
-   {
-      NVTX("MakePtAP");
-      Ah.MakePtAP(dA, Ph);
-   }
-
-   {
-      dbg("EliminateRowsCols");
-      NVTX("EliminateRowsCols");
-      HypreParMatrix *A_mat = Ah.As<HypreParMatrix>();
-      hypre_ParCSRMatrix *A = *A_mat;
-      A_mat->HypreReadWrite();
-
-      hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(A);
-      hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(A);
-
-      HYPRE_Int diag_nrows = hypre_CSRMatrixNumRows(diag);
-      HYPRE_Int offd_ncols = hypre_CSRMatrixNumCols(offd);
-      dbg("diag_nrows:%d offd_ncols:%d", diag_nrows, offd_ncols);
-
-      const int n_ess_dofs = ess_dofs.Size();
-      const auto ess_dofs_d = ess_dofs.Read();
-      dbg("n_ess_dofs:%d", n_ess_dofs);
-
-      // Start communication to figure out which columns need to be eliminated in
-      // the off-diagonal block
-      hypre_ParCSRCommHandle *comm_handle;
-      HYPRE_Int *int_buf_data, *eliminate_row, *eliminate_col;
-      {
-         eliminate_row = hypre_CTAlloc(HYPRE_Int, diag_nrows, HYPRE_MEMORY_DEVICE);
-         eliminate_col = hypre_CTAlloc(HYPRE_Int, offd_ncols, HYPRE_MEMORY_DEVICE);
-
-         // Get the communication package for A, creating it if it does not
-         // already exist.
-         hypre_ParCSRCommPkg *comm_pkg = hypre_ParCSRMatrixCommPkg(A);
-         if (!comm_pkg)
-         {
-            hypre_MatvecCommPkgCreate(A);
-            comm_pkg = hypre_ParCSRMatrixCommPkg(A);
-         }
-
-         MFEM_FORALL(i, diag_nrows,
-         {
-            eliminate_row[i] = 0;
-         });
-         MFEM_FORALL(i, n_ess_dofs,
-         {
-            eliminate_row[ess_dofs_d[i]] = 1;
-         });
-
-         // Use a matvec communication pattern to find (in eliminate_col) which of
-         // the local offd columns are to be eliminated
-         HYPRE_Int num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-         HYPRE_Int int_buf_sz = hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends);
-         dbg("num_sends:%d", num_sends);
-         int_buf_data = hypre_CTAlloc(HYPRE_Int, int_buf_sz, HYPRE_MEMORY_DEVICE);
-
-         hypre_ParCSRCommPkgCopySendMapElmtsToDevice(comm_pkg);
-         HYPRE_Int *send_map_elmts = hypre_ParCSRCommPkgDeviceSendMapElmts(comm_pkg);
-         MFEM_FORALL(i, int_buf_sz,
-         {
-            int k = send_map_elmts[i];
-            int_buf_data[i] = eliminate_row[k];
-         });
-
-         // Try to use device-aware MPI for the communication
-         comm_handle = hypre_ParCSRCommHandleCreate_v2(
-            11, comm_pkg, HYPRE_MEMORY_DEVICE, int_buf_data,
-            HYPRE_MEMORY_DEVICE, eliminate_col);
-      }
-
-      // Eliminate rows and columns in the diagonal block
-      {
-         dbg("Eliminate rows and columns in the diagonal block");
-         const auto I = diag->i;
-         const auto J = diag->j;
-         auto data = diag->data;
-
-         MFEM_FORALL(i, n_ess_dofs,
-         {
-            const int idof = ess_dofs_d[i];
-            for (int j=I[idof]; j<I[idof+1]; ++j)
-            {
-               const int jdof = J[j];
-               if (jdof != idof)
-               {
-                  data[j] = 0.0;
-                  for (int k=I[jdof]; k<I[jdof+1]; ++k)
-                  {
-                     if (J[k] == idof)
-                     {
-                        data[k] = 0.0;
-                        break;
-                     }
-                  }
-               }
-            }
-         });
-      }
-
-      // Eliminate rows in the off-diagonal block
-      {
-         dbg("Eliminate rows in the off-diagonal block");
-         const auto I = offd->i;
-         auto data = offd->data;
-         MFEM_FORALL(i, n_ess_dofs,
-         {
-            const int idof = ess_dofs_d[i];
-            for (int j=I[idof]; j<I[idof+1]; ++j)
-            {
-               data[j] = 0.0;
-            }
-         });
-      }
-
-      // Wait for MPI communication to finish
-      Array<HYPRE_Int> cols_to_eliminate;
-      {
-         dbg("Wait for MPI communication to finish");
-         hypre_ParCSRCommHandleDestroy(comm_handle);
-
-         // Convert a "column marker" array to a "column index" array.
-
-         // First we need to know how many columns are marked, and do a partial
-         // sum to figure out which index they correspond to.
-         HYPRE_Int *col_idx = hypre_CTAlloc(HYPRE_Int, offd_ncols, HYPRE_MEMORY_DEVICE);
-         thrust::exclusive_scan(
-            thrust::device, eliminate_col, eliminate_col + offd_ncols, col_idx
-         );
-
-         // Because we do an exclusive scan above, the last entry is missing
-         // from the partial sum. To compute the total sum, get last entry of
-         // both arrays and add them.
-         HYPRE_Int partial_sum, last_element;
-         hypre_TMemcpy(&partial_sum, col_idx + offd_ncols - 1, HYPRE_Int, 1, HYPRE_MEMORY_HOST, HYPRE_MEMORY_DEVICE);
-         hypre_TMemcpy(&last_element, eliminate_col + offd_ncols - 1, HYPRE_Int, 1, HYPRE_MEMORY_HOST, HYPRE_MEMORY_DEVICE);
-         HYPRE_Int ncols_to_eliminate = partial_sum + last_element;
-
-         // Perform the conversion to "column index" array.
-         cols_to_eliminate.SetSize(ncols_to_eliminate);
-         HYPRE_Int *cols = cols_to_eliminate.Write();
-         MFEM_FORALL(i, offd_ncols,
-         {
-            if (eliminate_col[i])
-            {
-               cols[col_idx[i]] = i;
-            }
-         });
-
-         hypre_TFree(int_buf_data, HYPRE_MEMORY_DEVICE);
-         hypre_TFree(eliminate_row, HYPRE_MEMORY_DEVICE);
-         hypre_TFree(eliminate_col, HYPRE_MEMORY_DEVICE);
-         hypre_TFree(col_idx, HYPRE_MEMORY_DEVICE);
-      }
-
-      // Eliminate columns in the off-diagonal block
-      {
-         dbg("Eliminate columns in the off-diagonal block");
-         const int ncols_to_eliminate = cols_to_eliminate.Size();
-         const int nrows_offd = hypre_CSRMatrixNumRows(offd);
-         const auto cols = cols_to_eliminate.Read();
-         const auto I = offd->i;
-         const auto J = offd->j;
-         auto data = offd->data;
-         dbg("ncols_to_eliminate:%d nrows_offd:%d", ncols_to_eliminate, nrows_offd);
-         // Note: could also try a different strategy, looping over nnz in the
-         // matrix and then doing a binary search in ncols_to_eliminate to see if
-         // the column should be eliminated.
-         MFEM_FORALL(idx, ncols_to_eliminate,
-         {
-            const int j = cols[idx];
-            for (int i=0; i<nrows_offd; ++i)
-            {
-               for (int jj=I[i]; jj<I[i+1]; ++jj)
-               {
-                  if (J[jj] == j)
-                  {
-                     data[jj] = 0.0;
-                     break;
-                  }
-               }
-            }
-         });
-      }
-   }
-}
-
-#endif // MFEM_USE_MPI
-
-} // namespace mfem

fem/lor_assembly.hpp

@@ -1,40 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_LOR_ASSEMBLY
-#define MFEM_LOR_ASSEMBLY
-
-#include "lor.hpp"
-#include "bilinearform.hpp"
-#include "pbilinearform.hpp"
-
-namespace mfem
-{
-
-void AssembleBatchedLOR(LORBase &lor_disc,
-                        BilinearForm &form_lo,
-                        FiniteElementSpace &fes_ho,
-                        const Array<int> &ess_dofs,
-                        OperatorHandle &A);
-
-#ifdef MFEM_USE_MPI
-
-void ParAssembleBatchedLOR(LORBase &lor_disc,
-                           BilinearForm &form_lo,
-                           FiniteElementSpace &fes_ho,
-                           const Array<int> &ess_dofs,
-                           OperatorHandle &A);
-
-#endif // MFEM_USE_MPI
-
-} // namespace mfem
-
-#endif

fem/lor_assembly_ker.cpp

@@ -1,533 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-#include "../general/forall.hpp"
-
-#define MFEM_DEBUG_COLOR 187
-#include "../general/debug.hpp"
-
-#define MFEM_NVTX_COLOR SlateBlue
-#include "../general/nvtx.hpp"
-
-namespace mfem
-{
-
-template<int D1D, int Q1D>
-void NodalInterpolation3D(const int NE,
-                          const Vector& localL, Vector& localH,
-                          const Array<double>& B);
-
-template <int order, bool USE_SMEM = true>
-void Assemble3DBatchedLOR(const Array<int> &dof_glob2loc_,
-                          const Array<int> &dof_glob2loc_offsets_,
-                          const Array<int> &el_dof_lex_,
-                          Mesh &mesh_ho,
-                          SparseMatrix &A_mat)
-{
-   const int nel_ho = mesh_ho.GetNE();
-
-   static constexpr int nv = 8;
-   static constexpr int dim = 3;
-   static constexpr int ddm2 = (dim*(dim+1))/2;
-   static constexpr int nd1d = order + 1;
-   static constexpr int ndof_per_el = nd1d*nd1d*nd1d;
-   static constexpr int nnz_per_row = 27;
-   static constexpr int nnz_per_el = nnz_per_row * ndof_per_el;
-   static constexpr int sz_grad_A = 3*3*2*2*2*2;
-   static constexpr int sz_grad_B = sz_grad_A*2;
-   static constexpr int sz_local_mat = 8*8;
-
-   static constexpr int GRID = USE_SMEM ? 0 : 128;
-   double *GM = nullptr;
-   static Vector *d_buffer = nullptr;
-   if (!USE_SMEM)
-   {
-      if (!d_buffer)
-      {
-         d_buffer = new Vector();
-         d_buffer->UseDevice(true);
-      }
-      d_buffer->SetSize(nnz_per_el*GRID);
-      GM = d_buffer->Write();
-   }
-
-   const auto el_dof_lex = Reshape(el_dof_lex_.Read(), ndof_per_el, nel_ho);
-   const auto dof_glob2loc = dof_glob2loc_.Read();
-   const auto K = dof_glob2loc_offsets_.Read();
-
-   const auto I = A_mat.ReadI();
-   const auto J = A_mat.ReadJ();
-   auto A = A_mat.ReadWriteData();
-
-   const GridFunction *nodal_gf = mesh_ho.GetNodes();
-   const FiniteElementSpace *nodal_fes = nodal_gf->FESpace();
-   const Operator *nodal_restriction = nodal_fes->GetElementRestriction(
-                                          ElementDofOrdering::LEXICOGRAPHIC);
-   const int nodal_nd1d = nodal_fes->GetMaxElementOrder() + 1;
-
-   IntegrationRules irs(0, Quadrature1D::GaussLobatto);
-   const IntegrationRule &ir = irs.Get(Geometry::Type::CUBE, 2*nd1d - 3);
-   MFEM_VERIFY(ir.Size() == ndof_per_el, "");
-
-   // Get the map from mesh nodes to LOR vertices
-   const DofToQuad& maps =
-      nodal_fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-
-   // Map from nodal E-vector to L-vector
-   Vector nodes_loc(nodal_restriction->Height());
-   nodes_loc.UseDevice(true);
-   nodal_restriction->Mult(*nodal_gf, nodes_loc);
-
-   // Get nodal points at the LOR vertices
-   Vector X_loc(dim*ndof_per_el*nel_ho);
-   X_loc.UseDevice(true);
-
-   // Get the LOR vertex coordinates
-   MFEM_VERIFY(nd1d==order+1, "nd1d!=order+1");
-   switch (nodal_nd1d)
-   {
-      case 2:
-      {
-         NodalInterpolation3D<2,nd1d>(nel_ho, nodes_loc, X_loc, maps.B);
-         break;
-      }
-      case 4:
-      {
-         NodalInterpolation3D<4,nd1d>(nel_ho, nodes_loc, X_loc, maps.B);
-         break;
-      }
-      case 6:
-      {
-         NodalInterpolation3D<6,nd1d>(nel_ho, nodes_loc, X_loc, maps.B);
-         break;
-      }
-      default: MFEM_ABORT("Unsuported mesh order!");
-   }
-   auto X = X_loc.Read();
-
-   // Last GRID dimension is lowered to avoid too many resources
-   MFEM_FORALL_3D_GRID(iel_ho, nel_ho, order, order, USE_SMEM?order:1, GRID,
-   {
-      const int bid = MFEM_BLOCK_ID(x);
-      MFEM_SHARED double smem[USE_SMEM ? nnz_per_el : 1];
-      double *V_ = USE_SMEM ? smem : GM + nnz_per_el*bid;
-      DeviceTensor<4> V(V_, nnz_per_row, nd1d, nd1d, nd1d);
-
-      // Assemble a sparse matrix over the macro-element by looping over each
-      // subelement.
-      // V(j,i) stores the jth nonzero in the ith row of the sparse matrix.
-      MFEM_FOREACH_THREAD(iz,z,nd1d)
-      {
-         MFEM_FOREACH_THREAD(iy,y,nd1d)
-         {
-            MFEM_FOREACH_THREAD(ix,x,nd1d)
-            {
-               //MFEM_UNROLL(27)
-               for (int j=0; j<nnz_per_row; ++j)
-               {
-                  V(j,ix,iy,iz) = 0.0;
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Compute geometric factors at quadrature points
-      MFEM_FOREACH_THREAD(kz,z,order)
-      {
-         MFEM_FOREACH_THREAD(ky,y,order)
-         {
-            MFEM_FOREACH_THREAD(kx,x,order)
-            {
-               double Q_[ddm2*nv];
-               double grad_A_[sz_grad_A];
-               double grad_B_[sz_grad_B];
-               double local_mat_[sz_local_mat];
-
-               DeviceTensor<4> Q(Q_, ddm2,2,2,2);
-               DeviceTensor<2> local_mat(local_mat_, 8, 8);
-               DeviceTensor<6> grad_A(grad_A_, 3, 3, 2, 2, 2, 2);
-               DeviceTensor<7> grad_B(grad_B_, 3, 3, 2, 2, 2, 2, 2);
-
-               // local_mat is the local (dense) stiffness matrix
-               for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }
-
-               // Intermediate quantities
-               // (see e.g. Mora and Demkowicz for notation).
-               for (int i=0; i<sz_grad_A; ++i) { grad_A[i] = 0.0; }
-               for (int i=0; i<sz_grad_B; ++i) { grad_B[i] = 0.0; }
-
-               const int v0 = kx + nd1d*(ky + nd1d*kz);
-               const int v1 = kx + 1 + nd1d*(ky + nd1d*kz);
-               const int v2 = kx + 1 + nd1d*(ky + 1 + nd1d*kz);
-               const int v3 = kx + nd1d*(ky + 1 + nd1d*kz);
-               const int v4 = kx + nd1d*(ky + nd1d*(kz + 1));
-               const int v5 = kx + 1 + nd1d*(ky + nd1d*(kz + 1));
-               const int v6 = kx + 1 + nd1d*(ky + 1 + nd1d*(kz + 1));
-               const int v7 = kx + nd1d*(ky + 1 + nd1d*(kz + 1));
-
-               const int e0 = dim*(v0 + ndof_per_el*iel_ho);
-               const int e1 = dim*(v1 + ndof_per_el*iel_ho);
-               const int e2 = dim*(v2 + ndof_per_el*iel_ho);
-               const int e3 = dim*(v3 + ndof_per_el*iel_ho);
-               const int e4 = dim*(v4 + ndof_per_el*iel_ho);
-               const int e5 = dim*(v5 + ndof_per_el*iel_ho);
-               const int e6 = dim*(v6 + ndof_per_el*iel_ho);
-               const int e7 = dim*(v7 + ndof_per_el*iel_ho);
-
-               const double v0x = X[e0 + 0];
-               const double v0y = X[e0 + 1];
-               const double v0z = X[e0 + 2];
-
-               const double v1x = X[e1 + 0];
-               const double v1y = X[e1 + 1];
-               const double v1z = X[e1 + 2];
-
-               const double v2x = X[e2 + 0];
-               const double v2y = X[e2 + 1];
-               const double v2z = X[e2 + 2];
-
-               const double v3x = X[e3 + 0];
-               const double v3y = X[e3 + 1];
-               const double v3z = X[e3 + 2];
-
-               const double v4x = X[e4 + 0];
-               const double v4y = X[e4 + 1];
-               const double v4z = X[e4 + 2];
-
-               const double v5x = X[e5 + 0];
-               const double v5y = X[e5 + 1];
-               const double v5z = X[e5 + 2];
-
-               const double v6x = X[e6 + 0];
-               const double v6y = X[e6 + 1];
-               const double v6z = X[e6 + 2];
-
-               const double v7x = X[e7 + 0];
-               const double v7y = X[e7 + 1];
-               const double v7z = X[e7 + 2];
-
-               //MFEM_UNROLL(2)
-               for (int iqz=0; iqz<2; ++iqz)
-               {
-                  //MFEM_UNROLL(2)
-                  for (int iqy=0; iqy<2; ++iqy)
-                  {
-                     //MFEM_UNROLL(2)
-                     for (int iqx=0; iqx<2; ++iqx)
-                     {
-
-                        const double x = iqx;
-                        const double y = iqy;
-                        const double z = iqz;
-                        const double w = 1.0/8.0;
-
-                        // c: (1-x)(1-y)(1-z)v0[c] + x (1-y)(1-z)v1[c] + x y (1-z)v2[c] + (1-x) y (1-z)v3[c]
-                        //  + (1-x)(1-y) z   v4[c] + x (1-y) z   v5[c] + x y z    v6[c] + (1-x) y z    v7[c]
-                        const double J11 = -(1-y)*(1-z)*v0x
-                                           + (1-y)*(1-z)*v1x + y*(1-z)*v2x - y*(1-z)*v3x
-                                           - (1-y)*z*v4x + (1-y)*z*v5x + y*z*v6x - y*z*v7x;
-
-                        const double J12 = -(1-x)*(1-z)*v0x
-                                           - x*(1-z)*v1x + x*(1-z)*v2x + (1-x)*(1-z)*v3x
-                                           - (1-x)*z*v4x - x*z*v5x + x*z*v6x + (1-x)*z*v7x;
-
-                        const double J13 = -(1-x)*(1-y)*v0x - x*(1-y)*v1x
-                                           - x*y*v2x - (1-x)*y*v3x + (1-x)*(1-y)*v4x
-                                           + x*(1-y)*v5x + x*y*v6x + (1-x)*y*v7x;
-
-                        const double J21 = -(1-y)*(1-z)*v0y + (1-y)*(1-z)*v1y
-                                           + y*(1-z)*v2y - y*(1-z)*v3y - (1-y)*z*v4y
-                                           + (1-y)*z*v5y + y*z*v6y - y*z*v7y;
-
-                        const double J22 = -(1-x)*(1-z)*v0y - x*(1-z)*v1y
-                                           + x*(1-z)*v2y + (1-x)*(1-z)*v3y- (1-x)*z*v4y -
-                                           x*z*v5y + x*z*v6y + (1-x)*z*v7y;
-
-                        const double J23 = -(1-x)*(1-y)*v0y - x*(1-y)*v1y
-                                           - x*y*v2y - (1-x)*y*v3y + (1-x)*(1-y)*v4y
-                                           + x*(1-y)*v5y + x*y*v6y + (1-x)*y*v7y;
-
-                        const double J31 = -(1-y)*(1-z)*v0z + (1-y)*(1-z)*v1z
-                                           + y*(1-z)*v2z - y*(1-z)*v3z- (1-y)*z*v4z +
-                                           (1-y)*z*v5z + y*z*v6z - y*z*v7z;
-
-                        const double J32 = -(1-x)*(1-z)*v0z - x*(1-z)*v1z
-                                           + x*(1-z)*v2z + (1-x)*(1-z)*v3z - (1-x)*z*v4z
-                                           - x*z*v5z + x*z*v6z + (1-x)*z*v7z;
-
-                        const double J33 = -(1-x)*(1-y)*v0z - x*(1-y)*v1z
-                                           - x*y*v2z - (1-x)*y*v3z + (1-x)*(1-y)*v4z
-                                           + x*(1-y)*v5z + x*y*v6z + (1-x)*y*v7z;
-
-                        const double detJ = J11 * (J22 * J33 - J32 * J23) -
-                                            J21 * (J12 * J33 - J32 * J13) +
-                                            J31 * (J12 * J23 - J22 * J13);
-                        const double w_detJ = w/detJ;
-
-                        // adj(J)
-                        const double A11 = (J22 * J33) - (J23 * J32);
-                        const double A12 = (J32 * J13) - (J12 * J33);
-                        const double A13 = (J12 * J23) - (J22 * J13);
-                        const double A21 = (J31 * J23) - (J21 * J33);
-                        const double A22 = (J11 * J33) - (J13 * J31);
-                        const double A23 = (J21 * J13) - (J11 * J23);
-                        const double A31 = (J21 * J32) - (J31 * J22);
-                        const double A32 = (J31 * J12) - (J11 * J32);
-                        const double A33 = (J11 * J22) - (J12 * J21);
-
-                        Q(0,iqz,iqy,iqx) = w_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-                        Q(1,iqz,iqy,iqx) = w_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-                        Q(2,iqz,iqy,iqx) = w_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-                        Q(3,iqz,iqy,iqx) = w_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-                        Q(4,iqz,iqy,iqx) = w_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-                        Q(5,iqz,iqy,iqx) = w_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
-                     }
-                  }
-               }
-
-               //MFEM_UNROLL(2)
-               for (int iqx=0; iqx<2; ++iqx)
-               {
-                  //MFEM_UNROLL(2)
-                  for (int jz=0; jz<2; ++jz)
-                  {
-                     // Note loop starts at iz=jz here, taking advantage of
-                     // symmetries.
-                     //MFEM_UNROLL(2)
-                     for (int iz=jz; iz<2; ++iz)
-                     {
-                        //MFEM_UNROLL(2)
-                        for (int iqy=0; iqy<2; ++iqy)
-                        {
-                           //MFEM_UNROLL(2)
-                           for (int iqz=0; iqz<2; ++iqz)
-                           {
-                              const double biz = (iz == iqz) ? 1.0 : 0.0;
-                              const double giz = (iz == 0) ? -1.0 : 1.0;
-
-                              const double bjz = (jz == iqz) ? 1.0 : 0.0;
-                              const double gjz = (jz == 0) ? -1.0 : 1.0;
-
-                              const double J11 = Q(0,iqz,iqy,iqx);
-                              const double J21 = Q(1,iqz,iqy,iqx);
-                              const double J31 = Q(2,iqz,iqy,iqx);
-                              const double J12 = J21;
-                              const double J22 = Q(3,iqz,iqy,iqx);
-                              const double J32 = Q(4,iqz,iqy,iqx);
-                              const double J13 = J31;
-                              const double J23 = J32;
-                              const double J33 = Q(5,iqz,iqy,iqx);
-
-                              grad_A(0,0,iqy,iz,jz,iqx) += J11*biz*bjz;
-                              grad_A(1,0,iqy,iz,jz,iqx) += J21*biz*bjz;
-                              grad_A(2,0,iqy,iz,jz,iqx) += J31*giz*bjz;
-                              grad_A(0,1,iqy,iz,jz,iqx) += J12*biz*bjz;
-                              grad_A(1,1,iqy,iz,jz,iqx) += J22*biz*bjz;
-                              grad_A(2,1,iqy,iz,jz,iqx) += J32*giz*bjz;
-                              grad_A(0,2,iqy,iz,jz,iqx) += J13*biz*gjz;
-                              grad_A(1,2,iqy,iz,jz,iqx) += J23*biz*gjz;
-                              grad_A(2,2,iqy,iz,jz,iqx) += J33*giz*gjz;
-                           }
-                           //MFEM_UNROLL(2)
-                           for (int jy=0; jy<2; ++jy)
-                           {
-                              //MFEM_UNROLL(2)
-                              for (int iy=0; iy<2; ++iy)
-                              {
-                                 const double biy = (iy == iqy) ? 1.0 : 0.0;
-                                 const double giy = (iy == 0) ? -1.0 : 1.0;
-
-                                 const double bjy = (jy == iqy) ? 1.0 : 0.0;
-                                 const double gjy = (jy == 0) ? -1.0 : 1.0;
-
-                                 grad_B(0,0,iy,jy,iz,jz,iqx) += biy*bjy*grad_A(0,0,iqy,iz,jz,iqx);
-                                 grad_B(1,0,iy,jy,iz,jz,iqx) += giy*bjy*grad_A(1,0,iqy,iz,jz,iqx);
-                                 grad_B(2,0,iy,jy,iz,jz,iqx) += biy*bjy*grad_A(2,0,iqy,iz,jz,iqx);
-                                 grad_B(0,1,iy,jy,iz,jz,iqx) += biy*gjy*grad_A(0,1,iqy,iz,jz,iqx);
-                                 grad_B(1,1,iy,jy,iz,jz,iqx) += giy*gjy*grad_A(1,1,iqy,iz,jz,iqx);
-                                 grad_B(2,1,iy,jy,iz,jz,iqx) += biy*gjy*grad_A(2,1,iqy,iz,jz,iqx);
-                                 grad_B(0,2,iy,jy,iz,jz,iqx) += biy*bjy*grad_A(0,2,iqy,iz,jz,iqx);
-                                 grad_B(1,2,iy,jy,iz,jz,iqx) += giy*bjy*grad_A(1,2,iqy,iz,jz,iqx);
-                                 grad_B(2,2,iy,jy,iz,jz,iqx) += biy*bjy*grad_A(2,2,iqy,iz,jz,iqx);
-                              }
-                           }
-                        }
-                        //MFEM_UNROLL(2)
-                        for (int jy=0; jy<2; ++jy)
-                        {
-                           //MFEM_UNROLL(2)
-                           for (int jx=0; jx<2; ++jx)
-                           {
-                              //MFEM_UNROLL(2)
-                              for (int iy=0; iy<2; ++iy)
-                              {
-                                 //MFEM_UNROLL(2)
-                                 for (int ix=0; ix<2; ++ix)
-                                 {
-                                    const double bix = (ix == iqx) ? 1.0 : 0.0;
-                                    const double gix = (ix == 0) ? -1.0 : 1.0;
-
-                                    const double bjx = (jx == iqx) ? 1.0 : 0.0;
-                                    const double gjx = (jx == 0) ? -1.0 : 1.0;
-
-                                    int ii_loc = ix + 2*iy + 4*iz;
-                                    int jj_loc = jx + 2*jy + 4*jz;
-
-                                    // Only store the lower-triangular part of
-                                    // the matrix (by symmetry).
-                                    if (jj_loc > ii_loc) { continue; }
-
-                                    double val = 0.0;
-                                    val += gix*gjx*grad_B(0,0,iy,jy,iz,jz,iqx);
-                                    val += bix*gjx*grad_B(1,0,iy,jy,iz,jz,iqx);
-                                    val += bix*gjx*grad_B(2,0,iy,jy,iz,jz,iqx);
-                                    val += gix*bjx*grad_B(0,1,iy,jy,iz,jz,iqx);
-                                    val += bix*bjx*grad_B(1,1,iy,jy,iz,jz,iqx);
-                                    val += bix*bjx*grad_B(2,1,iy,jy,iz,jz,iqx);
-                                    val += gix*bjx*grad_B(0,2,iy,jy,iz,jz,iqx);
-                                    val += bix*bjx*grad_B(2,2,iy,jy,iz,jz,iqx);
-                                    val += bix*bjx*grad_B(1,2,iy,jy,iz,jz,iqx);
-
-                                    local_mat(ii_loc, jj_loc) += val;
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-               }
-               // Assemble the local matrix into the macro-element sparse matrix
-               // in a format similar to coordinate format. The (I,J) arrays
-               // are implicit (not stored explicitly).
-               //MFEM_UNROLL(8)
-               for (int ii_loc=0; ii_loc<8; ++ii_loc)
-               {
-                  const int ix = ii_loc%2;
-                  const int iy = (ii_loc/2)%2;
-                  const int iz = ii_loc/2/2;
-
-                  for (int jj_loc=0; jj_loc<8; ++jj_loc)
-                  {
-                     const int jx = jj_loc%2;
-                     const int jy = (jj_loc/2)%2;
-                     const int jz = jj_loc/2/2;
-                     const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
-
-                     if (jj_loc <= ii_loc)
-                     {
-                        AtomicAdd(V(jj_off, ix+kx, iy+ky, iz+kz), local_mat(ii_loc, jj_loc));
-                     }
-                     else
-                     {
-                        AtomicAdd(V(jj_off, ix+kx, iy+ky, iz+kz), local_mat(jj_loc, ii_loc));
-                     }
-                  }
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Place the macro-element sparse matrix into the global sparse matrix.
-      MFEM_FOREACH_THREAD(iz,z,nd1d)
-      {
-         MFEM_FOREACH_THREAD(iy,y,nd1d)
-         {
-            MFEM_FOREACH_THREAD(ix,x,nd1d)
-            {
-               double col_ptr[nnz_per_row]; // 27
-
-               const int ii_el = ix + nd1d*(iy + nd1d*iz);
-               const int ii = el_dof_lex(ii_el, iel_ho);
-
-               // Set column pointer to avoid searching in the row
-               for (int j = I[ii], end = I[ii+1]; j < end; j++)
-               {
-                  const int jj = J[j];
-                  int jj_el = -1;
-                  for (int k = K[jj], k_end = K[jj+1]; k < k_end; k += 2)
-                  {
-                     if (dof_glob2loc[k] == iel_ho)
-                     {
-                        jj_el = dof_glob2loc[k+1];
-                        break;
-                     }
-                  }
-                  if (jj_el < 0) { continue; }
-                  const int jx = jj_el%nd1d;
-                  const int jy = (jj_el/nd1d)%nd1d;
-                  const int jz = jj_el/nd1d/nd1d;
-                  const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
-                  col_ptr[jj_off] = j;
-               }
-
-               const int jx_begin = (ix > 0) ? ix - 1 : 0;
-               const int jx_end = (ix < order) ? ix + 1 : order;
-
-               const int jy_begin = (iy > 0) ? iy - 1 : 0;
-               const int jy_end = (iy < order) ? iy + 1 : order;
-
-               const int jz_begin = (iz > 0) ? iz - 1 : 0;
-               const int jz_end = (iz < order) ? iz + 1 : order;
-
-               for (int jz=jz_begin; jz<=jz_end; ++jz)
-               {
-                  for (int jy=jy_begin; jy<=jy_end; ++jy)
-                  {
-                     for (int jx=jx_begin; jx<=jx_end; ++jx)
-                     {
-                        const int jj_off = (jx-ix+1) + 3*(jy-iy+1) + 9*(jz-iz+1);
-                        const double Vji = V(jj_off, ix, iy, iz);
-                        const int col_ptr_jj = col_ptr[jj_off];
-                        if ((ix == 0 && jx == 0) || (ix == order && jx == order) ||
-                            (iy == 0 && jy == 0) || (iy == order && jy == order) ||
-                            (iz == 0 && jz == 0) || (iz == order && jz == order))
-                        {
-                           AtomicAdd(A[col_ptr_jj], Vji);
-                        }
-                        else
-                        {
-                           A[col_ptr_jj] += Vji;
-                        }
-                     }
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-#define LOR_KERNEL_INSTANCE(order,use_smem) \
-template void Assemble3DBatchedLOR<order,use_smem>\
-    (const Array<int> &,const Array<int> &, const Array<int> &,\
-     Mesh &, SparseMatrix &)
-
-LOR_KERNEL_INSTANCE(1,true);
-LOR_KERNEL_INSTANCE(2,true);
-LOR_KERNEL_INSTANCE(3,true);
-LOR_KERNEL_INSTANCE(4,true);
-LOR_KERNEL_INSTANCE(5,true);
-LOR_KERNEL_INSTANCE(6,false);/*
-LOR_KERNEL_INSTANCE(7,false);
-LOR_KERNEL_INSTANCE(8,false);
-LOR_KERNEL_INSTANCE(9,false);
-LOR_KERNEL_INSTANCE(10,false);
-LOR_KERNEL_INSTANCE(11,false);
-LOR_KERNEL_INSTANCE(12,false);
-LOR_KERNEL_INSTANCE(13,false);
-LOR_KERNEL_INSTANCE(14,false);
-LOR_KERNEL_INSTANCE(15,false);
-LOR_KERNEL_INSTANCE(16,false);*/
-
-} // namespace mfem

fem/lor_assembly_ni3.cpp

@@ -1,128 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-#include "../general/forall.hpp"
-
-#define MFEM_DEBUG_COLOR 227
-#include "../general/debug.hpp"
-
-#define MFEM_NVTX_COLOR MediumVioletRed
-#include "../general/nvtx.hpp"
-
-namespace mfem
-{
-
-#define M1D 8
-
-template<int D1D, int Q1D>
-void NodalInterpolation3D(const int NE,
-                          const Vector& localL, Vector& localH,
-                          const Array<double> &B)
-{
-   MFEM_NVTX;
-   dbg("D1D:%d Q1D:%d", D1D, Q1D);
-
-   static constexpr int VDIM = 3;
-
-   const auto x_ = Reshape(localL.Read(), D1D, D1D, D1D, VDIM, NE);
-   const auto B_ = Reshape(B.Read(), Q1D, D1D);
-
-   auto y_ = Reshape(localH.Write(), VDIM, Q1D, Q1D, Q1D, NE);
-
-   {
-      NVTX("localH = 0.0");
-      localH = 0.0;
-   }
-
-   {
-      NVTX("NodalInterpolation3D Kernel");
-      MFEM_FORALL(e, NE,
-      {
-         for (int vd = 0; vd < VDIM; ++vd)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               double sol_xy[M1D][M1D];
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     sol_xy[qy][qx] = 0.0;
-                  }
-               }
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  double sol_x[M1D];
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     sol_x[qx] = 0;
-                  }
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     const double s = x_(dx, dy, dz, vd, e);
-                     for (int qx = 0; qx < Q1D; ++qx)
-                     {
-                        sol_x[qx] += B_(qx, dx) * s;
-                     }
-                  }
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     const double wy = B_(qy, dy);
-                     for (int qx = 0; qx < Q1D; ++qx)
-                     {
-                        sol_xy[qy][qx] += wy * sol_x[qx];
-                     }
-                  }
-               }
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  const double wz = B_(qz, dz);
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     for (int qx = 0; qx < Q1D; ++qx)
-                     {
-                        y_(vd, qx, qy, qz, e) += wz * sol_xy[qy][qx];
-                     }
-                  }
-               }
-            }
-         }
-      });
-   }
-}
-
-#define NODAL_INTERP_3D_INSTANCE(D1D,Q1D) \
-template void NodalInterpolation3D<D1D,Q1D>\
-    (const int, const Vector&, Vector&,const Array<double>&)
-
-NODAL_INTERP_3D_INSTANCE(2,2);
-NODAL_INTERP_3D_INSTANCE(2,3);
-NODAL_INTERP_3D_INSTANCE(2,4);
-NODAL_INTERP_3D_INSTANCE(2,5);
-NODAL_INTERP_3D_INSTANCE(2,6);
-NODAL_INTERP_3D_INSTANCE(2,7);
-
-NODAL_INTERP_3D_INSTANCE(4,2);
-NODAL_INTERP_3D_INSTANCE(4,3);
-NODAL_INTERP_3D_INSTANCE(4,4);
-NODAL_INTERP_3D_INSTANCE(4,5);
-NODAL_INTERP_3D_INSTANCE(4,6);
-NODAL_INTERP_3D_INSTANCE(4,7);
-
-NODAL_INTERP_3D_INSTANCE(6,2);
-NODAL_INTERP_3D_INSTANCE(6,3);
-NODAL_INTERP_3D_INSTANCE(6,4);
-NODAL_INTERP_3D_INSTANCE(6,5);
-NODAL_INTERP_3D_INSTANCE(6,6);
-NODAL_INTERP_3D_INSTANCE(6,7);
-
-} // namespace mfem

fem/nonlininteg.hpp

@@ -21,14 +21,6 @@
 namespace mfem
 {
 
-// E or L I/O vectors
-enum class ActionType
-{
-   E2E,   ///< E => E kernels
-   L2E,   ///< L => E kernels
-   L2L    ///< L => L kernels
-};
-
 /** @brief This class is used to express the local action of a general nonlinear
     finite element operator. In addition it may provide the capability to
     assemble the local gradient operator and to compute the local energy. */
@@ -40,9 +32,7 @@ protected:
    // CEED extension
    ceed::Operator* ceedOp;
 
-   MemoryType memory_type = MemoryType::DEFAULT;
-
-   ActionType action_type = ActionType::E2E;
+   MemoryType pa_mt = MemoryType::DEFAULT;
 
    NonlinearFormIntegrator(const IntegrationRule *ir = NULL)
       : IntRule(ir), ceedOp(NULL) { }
@@ -56,11 +46,8 @@ public:
    void SetIntegrationRule(const IntegrationRule &ir) { SetIntRule(&ir); }
 
    /// Set the memory type used for GeometricFactors and other large allocations
-   void SetMemoryType(MemoryType mt) { memory_type = mt; }
-   MFEM_DEPRECATED void SetPAMemoryType(MemoryType mt) { memory_type = mt; }
-
-   /// Set the kernel type used in PA extensions.
-   ActionType GetActionType() const { return action_type; }
+   /// in PA extensions.
+   void SetPAMemoryType(MemoryType mt) { pa_mt = mt; }
 
    /// Get the integration rule of the integrator (possibly NULL).
    const IntegrationRule *GetIntegrationRule() const { return IntRule; }

fem/pfespace.cpp

@@ -23,6 +23,8 @@
 #include <limits>
 #include <list>
 
+using namespace std;
+
 namespace mfem
 {
 
@@ -1010,9 +1012,36 @@ void ParFiniteElementSpace::GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
 {
    FiniteElementSpace::GetEssentialVDofs(bdr_attr_is_ess, ess_dofs, component);
 
-   // Make sure that processors without boundary elements mark
-   // their boundary dofs (if they have any).
-   Synchronize(ess_dofs);
+   if (Conforming())
+   {
+      // Make sure that processors without boundary elements mark
+      // their boundary dofs (if they have any).
+      Synchronize(ess_dofs);
+   }
+}
+
+void ParFiniteElementSpace::GetEssentialVDofs(EntitySets::EntityType type,
+                                              int set_index,
+                                              Array<int> &ess_dofs,
+                                              int component) const
+{
+   FiniteElementSpace::GetEssentialVDofs(type, set_index, ess_dofs, component);
+
+   if (Conforming())
+   {
+      // Make sure that processors without boundary elements mark
+      // their boundary dofs (if they have any).
+      Synchronize(ess_dofs);
+   }
+}
+
+void ParFiniteElementSpace::GetEssentialVDofs(EntitySets::EntityType type,
+                                              const string & set_name,
+                                              Array<int> &ess_vdofs,
+                                              int component) const
+{
+   GetEssentialVDofs(type, pmesh->ent_sets->GetSetIndex(type, set_name),
+                     ess_vdofs, component);
 }
 
 void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
@@ -1038,13 +1067,33 @@ void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
    {
       if (bool(ted[i]) != bool(true_ess_dofs2[i])) { counter++; }
    }
-   MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter
-               << ", rank = " << MyRank);
+   MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter);
 #endif
 
    MarkerToList(true_ess_dofs, ess_tdof_list);
 }
 
+void ParFiniteElementSpace::GetEssentialTrueDofs(EntitySets::EntityType type,
+                                                 int set_index,
+                                                 Array<int> &ess_tdof_list,
+                                                 int component)
+{
+   Array<int> ess_dofs, true_ess_dofs;
+
+   GetEssentialVDofs(type, set_index, ess_dofs, component);
+   GetRestrictionMatrix()->BooleanMult(ess_dofs, true_ess_dofs);
+   MarkerToList(true_ess_dofs, ess_tdof_list);
+}
+
+void ParFiniteElementSpace::GetEssentialTrueDofs(EntitySets::EntityType type,
+                                                 const string & set_name,
+                                                 Array<int> &ess_tdof_list,
+                                                 int component)
+{
+   GetEssentialTrueDofs(type, pmesh->ent_sets->GetSetIndex(type, set_name),
+                        ess_tdof_list, component);
+}
+
 int ParFiniteElementSpace::GetLocalTDofNumber(int ldof) const
 {
    if (Nonconforming())

fem/pfespace.hpp

@@ -355,12 +355,38 @@ public:
                                   Array<int> &ess_dofs,
                                   int component = -1) const;
 
+   /** Mark degrees of freedom associated with the entity set with the
+       specified entity type and set index. */
+   virtual void GetEssentialVDofs(EntitySets::EntityType type, int set_index,
+                                  Array<int> &ess_vdofs,
+                                  int component = -1) const;
+
+   /** Mark degrees of freedom associated with the entity set with the
+       specified entity type and set index. */
+   virtual void GetEssentialVDofs(EntitySets::EntityType type,
+                                  const std::string & set_name,
+                                  Array<int> &ess_vdofs,
+                                  int component = -1) const;
+
    /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
        boundary attributes marked in the array bdr_attr_is_ess. */
    virtual void GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
                                      Array<int> &ess_tdof_list,
                                      int component = -1);
 
+   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
+       entity set specified by the given entity type and set index. */
+   virtual void GetEssentialTrueDofs(EntitySets::EntityType type, int set_index,
+                                     Array<int> &ess_tdof_list,
+                                     int component = -1);
+
+   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
+       entity set specified by the given entity type and set name. */
+   virtual void GetEssentialTrueDofs(EntitySets::EntityType type,
+                                     const std::string & set_name,
+                                     Array<int> &ess_tdof_list,
+                                     int component = -1);
+
    /** If the given ldof is owned by the current processor, return its local
        tdof number, otherwise return -1 */
    int GetLocalTDofNumber(int ldof) const;

fem/qinterp/eval.hpp

@@ -65,13 +65,13 @@ static void Values2D(const int NE,
       MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
       MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
       ConstDeviceMatrix B(sB, D1D,Q1D);
       DeviceMatrix DD(sm0[tidz], MD1, MD1);
       DeviceMatrix DQ(sm1[tidz], MD1, MQ1);
       DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
       for (int c = 0; c < VDIM; c++)
       {
          kernels::internal::LoadX(e,D1D,c,x,DD);
@@ -126,14 +126,14 @@ static void Values3D(const int NE,
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
       MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
       ConstDeviceMatrix B(sB, D1D,Q1D);
       DeviceCube DDD(sm0, MD1,MD1,MD1);
       DeviceCube DDQ(sm1, MD1,MD1,MQ1);
       DeviceCube DQQ(sm0, MD1,MQ1,MQ1);
       DeviceCube QQQ(sm1, MQ1,MQ1,MQ1);
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
       for (int c = 0; c < VDIM; c++)
       {
          kernels::internal::LoadX(e,D1D,c,x,DDD);

fem/restriction.hpp

@@ -74,23 +74,6 @@ public:
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
        pattern given by this ElementRestriction, and the values of ea_data. */
    void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
-
-   /**
-    * @brief GatherMap
-    * @return the mapping from L dofs to E dofs.
-    */
-   const Array<int> &GatherMap() const { return gatherMap; }
-
-   /**
-    * @brief Indices
-    * @return the local to global indices.
-    */
-   const Array<int> &Indices() const { return indices; }
-   /**
-    * @brief Offsets
-    * @return the offsets for each global dof.
-    */
-   const Array<int> &Offsets() const { return offsets; }
 };
 
 /// Operator that converts L2 FiniteElementSpace L-vectors to E-vectors.

fem/tbilininteg.hpp

@@ -384,9 +384,9 @@ struct TDiffusionKernel<2,2,complex_t>
          const complex_t w_det_J = Q.get(q,i,k) / (J11 * J22 - J21 * J12);
          internal::MatrixOps<2,2>::Symm<Symm>::Set(
             A.layout.ind1(i), A,
-            w_det_J * (J12*J12 + J22*J22), // (1,1)
+            + w_det_J * (J12*J12 + J22*J22), // (1,1)
             - w_det_J * (J11*J12 + J21*J22), // (2,1)
-            w_det_J * (J11*J11 + J21*J21)  // (2,2)
+            + w_det_J * (J11*J11 + J21*J21)  // (2,2)
          );
       }
    }

fem/tintrules.hpp

@@ -19,8 +19,6 @@
 namespace mfem
 {
 
-static IntegrationRules LobattoIntRules(0, Quadrature1D::GaussLobatto);
-
 // Templated integration rules, cf. intrules.?pp
 
 template <Geometry::Type G, int Q, int Order, typename real_t>
@@ -54,7 +52,7 @@ public:
 
    static const IntegrationRule &GetIntRule()
    {
-      return LobattoIntRules.Get(geom, order);
+      return IntRules.Get(geom, order);
    }
 
    // Multi-component weight assignment. qpt_layout_t must be (qpts x n1 x ...)
@@ -203,10 +201,10 @@ public:
 
 template <int Dim, int Q, typename real_t>
 class GaussIntegrationRule
-   : public TProductIntegrationRule<Dim, Q, 2*Q-3, real_t>
+   : public TProductIntegrationRule<Dim, Q, 2*Q-1, real_t>
 {
 public:
-   typedef TProductIntegrationRule<Dim,Q,2*Q-3,real_t> base_class;
+   typedef TProductIntegrationRule<Dim,Q,2*Q-1,real_t> base_class;
 
    using base_class::geom;
    using base_class::order;
@@ -228,11 +226,11 @@ public:
 
    static const IntegrationRule &Get1DIntRule()
    {
-      return LobattoIntRules.Get(Geometry::SEGMENT, order);
+      return IntRules.Get(Geometry::SEGMENT, order);
    }
    static const IntegrationRule &GetIntRule()
    {
-      return LobattoIntRules.Get(geom, order);
+      return IntRules.Get(geom, order);
    }
 };
 
@@ -241,15 +239,15 @@ class TIntegrationRule;
 
 template <int Order, typename real_t>
 class TIntegrationRule<Geometry::SEGMENT, Order, real_t>
-   : public GaussIntegrationRule<1, Order/2+2, real_t> { };
+   : public GaussIntegrationRule<1, Order/2+1, real_t> { };
 
 template <int Order, typename real_t>
 class TIntegrationRule<Geometry::SQUARE, Order, real_t>
-   : public GaussIntegrationRule<2, Order/2+2, real_t> { };
+   : public GaussIntegrationRule<2, Order/2+1, real_t> { };
 
 template <int Order, typename real_t>
 class TIntegrationRule<Geometry::CUBE, Order, real_t>
-   : public GaussIntegrationRule<3, Order/2+2, real_t> { };
+   : public GaussIntegrationRule<3, Order/2+1, real_t> { };
 
 // Triangle integration rules (based on intrules.cpp)
 // These specializations define the number of quadrature points for each rule as

fem/tmop.hpp

@@ -1133,7 +1133,7 @@ protected:
 
    // Evaluation of the discrete target specification on different meshes.
    // Owned.
-   AdaptivityEvaluator *adapt_lim_eval;
+   AdaptivityEvaluator *adapt_eval;
 
    void SetDiscreteTargetBase(const GridFunction &tspec_);
    void SetTspecAtIndex(int idx, const GridFunction &tspec_);
@@ -1156,7 +1156,7 @@ public:
 #endif
         amr_el(-1), lim_min_size(-0.1),
         good_tspec(false), good_tspec_grad(false), good_tspec_hess(false),
-        adapt_lim_eval(NULL) { }
+        adapt_eval(NULL) { }
 
    virtual ~DiscreteAdaptTC();
 
@@ -1232,8 +1232,8 @@ public:
 
    void SetAdaptivityEvaluator(AdaptivityEvaluator *ae)
    {
-      if (adapt_lim_eval) { delete adapt_lim_eval; }
-      adapt_lim_eval = ae;
+      if (adapt_eval) { delete adapt_eval; }
+      adapt_eval = ae;
    }
 
    const Vector &GetTspecPert1H()   { return tspec_pert1h; }
@@ -1315,15 +1315,15 @@ protected:
    int integ_order;
 
    // Weight Coefficient multiplying the quality metric term.
-   Coefficient *metric_coeff; // not owned, if NULL -> metric_coeff is 1.
+   Coefficient *coeff1; // not owned, if NULL -> coeff1 is 1.
    // Normalization factor for the metric term.
    double metric_normal;
 
    // Nodes and weight Coefficient used for "limiting" the TMOP_Integrator.
    // These are both NULL when there is no limiting.
-   // The class doesn't own lim_nodes0 and lim_coeff.
-   const GridFunction *lim_nodes0;
-   Coefficient *lim_coeff;
+   // The class doesn't own nodes0 and coeff0.
+   const GridFunction *nodes0;
+   Coefficient *coeff0;
    // Limiting reference distance. Not owned.
    const GridFunction *lim_dist;
    // Limiting function. Owned.
@@ -1332,21 +1332,20 @@ protected:
    double lim_normal;
 
    // Adaptive limiting.
-   const GridFunction *adapt_lim_gf0;    // Not owned.
+   const GridFunction *zeta_0;       // Not owned.
 #ifdef MFEM_USE_MPI
-   const ParGridFunction *adapt_lim_pgf0;
+   const ParGridFunction *pzeta_0;
 #endif
-   GridFunction *adapt_lim_gf;           // Owned. Updated by adapt_lim_eval.
-   Coefficient *adapt_lim_coeff;         // Not owned.
-   AdaptivityEvaluator *adapt_lim_eval;  // Not owned.
+   GridFunction *zeta;               // Owned. Updated by adapt_eval.
+   Coefficient *coeff_zeta;          // Not owned.
+   AdaptivityEvaluator *adapt_eval;  // Not owned.
 
    // Surface fitting.
-   GridFunction *surf_fit_gf,
-                *surf_fit_gf_bar;       // Owned, Updated by surf_fit_eval.
-   const Array<bool> *surf_fit_marker;  // Not owned.
-   Coefficient *surf_fit_coeff;         // Not owned.
-   AdaptivityEvaluator *surf_fit_eval;  // Not owned.
-   double surf_fit_normal;
+   GridFunction *sigma, *sigma_bar;  // Owned. Updated by sigma_eval.
+   const Array<bool> *sigma_marker;  // Not owned.
+   Coefficient *coeff_sigma;         // Not owned.
+   AdaptivityEvaluator *sigma_eval;  // Not owned.
+   double sigma_normal;
 
    DiscreteAdaptTC *discr_tc;
 
@@ -1417,7 +1416,7 @@ protected:
 
    void ComputeNormalizationEnergies(const GridFunction &x,
                                      double &metric_energy, double &lim_energy,
-                                     double &surf_fit_gf_energy);
+                                     double &sigma_energy);
 
    void AssembleElementVectorExact(const FiniteElement &el,
                                    ElementTransformation &T,
@@ -1472,7 +1471,7 @@ protected:
 
    void DisableLimiting()
    {
-      lim_nodes0 = NULL; lim_coeff = NULL; lim_dist = NULL;
+      nodes0 = NULL; coeff0 = NULL; lim_dist = NULL;
       delete lim_func; lim_func = NULL;
    }
 
@@ -1532,14 +1531,12 @@ public:
    TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc,
                    TMOP_QualityMetric *hm)
       : h_metric(hm), metric(m), targetC(tc), IntegRules(NULL),
-        integ_order(-1), metric_coeff(NULL), metric_normal(1.0),
-        lim_nodes0(NULL), lim_coeff(NULL),
+        integ_order(-1), coeff1(NULL), metric_normal(1.0),
+        nodes0(NULL), coeff0(NULL),
         lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
-        adapt_lim_gf0(NULL), adapt_lim_gf(NULL), adapt_lim_coeff(NULL),
-        adapt_lim_eval(NULL),
-        surf_fit_gf(NULL), surf_fit_gf_bar(NULL), surf_fit_marker(NULL),
-        surf_fit_coeff(NULL),
-        surf_fit_eval(NULL), surf_fit_normal(1.0),
+        zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
+        sigma(NULL), sigma_bar(NULL), sigma_marker(NULL), coeff_sigma(NULL),
+        sigma_eval(NULL), sigma_normal(1.0),
         discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
    { PA.enabled = false; }
@@ -1567,7 +1564,7 @@ public:
 
        Note that the Coefficient is evaluated in the physical configuration and
        not in the target configuration which may be undefined. */
-   void SetCoefficient(Coefficient &w1) { metric_coeff = &w1; }
+   void SetCoefficient(Coefficient &w1) { coeff1 = &w1; }
 
    /** @brief Limiting of the mesh displacements (general version).
 
@@ -1634,7 +1631,7 @@ public:
    void GetSurfaceFittingErrors(double &err_avg, double &err_max);
 
    /// Update the original/reference nodes used for limiting.
-   void SetLimitingNodes(const GridFunction &n0) { lim_nodes0 = &n0; }
+   void SetLimitingNodes(const GridFunction &n0) { nodes0 = &n0; }
 
    /** @brief Computes the integral of W(Jacobian(Trt)) over a target zone.
        @param[in] el     Type of FiniteElement.

fem/tmop/tmop_pa.cpp

@@ -38,23 +38,23 @@ void TMOP_Integrator::AssembleGradPA(const Vector &xe,
    if (PA.dim == 2)
    {
       AssembleGradPA_2D(xe);
-      if (lim_coeff) { AssembleGradPA_C0_2D(xe); }
+      if (coeff0) { AssembleGradPA_C0_2D(xe); }
    }
 
    if (PA.dim == 3)
    {
       AssembleGradPA_3D(xe);
-      if (lim_coeff) { AssembleGradPA_C0_3D(xe); }
+      if (coeff0) { AssembleGradPA_C0_3D(xe); }
    }
 }
 
 void TMOP_Integrator::AssemblePA_Limiting()
 {
-   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : memory_type;
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
    // Return immediately if limiting is not enabled
-   if (lim_coeff == nullptr) { return; }
-   MFEM_VERIFY(lim_nodes0, "internal error");
+   if (coeff0 == nullptr) { return; }
+   MFEM_VERIFY(nodes0, "internal error");
 
    MFEM_VERIFY(PA.enabled, "AssemblePA_Limiting but PA is not enabled!");
    MFEM_VERIFY(lim_func, "No TMOP_LimiterFunction specification!")
@@ -68,14 +68,14 @@ void TMOP_Integrator::AssemblePA_Limiting()
 
    const ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
 
-   // H0 for lim_coeff, (dim x dim) Q-vector
+   // H0 for coeff0, (dim x dim) Q-vector
    PA.H0.UseDevice(true);
    PA.H0.SetSize(PA.dim * PA.dim * PA.nq * NE, mt);
 
-   // lim_coeff -> PA.C0 (Q-vector)
+   // coeff0 -> PA.C0 (Q-vector)
    PA.C0.UseDevice(true);
    if (ConstantCoefficient* cQ =
-          dynamic_cast<ConstantCoefficient*>(lim_coeff))
+          dynamic_cast<ConstantCoefficient*>(coeff0))
    {
       PA.C0.SetSize(1, Device::GetMemoryType());
       PA.C0.HostWrite();
@@ -90,17 +90,17 @@ void TMOP_Integrator::AssemblePA_Limiting()
          ElementTransformation& T = *fes->GetElementTransformation(e);
          for (int q = 0; q < ir.GetNPoints(); ++q)
          {
-            C0(q,e) = lim_coeff->Eval(T, ir.IntPoint(q));
+            C0(q,e) = coeff0->Eval(T, ir.IntPoint(q));
          }
       }
    }
 
-   // lim_nodes0 -> PA.X0 (E-vector)
-   MFEM_VERIFY(lim_nodes0->FESpace() == fes, "");
+   // nodes0 -> PA.X0 (E-vector)
+   MFEM_VERIFY(nodes0->FESpace() == fes, "");
    const Operator *n0_R = fes->GetElementRestriction(ordering);
    PA.X0.SetSize(n0_R->Height(), Device::GetMemoryType());
    PA.X0.UseDevice(true);
-   n0_R->Mult(*lim_nodes0, PA.X0);
+   n0_R->Mult(*nodes0, PA.X0);
 
    // Limiting distances: lim_dist -> PA.LD (E-vector)
    // TODO: remove the hack for the case lim_dist == NULL.
@@ -177,8 +177,8 @@ void TMOP_Integrator::ComputeAllElementTargets(const Vector &xe) const
 
 void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : memory_type;
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
    PA.enabled = true;
    PA.fes = &fes;
    Mesh *mesh = fes.GetMesh();
@@ -217,8 +217,8 @@ void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
    PA.Jtr_needs_update = true;
    PA.Jtr_debug_grad = false;
 
-   // Limiting: lim_coeff -> PA.C0, lim_nodes0 -> PA.X0, lim_dist -> PA.LD, PA.H0
-   if (lim_coeff) { AssemblePA_Limiting(); }
+   // Limiting: coeff0 -> PA.C0, nodes0 -> PA.X0, lim_dist -> PA.LD, PA.H0
+   if (coeff0) { AssemblePA_Limiting(); }
 }
 
 void TMOP_Integrator::AssembleGradDiagonalPA(Vector &de) const
@@ -236,13 +236,13 @@ void TMOP_Integrator::AssembleGradDiagonalPA(Vector &de) const
    if (PA.dim == 2)
    {
       AssembleDiagonalPA_2D(de);
-      if (lim_coeff) { AssembleDiagonalPA_C0_2D(de); }
+      if (coeff0) { AssembleDiagonalPA_C0_2D(de); }
    }
 
    if (PA.dim == 3)
    {
       AssembleDiagonalPA_3D(de);
-      if (lim_coeff) { AssembleDiagonalPA_C0_3D(de); }
+      if (coeff0) { AssembleDiagonalPA_C0_3D(de); }
    }
 }
 
@@ -258,13 +258,13 @@ void TMOP_Integrator::AddMultPA(const Vector &xe, Vector &ye) const
    if (PA.dim == 2)
    {
       AddMultPA_2D(xe,ye);
-      if (lim_coeff) { AddMultPA_C0_2D(xe,ye); }
+      if (coeff0) { AddMultPA_C0_2D(xe,ye); }
    }
 
    if (PA.dim == 3)
    {
       AddMultPA_3D(xe,ye);
-      if (lim_coeff) { AddMultPA_C0_3D(xe,ye); }
+      if (coeff0) { AddMultPA_C0_3D(xe,ye); }
    }
 }
 
@@ -283,13 +283,13 @@ void TMOP_Integrator::AddMultGradPA(const Vector &re, Vector &ce) const
    if (PA.dim == 2)
    {
       AddMultGradPA_2D(re,ce);
-      if (lim_coeff) { AddMultGradPA_C0_2D(re,ce); }
+      if (coeff0) { AddMultGradPA_C0_2D(re,ce); }
    }
 
    if (PA.dim == 3)
    {
       AddMultGradPA_3D(re,ce);
-      if (lim_coeff) { AddMultGradPA_C0_3D(re,ce); }
+      if (coeff0) { AddMultGradPA_C0_3D(re,ce); }
    }
 }
 
@@ -307,13 +307,13 @@ double TMOP_Integrator::GetLocalStateEnergyPA(const Vector &xe) const
    if (PA.dim == 2)
    {
       energy = GetLocalStateEnergyPA_2D(xe);
-      if (lim_coeff) { energy += GetLocalStateEnergyPA_C0_2D(xe); }
+      if (coeff0) { energy += GetLocalStateEnergyPA_C0_2D(xe); }
    }
 
    if (PA.dim == 3)
    {
       energy = GetLocalStateEnergyPA_3D(xe);
-      if (lim_coeff) { energy += GetLocalStateEnergyPA_C0_3D(xe); }
+      if (coeff0) { energy += GetLocalStateEnergyPA_C0_3D(xe); }
    }
 
    return energy;

fem/tmop/tmop_pa.hpp

@@ -181,9 +181,9 @@ template<int T_D1D, int T_Q1D, int T_MAX> return_t kernel(__VA_ARGS__)
 if (K##kernel.Find(id)) { return K##kernel.At(id)(__VA_ARGS__,0,0); }\
 else {\
    constexpr int T_MAX = 4;\
-   const int d1d = (id>>4)&0xF, q1d = id&0xF;\
-   MFEM_VERIFY(d1d <= MAX_D1D && q1d <= MAX_Q1D, "Max size error!");\
-   return kernel<0,0,T_MAX>(__VA_ARGS__,d1d,q1d); }
+   const int D1D = (id>>4)&0xF, Q1D = id&0xF;\
+   MFEM_VERIFY(D1D <= MAX_D1D && Q1D <= MAX_Q1D, "Max size error!");\
+   return kernel<0,0,T_MAX>(__VA_ARGS__,D1D,Q1D); }
 
 } // namespace kernels
 

fem/tmop/tmop_pa_da3.cpp

@@ -55,8 +55,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, DatcSize,
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
       MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
 
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
       ConstDeviceMatrix B(sB, D1D, Q1D);
       DeviceCube DDD(sm0, MD1,MD1,MD1);
       DeviceCube DDQ(sm1, MD1,MD1,MQ1);
@@ -90,6 +88,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, DatcSize,
       }
       min = min_size[0];
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
       kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
       kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);

fem/tmop/tmop_pa_h2d.cpp

@@ -97,9 +97,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AssembleDiagonalPA_Kernel_2D,
                   const double *Jtr = &J(0,0,qx,qy,e);
 
                   // Jrt = Jtr^{-1}
-                  double jrt_data[4];
-                  ConstDeviceMatrix Jrt(jrt_data,2,2);
-                  kernels::CalcInverse<2>(Jtr, jrt_data);
+                  double j[4];
+                  ConstDeviceMatrix Jrt(j,2,2);
+                  kernels::CalcInverse<2>(Jtr, j);
 
                   const double gg = G(qy,dy) * G(qy,dy);
                   const double gb = G(qy,dy) * B(qy,dy);

fem/tmop/tmop_pa_h2m_c0.cpp

@@ -68,8 +68,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_2D,
             double Xh[2];
             kernels::internal::PullEval<MQ1,NBZ>(Q1D,qx,qy,QQ,Xh);
 
-            double H_data[4];
-            DeviceMatrix H(H_data,2,2);
+            double B[4];
+            DeviceMatrix H(B,2,2);
             for (int i = 0; i < DIM; i++)
             {
                for (int j = 0; j < DIM; j++)
@@ -78,9 +78,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_2D,
                }
             }
 
-            // p2 = H . Xh
+            // p2 = B . Xh
             double p2[2];
-            kernels::Mult(2,2,H_data,Xh,p2);
+            kernels::Mult(2,2,B,Xh,p2);
             kernels::internal::PushEval<MQ1,NBZ>(Q1D,qx,qy,p2,QQ);
          }
       }

fem/tmop/tmop_pa_h3m_c0.cpp

@@ -70,8 +70,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_3D,
                double Xh[3];
                kernels::internal::PullEval<MQ1>(Q1D,qx,qy,qz,QQQ,Xh);
 
-               double H_data[9];
-               DeviceMatrix H(H_data,3,3);
+               double B[9];
+               DeviceMatrix H(B,3,3);
                for (int i = 0; i < DIM; i++)
                {
                   for (int j = 0; j < DIM; j++)
@@ -80,9 +80,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultGradPA_Kernel_C0_3D,
                   }
                }
 
-               // p2 = H . Xh
+               // p2 = B . Xh
                double p2[3];
-               kernels::Mult(3,3,H_data,Xh,p2);
+               kernels::Mult(3,3,B,Xh,p2);
                kernels::internal::PushEval<MQ1>(Q1D,qx,qy,qz,p2,QQQ);
             }
          }

fem/tmop/tmop_pa_h3s_c0.cpp

@@ -55,7 +55,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_Kernel_C0_3D,
       constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
 
       MFEM_SHARED double sBLD[MQ1*MD1];
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -67,6 +66,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_Kernel_C0_3D,
 
       kernels::internal::LoadX(e,D1D,LD,DDD);
 
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
+
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);
       kernels::internal::EvalZ(D1D,Q1D,BLD,DQQ,QQQ);

fem/tmop/tmop_pa_p3_c0.cpp

@@ -62,7 +62,6 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_3D,
 
       MFEM_SHARED double B[MQ1*MD1];
       MFEM_SHARED double sBLD[MQ1*MD1];
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -87,6 +86,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_C0_3D,
       kernels::internal::LoadX<MD1>(e,D1D,X1,DDD1);
 
       kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
 
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);

fem/tmop/tmop_pa_w3_c0.cpp

@@ -63,7 +63,6 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_3D,
 
       MFEM_SHARED double B[MQ1*MD1];
       MFEM_SHARED double sBLD[MQ1*MD1];
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
       ConstDeviceMatrix BLD(sBLD, D1D, Q1D);
 
       MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
@@ -88,6 +87,7 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_C0_3D,
       kernels::internal::LoadX<MD1>(e,D1D,X1,DDD1);
 
       kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,B);
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,bld,sBLD);
 
       kernels::internal::EvalX(D1D,Q1D,BLD,DDD,DDQ);
       kernels::internal::EvalY(D1D,Q1D,BLD,DDQ,DQQ);

fem/tmop_amr.cpp

@@ -837,26 +837,26 @@ void TMOPHRSolver::ParUpdate()
 }
 #endif
 
-void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh_, NonlinearForm *nlf_)
+void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
 {
-   const FiniteElementSpace &fes = *mesh_->GetNodalFESpace();
+   const FiniteElementSpace &fes = *mesh->GetNodalFESpace();
 
    // Update Nonlinear form and Set Essential BC
-   nlf_->Update();
+   nlf->Update();
    const int dim = fes.GetFE(0)->GetDim();
    if (move_bnd == false)
    {
-      Array<int> ess_bdr(mesh_->bdr_attributes.Max());
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
       ess_bdr = 1;
-      nlf_->SetEssentialBC(ess_bdr);
+      nlf->SetEssentialBC(ess_bdr);
    }
    else
    {
       const int nd  = fes.GetBE(0)->GetDof();
       int n = 0;
-      for (int i = 0; i < mesh_->GetNBE(); i++)
+      for (int i = 0; i < mesh->GetNBE(); i++)
       {
-         const int attr = mesh_->GetBdrElement(i)->GetAttribute();
+         const int attr = mesh->GetBdrElement(i)->GetAttribute();
          MFEM_VERIFY(!(dim == 2 && attr == 3),
                      "Boundary attribute 3 must be used only for 3D meshes. "
                      "Adjust the attributes (1/2/3/4 for fixed x/y/z/all "
@@ -866,9 +866,9 @@ void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh_, NonlinearForm *nlf_)
       }
       Array<int> ess_vdofs(n), vdofs;
       n = 0;
-      for (int i = 0; i < mesh_->GetNBE(); i++)
+      for (int i = 0; i < mesh->GetNBE(); i++)
       {
-         const int attr = mesh_->GetBdrElement(i)->GetAttribute();
+         const int attr = mesh->GetBdrElement(i)->GetAttribute();
          fes.GetBdrElementVDofs(i, vdofs);
          if (attr == 1) // Fix x components.
          {
@@ -891,7 +891,7 @@ void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh_, NonlinearForm *nlf_)
             { ess_vdofs[n++] = vdofs[j]; }
          }
       }
-      nlf_->SetEssentialVDofs(ess_vdofs);
+      nlf->SetEssentialVDofs(ess_vdofs);
    }
 }
 

fem/tmop_tools.cpp

@@ -240,7 +240,7 @@ ParAdvectorCGOper::ParAdvectorCGOper(const Vector &x_start,
    ConvectionIntegrator *Kinteg = new ConvectionIntegrator(u_coeff);
    if (al == AssemblyLevel::PARTIAL)
    {
-      Kinteg->SetMemoryType(mt);
+      Kinteg->SetPAMemoryType(mt);
    }
    K.AddDomainIntegrator(Kinteg);
    K.SetAssemblyLevel(al);
@@ -250,7 +250,7 @@ ParAdvectorCGOper::ParAdvectorCGOper(const Vector &x_start,
    MassIntegrator *Minteg = new MassIntegrator;
    if (al == AssemblyLevel::PARTIAL)
    {
-      Minteg->SetMemoryType(mt);
+      Minteg->SetPAMemoryType(mt);
    }
    M.AddDomainIntegrator(Minteg);
    M.SetAssemblyLevel(al);
@@ -444,19 +444,15 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (untangling == false && min_detT_out < 0.0)
       {
          // No untangling, and detJ got negative -- no good.
-         if (print_options.iterations)
-         {
-            mfem::out << "Scale = " << scale << " Neg det(J) found.\n";
-         }
+         if (print_level >= 0)
+         { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
          scale *= detJ_factor; continue;
       }
       if (untangling == true && min_detT_out < *min_det_ptr)
       {
          // Untangling, and detJ got even more negative -- no good.
-         if (print_options.iterations)
-         {
-            mfem::out << "Scale = " << scale << " Neg det(J) decreased.\n";
-         }
+         if (print_level >= 0)
+         { mfem::out << "Scale = " << scale << " Neg det(J) decreased.\n"; }
          scale *= detJ_factor; continue;
       }
 
@@ -481,7 +477,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (energy_out > energy_in + 0.2*fabs(energy_in) ||
           std::isnan(energy_out) != 0)
       {
-         if (print_options.iterations)
+         if (print_level >= 0)
          {
             mfem::out << "Scale = " << scale << " Increasing energy: "
                       << energy_in << " --> " << energy_out << '\n';
@@ -496,7 +492,7 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
       if (norm_out > 1.2*norm_in)
       {
-         if (print_options.iterations)
+         if (print_level >= 0)
          {
             mfem::out << "Scale = " << scale << " Norm increased: "
                       << norm_in << " --> " << norm_out << '\n';
@@ -512,15 +508,13 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       if (min_detT_out > 0.0)
       {
          *min_det_ptr = 0.0;
-         if (print_options.summary || print_options.iterations ||
-             print_options.first_and_last)
+         if (print_level >= 0)
          { mfem::out << "The mesh has been untangled at the used points!\n"; }
       }
       else { *min_det_ptr = untangle_factor * min_detT_out; }
    }
 
-   if (print_options.summary || print_options.iterations ||
-       print_options.first_and_last)
+   if (print_level >= 0)
    {
       if (untangling)
       {
@@ -574,9 +568,9 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
    if (parallel)
    {
 #ifdef MFEM_USE_MPI
-      const ParNonlinearForm *pnlf =
+      const ParNonlinearForm *nlf =
          dynamic_cast<const ParNonlinearForm *>(oper);
-      const ParFiniteElementSpace *pfesc = pnlf->ParFESpace();
+      const ParFiniteElementSpace *pfesc = nlf->ParFESpace();
       Vector x_loc(pfesc->GetVSize());
       pfesc->GetProlongationMatrix()->Mult(x, x_loc);
       for (int i = 0; i < integs.Size(); i++)

fem/transfer.cpp

@@ -1192,114 +1192,6 @@ void Prolongation2D(const int NE, const int D1D, const int Q1D,
    });
 }
 
-template <int D1D, int Q1D, int NBZ = 1>
-void FastProlongation3D(const int lND,
-                        const int hNQ,
-                        const int NE,
-                        const int *lmap,
-                        const Vector& x,
-                        const int *hmap,
-                        Vector& y,
-                        const Array<double>& B,
-                        const Vector& mask)
-{
-   const auto B_ = Reshape(B.Read(), Q1D, D1D);
-   const auto lMAP = Reshape(lmap, D1D,D1D,D1D, NE);
-   const auto hMAP = Reshape(hmap, Q1D,Q1D,Q1D, NE);
-   const auto X = Reshape(x.Read(), lND);
-   const auto M = Reshape(mask.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), hNQ);
-
-   y = 0.0;
-
-   assert(Q1D<=8);
-
-   MFEM_FORALL_3D(be, (NE+NBZ-1)/NBZ, Q1D, Q1D, NBZ,
-   {
-      double u[Q1D];
-      const int tz = MFEM_THREAD_ID(z);
-      const int e = be * MFEM_THREAD_SIZE(z) + tz;
-
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
-
-      // Load input, B & X interpolation
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            if (tz == 0) { s_B[qx][dy] = B_(qx,dy); }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double Bx = B_(qx,dx);
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const int gid = lMAP(dx,dy,dz,e);
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  u[dz] += X(idx)* Bx;
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Y interpolation
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double zyX = s_q[tz][dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Z interpolation & Q-function
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            // Z interpolation
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const double zYX = s_q[tz][dz][qy][qx];
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
-            }
-
-            // Q-function
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const int gid = hMAP(qx,qy,qz,e);
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               const double value = M(qx, qy, qz, e) * u[qz];
-               AtomicAdd(Y[idx], value);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
 void Prolongation3D(const int NE, const int D1D, const int Q1D,
                     const Vector& localL, Vector& localH,
                     const Array<double>& B, const Vector& mask)
@@ -1411,7 +1303,6 @@ void Restriction2D(const int NE, const int D1D, const int Q1D,
       }
    });
 }
-
 void Restriction3D(const int NE, const int D1D, const int Q1D,
                    const Vector& localH, Vector& localL,
                    const Array<double>& Bt, const Vector& mask)
@@ -1473,129 +1364,6 @@ void Restriction3D(const int NE, const int D1D, const int Q1D,
       }
    });
 }
-
-template<int D1D, int Q1D, int NBZ = 1>
-void FastRestriction3D(const int lND,
-                       const int hNQ,
-                       const int NE,
-                       const int *hmap,
-                       const Vector& x,
-                       const int *lmap,
-                       Vector& y,
-                       const Array<double>& Bt,
-                       const Vector& mask)
-{
-   const auto Bt_ = Reshape(Bt.Read(), D1D, Q1D);
-   const auto lMAP = Reshape(lmap, D1D,D1D,D1D, NE);
-   const auto hMAP = Reshape(hmap, Q1D,Q1D,Q1D, NE);
-   const auto M = Reshape(mask.Read(), Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x.Read(), hNQ);
-   auto Y = Reshape(y.ReadWrite(), lND);
-
-   y = 0.0;
-
-   assert(Q1D<=8);
-
-   MFEM_FORALL_3D(be, (NE+NBZ-1)/NBZ, Q1D, Q1D, NBZ,
-   {
-      double u[Q1D];
-      const int tz = MFEM_THREAD_ID(z);
-      const int e = be * MFEM_THREAD_SIZE(z) + tz;
-
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
-
-      // Load B
-      if (tz == 0)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               { s_B[qx][dy] = Bt_(dy,qx); }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Q-function & Zt projection
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            // Q-function
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const int gid = hMAP(qx,qy,qz,e);
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               s_q[tz][qz][qy][qx] = M(qx, qy, qz, e) * X(idx);
-            }
-
-            // Zt projection
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double ZYX = s_q[tz][qz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Yt projection
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double zYX = s_q[tz][dz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Xt projection & save output
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double zyX = s_q[tz][dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const int gid = lMAP(dx,dy,dz,e);
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               const double value = u[dx];
-               AtomicAdd(Y[idx], value);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
 } // namespace TransferKernels
 
 
@@ -1612,69 +1380,22 @@ void TensorProductPRefinementTransferOperator::Mult(const Vector& x,
       return;
    }
 
-   if (Device::FastKernelsEnabled())
+   elem_restrict_lex_l->Mult(x, localL);
+   if (dim == 2)
    {
-      assert(dim==3);
-      constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-      // lFESpace map idx
-      const int lND = lFESpace.GetNDofs();
-      const Operator *lERop = lFESpace.GetElementRestriction(ordering);
-      const ElementRestriction* lER = dynamic_cast<const ElementRestriction*>(lERop);
-      assert(lER);
-      const int *lmap = lER->GatherMap().Read();
-      assert(lER->GatherMap().Size() == D1D*D1D*D1D*NE);
-      // hFESpace map idx
-      const int hNQ = hFESpace.GetNDofs();
-      const Operator *hERop = elem_restrict_lex_h;
-      const ElementRestriction* hER = dynamic_cast<const ElementRestriction*>(hERop);
-      assert(hER);
-      const int *hmap = hER->GatherMap().Read();
-      assert(hER->GatherMap().Size() == Q1D*Q1D*Q1D*NE);
-
-      void (*Ker)(const int lND,
-                  const int hNQ,
-                  const int NE,
-                  const int *lmap,
-                  const Vector& x,
-                  const int *hmap,
-                  Vector& y,
-                  const Array<double>& B,
-                  const Vector& mask) = nullptr;
-
-      const int id = (D1D << 4) | Q1D;
-      switch (id) // orders 1~8
-      {
-         case 0x23: Ker=TransferKernels::FastProlongation3D<2,3>; break; // 1
-         case 0x34: Ker=TransferKernels::FastProlongation3D<3,4>; break; // 2
-         case 0x35: Ker=TransferKernels::FastProlongation3D<3,5>; break;
-         case 0x45: Ker=TransferKernels::FastProlongation3D<4,5>; break; // 3
-         case 0x56: Ker=TransferKernels::FastProlongation3D<5,6>; break; // 4
-         case 0x57: Ker=TransferKernels::FastProlongation3D<5,7>; break;
-         case 0x67: Ker=TransferKernels::FastProlongation3D<6,7>; break; // 5
-         case 0x78: Ker=TransferKernels::FastProlongation3D<7,8>; break; // 6
-         default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
-      }
-      Ker(lND, hNQ, NE, lmap, x, hmap, y, B, mask);
+      TransferKernels::Prolongation2D(NE, D1D, Q1D, localL, localH, B, mask);
+   }
+   else if (dim == 3)
+   {
+      TransferKernels::Prolongation3D(NE, D1D, Q1D, localL, localH, B, mask);
    }
    else
    {
-      elem_restrict_lex_l->Mult(x, localL);
-      if (dim == 2)
-      {
-         TransferKernels::Prolongation2D(NE, D1D, Q1D, localL, localH, B, mask);
-      }
-      else if (dim == 3)
-      {
-         TransferKernels::Prolongation3D(NE, D1D, Q1D, localL, localH, B, mask);
-      }
-      else
-      {
-         MFEM_ABORT("TensorProductPRefinementTransferOperator::Mult not "
-                    "implemented for dim = "
-                    << dim);
-      }
-      elem_restrict_lex_h->MultTranspose(localH, y);
+      MFEM_ABORT("TensorProductPRefinementTransferOperator::Mult not "
+                 "implemented for dim = "
+                 << dim);
    }
+   elem_restrict_lex_h->MultTranspose(localH, y);
 }
 
 void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
@@ -1685,68 +1406,22 @@ void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
       return;
    }
 
-   if (Device::FastKernelsEnabled())
+   elem_restrict_lex_h->Mult(x, localH);
+   if (dim == 2)
    {
-      assert(dim==3);
-      // lFESpace map idx
-      const int lND = lFESpace.GetNDofs();
-      constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-      const Operator *lERop = lFESpace.GetElementRestriction(ordering);
-      const ElementRestriction* lER = dynamic_cast<const ElementRestriction*>(lERop);
-      assert(lER);
-      const int *lmap = lER->GatherMap().Read();
-      // hFESpace map idx
-      const int hNQ = hFESpace.GetNDofs();
-      const Operator *hERop = elem_restrict_lex_h;
-      const ElementRestriction* hER = dynamic_cast<const ElementRestriction*>(hERop);
-      assert(hER);
-      const int *hmap = hER->GatherMap().Read();
-      assert(hER->GatherMap().Size() == Q1D*Q1D*Q1D*NE);
-
-      void (*Ker)(const int lND,
-                  const int hNQ,
-                  const int NE,
-                  const int *hmap,
-                  const Vector& x,
-                  const int *lmap,
-                  Vector& y,
-                  const Array<double>& Bt,
-                  const Vector& mask) = nullptr;
-
-      const int id = (D1D << 4) | Q1D;
-      switch (id) // orders 1~8
-      {
-         case 0x23: Ker=TransferKernels::FastRestriction3D<2,3>; break; // 1
-         case 0x34: Ker=TransferKernels::FastRestriction3D<3,4>; break; // 2
-         case 0x35: Ker=TransferKernels::FastRestriction3D<3,5>; break;
-         case 0x45: Ker=TransferKernels::FastRestriction3D<4,5>; break; // 3
-         case 0x56: Ker=TransferKernels::FastRestriction3D<5,6>; break; // 4
-         case 0x57: Ker=TransferKernels::FastRestriction3D<5,7>; break;
-         case 0x67: Ker=TransferKernels::FastRestriction3D<6,7>; break; // 5
-         case 0x78: Ker=TransferKernels::FastRestriction3D<7,8>; break; // 6
-         default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
-      }
-      Ker(lND, hNQ, NE, hmap, x, lmap, y, Bt, mask);
+      TransferKernels::Restriction2D(NE, D1D, Q1D, localH, localL, Bt, mask);
+   }
+   else if (dim == 3)
+   {
+      TransferKernels::Restriction3D(NE, D1D, Q1D, localH, localL, Bt, mask);
    }
    else
    {
-      elem_restrict_lex_h->Mult(x, localH);
-      if (dim == 2)
-      {
-         TransferKernels::Restriction2D(NE, D1D, Q1D, localH, localL, Bt, mask);
-      }
-      else if (dim == 3)
-      {
-         TransferKernels::Restriction3D(NE, D1D, Q1D, localH, localL, Bt, mask);
-      }
-      else
-      {
-         MFEM_ABORT("TensorProductPRefinementTransferOperator::MultTranspose not "
-                    "implemented for dim = "
-                    << dim);
-      }
-      elem_restrict_lex_l->MultTranspose(localL, y);
+      MFEM_ABORT("TensorProductPRefinementTransferOperator::MultTranspose not "
+                 "implemented for dim = "
+                 << dim);
    }
+   elem_restrict_lex_l->MultTranspose(localL, y);
 }
 
 #ifdef MFEM_USE_MPI

general/backends.hpp

@@ -18,7 +18,6 @@
 #include <cusparse.h>
 #include <library_types.h>
 #include <cuda_runtime.h>
-#include <curand.h>
 #include <cuda.h>
 #endif
 #include "cuda.hpp"
@@ -40,26 +39,20 @@
 #endif
 
 #if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-#define MFEM_GLOBAL
 #define MFEM_DEVICE
 #define MFEM_LAMBDA
-#define MFEM_GLOBAL
 #define MFEM_HOST_DEVICE
 // MFEM_DEVICE_SYNC is made available for debugging purposes
 #define MFEM_DEVICE_SYNC
 // MFEM_STREAM_SYNC is used for UVM and MPI GPU-Aware kernels
 #define MFEM_STREAM_SYNC
-#define MFEM_LAUNCH_BOUNDS(...)
-#define MFEM_LAUNCH_KERNEL(Kernel,Grid,Block,Smem,...) Kernel(__VA_ARGS__)
 #endif
 
 #if !((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) || \
       (defined(MFEM_USE_HIP)  && defined(__HIP_DEVICE_COMPILE__)))
 #define MFEM_SHARED
 #define MFEM_SYNC_THREAD
-#define MFEM_GRID_DIM(k) 1
 #define MFEM_BLOCK_ID(k) 0
-#define MFEM_BLOCK_DIM(k) 1
 #define MFEM_THREAD_ID(k) 0
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
@@ -98,12 +91,4 @@ MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 #endif
 }
 
-/// Helper function to return and increment a given pointer with a given size
-template<typename T> MFEM_HOST_DEVICE static
-inline T *DeviceMemAlloc(T* &mem, size_t size) noexcept
-{
-   T* base = mem;
-   return (mem += size, base);
-}
-
 #endif // MFEM_BACKENDS_HPP