Merge remote-tracking branch 'origin/master' into eqdsk-dev

# Conflicts:
#	makefile
#	miniapps/plasma/CMakeLists.txt
#	miniapps/plasma/makefile
#	miniapps/plasma/plasma.hpp

.github/workflows/builds-and-tests.yml

@@ -142,6 +142,10 @@ jobs:
 
     continue-on-error: ${{ matrix.enzyme && true || false }}
 
+    # Enable ccache for all jobs except Windows (would need sccache).
+    env:
+      USE_CCACHE: ${{ matrix.os != 'windows-latest' }}
+
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -290,6 +294,52 @@ jobs:
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
           echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
 
+      # Restore the compiler cache (ccache). The key embeds the run id, so new
+      # runs save a fresh snapshot; the restore-keys prefix warm-starts from the
+      # most recent prior run (incl. the base branch for PRs).
+      - name: cache ccache
+        if: ${{ env.USE_CCACHE == 'true' }}
+        uses: actions/cache@v5
+        with:
+          path: .ccache
+          key: ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-${{ github.run_id }}
+          restore-keys: |
+            ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-
+
+      # Configure ccache and select how it is injected into the MFEM build:
+      # - make:  set CXX="ccache g++"; for MPI, OMPI_CXX="ccache g++" so mpicxx
+      #          runs ccache around g++ (not ccache around the mpicxx wrapper).
+      # - cmake: set CMAKE_<LANG>_COMPILER_LAUNCHER=ccache.
+      # - enzyme: wrap the brew clang++ via OMPI_CXX.
+      # The chosen options are passed through build-mfem's 'config-options'
+      # input (see the build step below).
+      - name: configure ccache
+        if: ${{ env.USE_CCACHE == 'true' }}
+        run: |
+          command -v ccache >/dev/null 2>&1 || {
+            if [[ "${{ runner.os }}" == "Linux" ]]; then
+              sudo apt-get update && sudo apt-get install -y ccache
+            else
+              brew install ccache
+            fi
+          }
+          echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
+          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
+          # Ignore header timestamps (restamped by each checkout) so direct mode hits.
+          echo "CCACHE_SLOPPINESS=include_file_mtime,include_file_ctime,time_macros" >> $GITHUB_ENV
+          # Hash absolute paths relative to the workspace.
+          echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
+          if [[ "${{ matrix.enzyme }}" == "true" ]]; then
+            echo "OMPI_CXX=ccache $LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
+          elif [[ "${{ matrix.build-system }}" == "cmake" ]]; then
+            echo 'CCACHE_CONFIG_OPTS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache' >> $GITHUB_ENV
+          else
+            echo "OMPI_CXX=ccache g++" >> $GITHUB_ENV
+            echo 'CCACHE_CONFIG_OPTS=CXX="ccache g++" MPICXX="mpicxx"' >> $GITHUB_ENV
+          fi
+        shell: bash
+
       # MFEM build and test
       - name: build
         uses: mfem/github-actions/build-mfem@v2.7
@@ -305,9 +355,14 @@ jobs:
           metis-dir: ${{ env.METIS_TOP_DIR }}
           mfem-dir: ${{ env.MFEM_TOP_DIR }}
           precision: ${{ matrix.precision }}
-          config-options: ${{ matrix.config-opts }}
+          config-options: ${{ matrix.config-opts }} ${{ env.CCACHE_CONFIG_OPTS }}
           library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
 
+      - name: ccache stats
+        if: ${{ env.USE_CCACHE == 'true' }}
+        run: ccache -s
+        shell: bash
+
       # Run checks (and only checks) on debug targets
       - name: checks
         if: matrix.build-system == 'make' && matrix.target == 'dbg'

.github/workflows/cleanup-pr-caches.yml

@@ -0,0 +1,42 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+# A closed PR's caches can never be restored again, so delete them to free
+# space against the 10 GB per-repo cache limit.
+name: Cleanup PR caches
+
+on:
+  pull_request:
+    types: [closed]
+
+permissions:
+  actions: write
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Delete caches for the closed PR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          PR_REF: refs/pull/${{ github.event.pull_request.number }}/merge
+        run: |
+          echo "Deleting caches for $PR_REF"
+          while :; do
+            ids=$(gh cache list --ref "$PR_REF" --limit 100 --json id --jq '.[].id')
+            [ -n "$ids" ] || break
+            echo "$ids" | while read -r id; do
+              [ -n "$id" ] || continue
+              echo "Deleting cache $id"
+              gh cache delete "$id" || echo "  (already gone)"
+            done
+          done

.github/workflows/repo-check.yml

@@ -13,6 +13,7 @@ name: "Checks"
 
 permissions:
   actions: write
+  pull-requests: read
 
 on:
   push:
@@ -29,6 +30,11 @@ concurrency:
 # by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
 # and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
 # github.repository").
+#
+# The logic for the branch-history check is slightly different, since that check
+# also inspects the PR's labels to allow for overriding failures. In this case,
+# we run on all 'pull_request' triggers, but only run for 'push' triggers that
+# do not correspond to any open PRs.
 
 jobs:
   file-headers-check:
@@ -128,10 +134,7 @@ jobs:
 
   branch-history:
     if: |
-      github.ref != 'refs/heads/next' &&
-      github.ref != 'refs/heads/master' &&
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+      github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
@@ -139,7 +142,27 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: check for pull request
+        id: check_pr
+        if: github.event_name == 'push'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          pr_exists=$(gh pr list --repo "$GITHUB_REPOSITORY" \
+            --head "$GITHUB_REF_NAME" \
+            --state open \
+            --json number \
+            --jq 'length > 0')
+          echo "pr_exists=$pr_exists" >> "$GITHUB_OUTPUT"
+
       - name: branch-history
+        id: branch_history
+        if: |
+          (github.event_name == 'pull_request' ||
+           github.event_name == 'workflow_dispatch' ||
+           steps.check_pr.outputs.pr_exists == 'false')
+        continue-on-error: ${{ contains(github.event.pull_request.labels.*.name,
+                                        'branch-history-override') }}
         run: |
           # We override origin to make sure we point to the main repo.
           # This is to have consistent test results on PRs from forks.
@@ -147,3 +170,9 @@ jobs:
           git remote add origin https://github.com/mfem/mfem.git
           git checkout -b gh-actions-branch-history
           ./config/githooks/pre-push --history
+
+      - name: report branch-history override
+        if: steps.branch_history.outcome == 'failure'
+        run: |
+          echo "::warning::branch-history check failed, but the" \
+            "'branch-history-override' label is set."

.gitignore

@@ -260,6 +260,7 @@ miniapps/meshing/polar-nc
 miniapps/meshing/mesh-quality
 miniapps/meshing/hpref
 miniapps/meshing/phpref
+miniapps/meshing/pref321
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -355,6 +356,11 @@ miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
 
+miniapps/plasma/g_eqdsk_viewer
+miniapps/plasma/gnuplot_eqdsk.*
+miniapps/plasma/G_EQDSK_Viewer*
+miniapps/plasma/ParaView
+
 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
 miniapps/shifted/ParaViewLSF

.gitlab/dane-baseline.yml

@@ -102,12 +102,14 @@ report_baseline:
         mkdir -p ${MACHINE_NAME}
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
+        status=0
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir} || { status=1; }
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
         # We create an autotest-email.html file, because that's how we signal
         # that there was an error / diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+        if [[ $status -ne 0 ]] || \
+           [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
            [[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi

CHANGELOG

@@ -46,8 +46,19 @@ Discretization improvements
 
 - Extend FindPointsGSLIB to support surface meshes.
 
+- Added support for complex-valued mixed bilinear forms via the new classes
+  MixedSesquilinearForm and ParMixedSesquilinearForm, mirroring the existing
+  SesquilinearForm classes. Rectangular complex operators are now also
+  handled correctly by ComplexSparseMatrix::GetSystemMatrix and
+  ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
+  trial and test spaces.
+
 Meshing improvements
 --------------------
+- Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
+  meshes with arbitrary spacing in each direction. This enables in particular
+  3:1 refinement in parallel, as demonstrated in the new meshing miniapp pref321.
+
 - Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
   bounds on the determinant of the mesh transformation Jacobian.
 
@@ -70,9 +81,15 @@ Linear and nonlinear solvers
 
 GPU computing
 -------------
+- Improved partial assembly for VectorDivergenceIntegrator with shared-memory
+  kernels, kernel registration, and transpose support.
+
 - Improved partial-assembly diagonal kernels for VectorMassIntegrator (shared-
   memory specializations) and ElasticityIntegrator (no scratch Q-vector).
 
+- Added PA gradient and diagonal support for VectorConvectionNLFIntegrator
+ (AssembleGradPA, AddMultGradPA, AssembleGradDiagonalPA).
+
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
 - Added NVIDIA cuDSS library interface. Implementation examples have been
@@ -81,6 +98,12 @@ GPU computing
 
 - Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
 
+- Changed VectorFEMassIntegrator to use kernel specialization dispatch for
+  partial assembly.
+
+- Added support for FiniteElement::MapType::INTEGRAL spaces to
+  QuadratureInterpolator.
+
 New and updated examples and miniapps
 -------------------------------------
 - The Lorentz miniapp (in miniapps/electromagnetics) has been updated to

CMakeLists.txt

@@ -239,6 +239,13 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
+# Shadow warnings for clang only; GCC's -Wshadow flags more.
+if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall -Wshadow")
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall")
+endif()
+
 # Shared build on Windows
 if (WIN32 AND BUILD_SHARED_LIBS)
   # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?

config/defaults.mk

@@ -27,7 +27,13 @@ MPICXX = mpicxx
 
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
-DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
+
+# Shadow warnings for clang only; GCC's -Wshadow flags more.
+SHADOW_WARNING_FLAG = $(if $(findstring clang,\
+ $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
+WARNING_FLAGS = -pedantic -Wall $(SHADOW_WARNING_FLAG)
+
+DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
 CXX_XCOMPILER =

config/githooks/README.md

@@ -39,3 +39,8 @@ when a picture was added for documentation.
 If that is the case, make sure the failure is indeed justified, and rerun the
 push command with the `--no-verify` option. This will skip the hooks, allowing
 you to push those changes.
+
+The `branch-history` check is run automatically through GitHub Actions. If a
+branch is known to have a large number of changes that are legitimate, the
+check can be overridden by setting  the label 'branch-history-override' on the
+pull request.

data/tinyzoo-2d.mesh

@@ -0,0 +1,38 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+
+dimension
+2
+
+elements
+2
+1 3 0 1 4 3
+1 2 1 2 4
+
+boundary
+5
+1 1 0 1
+1 1 1 2
+1 1 2 4
+1 1 4 3
+1 1 3 0
+
+vertices
+5
+2
+0 0
+1 0
+2 0
+0 1
+1 1

doc/CodeDocumentation.dox

@@ -201,6 +201,7 @@ namespace mfem {
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
  * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
+ * - <a class="el" href="nurbs__surface_8cpp_source.html">NURBS Surface</a>: interpolate a 3D Surface in a NURBS Patch
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -245,6 +246,9 @@ namespace mfem {
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
+ * - <a class="el" href="reflector_8cpp_source.html">Reflector Miniapp</a>: reflect a mesh about a plane
+ * - <a class="el" href="ref321_8cpp_source.html">3:1 Refinement Miniapp</a>: perform 3:1 anisotropic mesh refinements
+ * - <a class="el" href="pref321_8cpp_source.html">3:1 Refinement Miniapp</a>: parallel 3:1 anisotropic mesh refinements
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

fem/CMakeLists.txt

@@ -57,6 +57,8 @@ set(SRCS
   integ/lininteg_domain_grad.cpp
   integ/lininteg_domain_vectorfe.cpp
   integ/nonlininteg_vecconvection_pa.cpp
+  integ/nonlininteg_vecconvection_pa_diag.cpp
+  integ/nonlininteg_vecconvection_pa_grad.cpp
   integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
@@ -204,7 +206,11 @@ set(HDRS
   integ/bilininteg_mass_kernels.hpp
   integ/bilininteg_mass_pa_simplices.hpp
   integ/bilininteg_vecdiffusion_pa.hpp
+  integ/bilininteg_vecdiv_pa.hpp
   integ/bilininteg_vecmass_pa.hpp
+  integ/nonlininteg_vecconvection_pa.hpp
+  integ/nonlininteg_vecconvection_pa_diag.hpp
+  integ/nonlininteg_vecconvection_pa_grad.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp

fem/bilininteg.hpp

@@ -3003,11 +3003,10 @@ public:
     vector (diagonal matrix), or matrix), trial function $u$ is in $H(curl$ or
     $H(div)$, and test function $v$ is in $H(curl$, $H(div)$, or $v=(v_1,\dots,v_n)$, where
     $v_i$ are in $H^1$. */
-class VectorFEMassIntegrator: public BilinearFormIntegrator
+class VectorFEMassIntegrator : public BilinearFormIntegrator
 {
 private:
-   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq)
-   { Q = q; DQ = dq; MQ = mq; }
+   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq);
 
 #ifndef MFEM_THREAD_SAFE
    Vector shape;
@@ -3030,7 +3029,8 @@ protected:
    const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
    const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
+   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D;
+   FiniteElement::DerivType trial_fetype, test_fetype;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
@@ -3061,6 +3061,29 @@ public:
                    const bool add) override;
 
    const Coefficient *GetCoefficient() const { return Q; }
+
+   using ApplyKernelType =
+      void (*)(const int NE, bool symmetric, const bool scalar_coeff,
+               const Array<real_t> &trialBO, const Array<real_t> &trialBC,
+               const Array<real_t> &testBOt, const Array<real_t> &testBCt,
+               const Vector &pa_data, const Vector &x, Vector &y,
+               const int triald1d, const int testd1d, const int q1d);
+
+   /// parameters: trial_fetype, test_fetype, ndims, trial_d1d, test_d1d, q1d
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType,
+                         (FiniteElement::DerivType, FiniteElement::DerivType,
+                          int, int, int, int));
+
+   struct Kernels { Kernels(); };
+
+   template <FiniteElement::DerivType TrialType,
+             FiniteElement::DerivType TestType, int DIM, int TRIAL_D1D,
+             int TEST_D1D, int Q1D>
+   static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<TrialType, TestType, DIM, TRIAL_D1D,
+                     TEST_D1D, Q1D>::Add();
+   }
 };
 
 /** Integrator for $(Q \nabla \cdot u, v)$ where $u=(u_1,\cdots,u_n)$ and all $u_i$ are in the same
@@ -3106,6 +3129,24 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultTransposePA(const Vector &x, Vector &y) const override;
 
+   using VectorDivergenceAddMultPAType =
+      void (*)(const int ne,
+               const Array<real_t> &b, const Array<real_t> &g, const Array<real_t> &bt,
+               const Vector &op, const Vector &x, Vector &y,
+               const int tr_d1d, const int te_d1d, const int q1d);
+   MFEM_REGISTER_KERNELS(VectorDivergenceAddMultPA,
+                         VectorDivergenceAddMultPAType,
+                         (int, int, int, int));
+
+   using VectorDivergenceAddMultTransposePAType =
+      void (*)(const int ne,
+               const Array<real_t> &bt, const Array<real_t> &gt, const Array<real_t> &b,
+               const Vector &q, const Vector &x, Vector &y,
+               const int tr_d1d, const int te_d1d, const int q1d);
+   MFEM_REGISTER_KERNELS(VectorDivergenceAddMultTransposePA,
+                         VectorDivergenceAddMultTransposePAType,
+                         (int, int, int, int));
+
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe,
                                          const ElementTransformation &Trans);

fem/complex_fem.cpp

@@ -237,6 +237,81 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
    gfi->SyncAliasMemory(*this);
 }
 
+real_t
+ComplexGridFunction::ComputeLpError(const real_t p,
+                                    Coefficient &exsolr,
+                                    Coefficient &exsoli,
+                                    Coefficient *weight,
+                                    const IntegrationRule *irs[],
+                                    const Array<int> *elems) const
+{
+   real_t error = 0.0;
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Vector valsr;
+   Vector valsi;
+
+   const GridFunction& gf_r = real();
+   const GridFunction& gf_i = imag();
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      if (elems != NULL && (*elems)[i] == 0) { continue; }
+      fe = fes->GetFE(i);
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[fe->GetGeomType()];
+      }
+      else
+      {
+         int intorder = 2*fe->GetOrder() + 3;
+         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+      }
+      real_t elem_error = 0.0;
+      gf_r.GetValues(i, *ir, valsr);
+      gf_i.GetValues(i, *ir, valsi);
+      T = fes->GetElementTransformation(i);
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         T->SetIntPoint(&ip);
+         real_t diffr = valsr(j) - exsolr.Eval(*T, ip);
+         real_t diffi = valsi(j) - exsoli.Eval(*T, ip);
+         real_t diff = hypot(diffr, diffi);
+         if (p < infinity())
+         {
+            diff = pow(diff, p);
+            if (weight)
+            {
+               diff *= weight->Eval(*T, ip);
+            }
+            elem_error += ip.weight * T->Weight() * diff;
+         }
+         else
+         {
+            if (weight)
+            {
+               diff *= weight->Eval(*T, ip);
+            }
+            error = std::max(error, diff);
+         }
+      }
+      if (p < infinity())
+      {
+         // negative quadrature weights may cause the error to be negative
+         error += fabs(elem_error);
+      }
+   }
+
+   if (p < infinity())
+   {
+      error = pow(error, 1./p);
+   }
+
+   return error;
+}
+
 void ComplexGridFunction::Save(std::ostream &os) const
 {
    os << "ComplexGridFunction\n";
@@ -643,8 +718,8 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
-        A_i.Type() == Operator::MFEM_SPARSEMAT )
+   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
    {
       ComplexSparseMatrix * A_sp =
          new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
@@ -704,8 +779,8 @@ SesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
-        A_i.Type() == Operator::MFEM_SPARSEMAT )
+   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
    {
       ComplexSparseMatrix * A_sp =
          new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
@@ -768,6 +843,426 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
    if ( blfi ) { blfi->Update(nfes); }
 }
 
+bool
+MixedSesquilinearForm::RealInteg()
+{
+   int nint = mblfr->GetDBFI()->Size() + mblfr->GetBBFI()->Size() +
+              mblfr->GetFBFI()->Size() + mblfr->GetBFBFI()->Size() +
+              mblfr->GetTFBFI()->Size() + mblfr->GetBTFBFI()->Size();
+   return (nint != 0);
+}
+
+bool
+MixedSesquilinearForm::ImagInteg()
+{
+   int nint = mblfi->GetDBFI()->Size() + mblfi->GetBBFI()->Size() +
+              mblfi->GetFBFI()->Size() + mblfi->GetBFBFI()->Size() +
+              mblfi->GetTFBFI()->Size() + mblfi->GetBTFBFI()->Size();
+   return (nint != 0);
+}
+
+MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace * trial_fes,
+                                             FiniteElementSpace * test_fes,
+                                             ComplexOperator::Convention convention)
+   : conv(convention),
+     mblfr(new mfem::MixedBilinearForm(trial_fes, test_fes)),
+     mblfi(new mfem::MixedBilinearForm(trial_fes, test_fes))
+{
+}
+
+MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace * trial_fes,
+                                             FiniteElementSpace * test_fes,
+                                             MixedBilinearForm * bfr,
+                                             MixedBilinearForm * bfi,
+                                             ComplexOperator::Convention convention)
+   : conv(convention),
+     mblfr(new MixedBilinearForm(trial_fes, test_fes, bfr)),
+     mblfi(new MixedBilinearForm(trial_fes, test_fes, bfi))
+{
+}
+
+MixedSesquilinearForm::~MixedSesquilinearForm()
+{
+   delete mblfr;
+   delete mblfi;
+}
+
+void
+MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                                           BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      mblfr->AddDomainIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddDomainIntegrator(bfi_imag);
+   }
+}
+
+void
+MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                                           BilinearFormIntegrator * bfi_imag,
+                                           Array<int> & elem_marker)
+{
+   if (bfi_real)
+   {
+      mblfr->AddDomainIntegrator(bfi_real, elem_marker);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddDomainIntegrator(bfi_imag, elem_marker);
+   }
+}
+
+void
+MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                                             BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      mblfr->AddBoundaryIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddBoundaryIntegrator(bfi_imag);
+   }
+}
+
+void
+MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                                             BilinearFormIntegrator * bfi_imag,
+                                             Array<int> & bdr_marker)
+{
+   if (bfi_real)
+   {
+      mblfr->AddBoundaryIntegrator(bfi_real, bdr_marker);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddBoundaryIntegrator(bfi_imag, bdr_marker);
+   }
+}
+
+void
+MixedSesquilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *
+                                                 bfi_real,
+                                                 BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      mblfr->AddInteriorFaceIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddInteriorFaceIntegrator(bfi_imag);
+   }
+}
+
+void
+MixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                            BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      mblfr->AddBdrFaceIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddBdrFaceIntegrator(bfi_imag);
+   }
+}
+
+void
+MixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                            BilinearFormIntegrator * bfi_imag,
+                                            Array<int> & bdr_marker)
+{
+   if (bfi_real)
+   {
+      mblfr->AddBdrFaceIntegrator(bfi_real, bdr_marker);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddBdrFaceIntegrator(bfi_imag, bdr_marker);
+   }
+}
+
+void MixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator *
+                                                   bfi_real,
+                                                   BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      mblfr->AddTraceFaceIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddTraceFaceIntegrator(bfi_imag);
+   }
+}
+
+void MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
+                                                      *bfi_real,
+                                                      BilinearFormIntegrator *bfi_imag)
+{
+   if (bfi_real)
+   {
+      mblfr->AddBdrTraceFaceIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddBdrTraceFaceIntegrator(bfi_imag);
+   }
+}
+
+void MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
+                                                      *bfi_real,
+                                                      BilinearFormIntegrator *bfi_imag,
+                                                      Array<int> &bdr_marker)
+{
+   if (bfi_real)
+   {
+      mblfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker);
+   }
+   if (bfi_imag)
+   {
+      mblfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker);
+   }
+}
+
+void
+MixedSesquilinearForm::Assemble(int skip_zeros)
+{
+   mblfr->Assemble(skip_zeros);
+   mblfi->Assemble(skip_zeros);
+}
+
+void
+MixedSesquilinearForm::Finalize(int skip_zeros)
+{
+   mblfr->Finalize(skip_zeros);
+   mblfi->Finalize(skip_zeros);
+}
+
+ComplexSparseMatrix *
+MixedSesquilinearForm::AssembleComplexSparseMatrix()
+{
+   return new mfem::ComplexSparseMatrix(
+             &mblfr->SpMat(), &mblfi->SpMat(), false, false, conv);
+}
+
+void
+MixedSesquilinearForm::FormRectangularLinearSystem(const Array<int> &
+                                                   ess_trial_tdof_list,
+                                                   const Array<int> & ess_test_tdof_list,
+                                                   Vector & x,
+                                                   Vector & b,
+                                                   OperatorHandle & A,
+                                                   Vector & X,
+                                                   Vector & B)
+{
+   FiniteElementSpace * fes_trial = mblfr->TrialFESpace();
+   FiniteElementSpace * fes_test = mblfr->TestFESpace();
+   const int vsize_trial = fes_trial->GetVSize();
+   const int vsize_test = fes_test->GetVSize();
+
+   // Allocate temporary Vector
+   Vector b_0;
+   b_0.UseDevice(true);
+   b_0.SetSize(vsize_test);
+   b_0 = 0.0;
+
+   // Extract the real and imaginary parts of the input Vectors
+   MFEM_ASSERT(x.Size() == 2 * vsize_trial,
+               "Input GridFunction of incorrect size!");
+   x.Read();
+   Vector x_r;
+   x_r.MakeRef(x, 0, vsize_trial);
+   Vector x_i;
+   x_i.MakeRef(x, vsize_trial, vsize_trial);
+
+   MFEM_ASSERT(b.Size() == 2 * vsize_test, "Input LinearForm of incorrect size!");
+   b.Read();
+   Vector b_r;
+   b_r.MakeRef(b, 0, vsize_test);
+   Vector b_i;
+   b_i.MakeRef(b, vsize_test, vsize_test);
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      b_i *= -1.0;
+   }
+
+   const int tvsize_trial = fes_trial->GetTrueVSize();
+   const int tvsize_test = fes_test->GetTrueVSize();
+   OperatorHandle A_r, A_i;
+
+   X.UseDevice(true);
+   X.SetSize(2 * tvsize_trial);
+   X = 0.0;
+
+   B.UseDevice(true);
+   B.SetSize(2 * tvsize_test);
+   B = 0.0;
+
+   Vector X_r;
+   X_r.MakeRef(X, 0, tvsize_trial);
+   Vector X_i;
+   X_i.MakeRef(X, tvsize_trial, tvsize_trial);
+   Vector B_r;
+   B_r.MakeRef(B, 0, tvsize_test);
+   Vector B_i;
+   B_i.MakeRef(B, tvsize_test, tvsize_test);
+
+   Vector X_0, B_0;
+
+   if (RealInteg())
+   {
+      b_0 = b_r;
+      mblfr->FormRectangularLinearSystem(
+         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_r, X_0, B_0);
+      X_r = X_0;
+      B_r = B_0;
+
+      b_0 = b_i;
+      mblfr->FormRectangularLinearSystem(
+         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_r, X_0, B_0);
+      X_i = X_0;
+      B_i = B_0;
+
+      if (ImagInteg())
+      {
+         b_0 = 0.0;
+         mblfi->FormRectangularLinearSystem(
+            ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
+         B_r -= B_0;
+
+         b_0 = 0.0;
+         mblfi->FormRectangularLinearSystem(
+            ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
+         B_i += B_0;
+      }
+   }
+   else if (ImagInteg())
+   {
+      b_0 = b_i;
+      mblfi->FormRectangularLinearSystem(
+         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
+      X_r = X_0;
+      B_i = B_0;
+
+      b_0 = b_r;
+      b_0 *= -1.0;
+      mblfi->FormRectangularLinearSystem(
+         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
+      X_i = X_0;
+      B_r = B_0;
+      B_r *= -1.0;
+   }
+   else
+   {
+      MFEM_ABORT("Real and Imaginary part of the Mixed Sesquilinear form are empty");
+   }
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      B_i *= -1.0;
+      b_i *= -1.0;
+   }
+
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
+   b_r.SyncAliasMemory(b);
+   b_i.SyncAliasMemory(b);
+
+   X_r.SyncAliasMemory(X);
+   X_i.SyncAliasMemory(X);
+   B_r.SyncAliasMemory(B);
+   B_i.SyncAliasMemory(B);
+
+   // A = A_r + i A_i
+   A.Clear();
+   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   {
+      ComplexSparseMatrix * A_hyp =
+         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
+                                 A_i.As<SparseMatrix>(),
+                                 A_r.OwnsOperator(),
+                                 A_i.OwnsOperator(),
+                                 conv);
+      A.Reset<ComplexSparseMatrix>(A_hyp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
+                                                   A_i.As<Operator>(),
+                                                   A_r.OwnsOperator(),
+                                                   A_i.OwnsOperator(),
+                                                   conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
+void
+MixedSesquilinearForm::FormRectangularSystemMatrix(const mfem::Array<int> &
+                                                   ess_trial_tdof_list,
+                                                   const mfem::Array<int> & ess_test_tdof_list,
+                                                   mfem::OperatorHandle & A)
+{
+   OperatorHandle A_r, A_i;
+   if (RealInteg())
+   {
+      mblfr->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
+                                         A_r);
+   }
+   if (ImagInteg())
+   {
+      mblfi->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
+                                         A_i);
+   }
+   if (!RealInteg() && !ImagInteg())
+   {
+      MFEM_ABORT("Both Real and Imaginary part of the Mixed Sesquilinear form are empty");
+   }
+
+   // A = A_r + i A_i
+   A.Clear();
+   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   {
+      ComplexSparseMatrix * A_hyp =
+         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
+                                 A_i.As<SparseMatrix>(),
+                                 A_r.OwnsOperator(),
+                                 A_i.OwnsOperator(),
+                                 conv);
+      A.Reset<ComplexSparseMatrix>(A_hyp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
+                                                   A_i.As<Operator>(),
+                                                   A_r.OwnsOperator(),
+                                                   A_i.OwnsOperator(),
+                                                   conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
+void
+MixedSesquilinearForm::Update()
+{
+   mblfr->Update();
+   mblfi->Update();
+}
+
 
 #ifdef MFEM_USE_MPI
 
@@ -1539,8 +2034,8 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ( A_r.Type() == Operator::Hypre_ParCSR ||
-        A_i.Type() == Operator::Hypre_ParCSR )
+   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
    {
       ComplexHypreParMatrix * A_hyp =
          new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
@@ -1607,8 +2102,8 @@ ParSesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ( A_r.Type() == Operator::Hypre_ParCSR ||
-        A_i.Type() == Operator::Hypre_ParCSR )
+   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
    {
       ComplexHypreParMatrix * A_hyp =
          new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
@@ -1666,6 +2161,434 @@ ParSesquilinearForm::Update(FiniteElementSpace *nfes)
    if ( pblfi ) { pblfi->Update(nfes); }
 }
 
+bool
+ParMixedSesquilinearForm::RealInteg()
+{
+   int nint = pmblfr->GetDBFI()->Size() + pmblfr->GetBBFI()->Size() +
+              pmblfr->GetFBFI()->Size() + pmblfr->GetBFBFI()->Size() +
+              pmblfr->GetTFBFI()->Size() + pmblfr->GetBTFBFI()->Size();
+   return (nint != 0);
+}
+
+bool
+ParMixedSesquilinearForm::ImagInteg()
+{
+   int nint = pmblfi->GetDBFI()->Size() + pmblfi->GetBBFI()->Size() +
+              pmblfi->GetFBFI()->Size() + pmblfi->GetBFBFI()->Size() +
+              pmblfi->GetTFBFI()->Size() + pmblfi->GetBTFBFI()->Size();
+   return (nint != 0);
+}
+
+ParMixedSesquilinearForm::ParMixedSesquilinearForm(ParFiniteElementSpace *
+                                                   trial_fes,
+                                                   ParFiniteElementSpace * test_fes,
+                                                   ComplexOperator::Convention convention)
+   : conv(convention),
+     pmblfr(new ParMixedBilinearForm(trial_fes, test_fes)),
+     pmblfi(new ParMixedBilinearForm(trial_fes, test_fes))
+{
+}
+
+ParMixedSesquilinearForm::ParMixedSesquilinearForm(ParFiniteElementSpace *
+                                                   trial_fes,
+                                                   ParFiniteElementSpace * test_fes,
+                                                   ParMixedBilinearForm * pbfr,
+                                                   ParMixedBilinearForm * pbfi,
+                                                   ComplexOperator::Convention convention)
+   : conv(convention),
+     pmblfr(new ParMixedBilinearForm(trial_fes, test_fes, pbfr)),
+     pmblfi(new ParMixedBilinearForm(trial_fes, test_fes, pbfi))
+{
+}
+
+ParMixedSesquilinearForm::~ParMixedSesquilinearForm()
+{
+   delete pmblfr;
+   delete pmblfi;
+}
+
+void
+ParMixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                                              BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddDomainIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddDomainIntegrator(bfi_imag);
+   }
+}
+
+void
+ParMixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                                              BilinearFormIntegrator * bfi_imag,
+                                              Array<int> & elem_marker)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddDomainIntegrator(bfi_real, elem_marker);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddDomainIntegrator(bfi_imag, elem_marker);
+   }
+}
+
+void
+ParMixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *
+                                                bfi_real,
+                                                BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddBoundaryIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddBoundaryIntegrator(bfi_imag);
+   }
+}
+
+void
+ParMixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *
+                                                bfi_real,
+                                                BilinearFormIntegrator * bfi_imag,
+                                                Array<int> & bdr_marker)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddBoundaryIntegrator(bfi_real, bdr_marker);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddBoundaryIntegrator(bfi_imag, bdr_marker);
+   }
+}
+
+void
+ParMixedSesquilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *
+                                                    bfi_real,
+                                                    BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddInteriorFaceIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddInteriorFaceIntegrator(bfi_imag);
+   }
+}
+
+void
+ParMixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *
+                                               bfi_real,
+                                               BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddBdrFaceIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddBdrFaceIntegrator(bfi_imag);
+   }
+}
+
+void
+ParMixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *
+                                               bfi_real,
+                                               BilinearFormIntegrator * bfi_imag,
+                                               Array<int> & bdr_marker)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddBdrFaceIntegrator(bfi_real, bdr_marker);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddBdrFaceIntegrator(bfi_imag, bdr_marker);
+   }
+}
+
+void ParMixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator *
+                                                      bfi_real,
+                                                      BilinearFormIntegrator * bfi_imag)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddTraceFaceIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddTraceFaceIntegrator(bfi_imag);
+   }
+}
+
+void ParMixedSesquilinearForm::AddBdrTraceFaceIntegrator(
+   BilinearFormIntegrator *bfi_real,
+   BilinearFormIntegrator *bfi_imag)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddBdrTraceFaceIntegrator(bfi_real);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddBdrTraceFaceIntegrator(bfi_imag);
+   }
+}
+
+void ParMixedSesquilinearForm::AddBdrTraceFaceIntegrator(
+   BilinearFormIntegrator *bfi_real,
+   BilinearFormIntegrator *bfi_imag,
+   Array<int> &bdr_marker)
+{
+   if (bfi_real)
+   {
+      pmblfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker);
+   }
+   if (bfi_imag)
+   {
+      pmblfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker);
+   }
+}
+
+void
+ParMixedSesquilinearForm::Assemble(int skip_zeros)
+{
+   pmblfr->Assemble(skip_zeros);
+   pmblfi->Assemble(skip_zeros);
+}
+
+void
+ParMixedSesquilinearForm::Finalize(int skip_zeros)
+{
+   pmblfr->Finalize(skip_zeros);
+   pmblfi->Finalize(skip_zeros);
+}
+
+ComplexHypreParMatrix *
+ParMixedSesquilinearForm::ParallelAssemble()
+{
+   return new ComplexHypreParMatrix(
+             pmblfr->ParallelAssemble(), pmblfi->ParallelAssemble(), true, true, conv);
+}
+
+void
+ParMixedSesquilinearForm::FormRectangularLinearSystem(const Array<int> &
+                                                      ess_trial_tdof_list,
+                                                      const Array<int> & ess_test_tdof_list,
+                                                      Vector & x,
+                                                      Vector & b,
+                                                      OperatorHandle & A,
+                                                      Vector & X,
+                                                      Vector & B)
+{
+   FiniteElementSpace * pfes_trial = pmblfr->TrialFESpace();
+   FiniteElementSpace * pfes_test = pmblfr->TestFESpace();
+   const int vsize_trial = pfes_trial->GetVSize();
+   const int vsize_test = pfes_test->GetVSize();
+
+   // Allocate temporary Vector
+   Vector b_0;
+   b_0.UseDevice(true);
+   b_0.SetSize(vsize_test);
+   b_0 = 0.0;
+
+   // Extract the real and imaginary parts of the input Vectors
+   MFEM_ASSERT(x.Size() == 2 * vsize_trial,
+               "Input GridFunction of incorrect size!");
+   x.Read();
+   Vector x_r;
+   x_r.MakeRef(x, 0, vsize_trial);
+   Vector x_i;
+   x_i.MakeRef(x, vsize_trial, vsize_trial);
+
+   MFEM_ASSERT(b.Size() == 2 * vsize_test, "Input LinearForm of incorrect size!");
+   b.Read();
+   Vector b_r;
+   b_r.MakeRef(b, 0, vsize_test);
+   Vector b_i;
+   b_i.MakeRef(b, vsize_test, vsize_test);
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      b_i *= -1.0;
+   }
+
+   const int tvsize_trial = pfes_trial->GetTrueVSize();
+   const int tvsize_test = pfes_test->GetTrueVSize();
+   OperatorHandle A_r, A_i;
+
+   X.UseDevice(true);
+   X.SetSize(2 * tvsize_trial);
+   X = 0.0;
+
+   B.UseDevice(true);
+   B.SetSize(2 * tvsize_test);
+   B = 0.0;
+
+   Vector X_r;
+   X_r.MakeRef(X, 0, tvsize_trial);
+   Vector X_i;
+   X_i.MakeRef(X, tvsize_trial, tvsize_trial);
+   Vector B_r;
+   B_r.MakeRef(B, 0, tvsize_test);
+   Vector B_i;
+   B_i.MakeRef(B, tvsize_test, tvsize_test);
+
+   Vector X_0, B_0;
+
+   if (RealInteg())
+   {
+      b_0 = b_r;
+      pmblfr->FormRectangularLinearSystem(
+         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_r, X_0, B_0);
+      X_r = X_0;
+      B_r = B_0;
+
+      b_0 = b_i;
+      pmblfr->FormRectangularLinearSystem(
+         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_r, X_0, B_0);
+      X_i = X_0;
+      B_i = B_0;
+
+      if (ImagInteg())
+      {
+         b_0 = 0.0;
+         pmblfi->FormRectangularLinearSystem(
+            ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
+         B_r -= B_0;
+
+         b_0 = 0.0;
+         pmblfi->FormRectangularLinearSystem(
+            ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
+         B_i += B_0;
+      }
+   }
+   else if (ImagInteg())
+   {
+      b_0 = b_i;
+      pmblfi->FormRectangularLinearSystem(
+         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
+      X_r = X_0;
+      B_i = B_0;
+
+      b_0 = b_r;
+      b_0 *= -1.0;
+      pmblfi->FormRectangularLinearSystem(
+         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
+      X_i = X_0;
+      B_r = B_0;
+      B_r *= -1.0;
+   }
+   else
+   {
+      MFEM_ABORT("Real and Imaginary part of the Mixed Sesquilinear form are empty");
+   }
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      B_i *= -1.0;
+      b_i *= -1.0;
+   }
+
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
+   b_r.SyncAliasMemory(b);
+   b_i.SyncAliasMemory(b);
+
+   X_r.SyncAliasMemory(X);
+   X_i.SyncAliasMemory(X);
+   B_r.SyncAliasMemory(B);
+   B_i.SyncAliasMemory(B);
+
+   // A = A_r + i A_i
+   A.Clear();
+   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   {
+      ComplexHypreParMatrix * A_hyp =
+         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
+                                   A_i.As<HypreParMatrix>(),
+                                   A_r.OwnsOperator(),
+                                   A_i.OwnsOperator(),
+                                   conv);
+      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
+                                                   A_i.As<Operator>(),
+                                                   A_r.OwnsOperator(),
+                                                   A_i.OwnsOperator(),
+                                                   conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
+void
+ParMixedSesquilinearForm::FormRectangularSystemMatrix(const Array<int> &
+                                                      ess_trial_tdof_list,
+                                                      const Array<int> & ess_test_tdof_list,
+                                                      OperatorHandle & A)
+{
+   OperatorHandle A_r, A_i;
+   if (RealInteg())
+   {
+      pmblfr->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
+                                          A_r);
+   }
+   if (ImagInteg())
+   {
+      pmblfi->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
+                                          A_i);
+   }
+   if (!RealInteg() && !ImagInteg())
+   {
+      MFEM_ABORT("Both Real and Imaginary part of the Mixed Sesquilinear form are empty");
+   }
+
+   // A = A_r + i A_i
+   A.Clear();
+   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   {
+      ComplexHypreParMatrix * A_hyp =
+         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
+                                   A_i.As<HypreParMatrix>(),
+                                   A_r.OwnsOperator(),
+                                   A_i.OwnsOperator(),
+                                   conv);
+      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
+                                                   A_i.As<Operator>(),
+                                                   A_r.OwnsOperator(),
+                                                   A_i.OwnsOperator(),
+                                                   conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
+void
+ParMixedSesquilinearForm::Update()
+{
+   pmblfr->Update();
+   pmblfi->Update();
+}
+
+
+
 #endif // MFEM_USE_MPI
 
 }

fem/complex_fem.hpp

@@ -166,6 +166,75 @@ public:
       return sqrt(err_r * err_r + err_i * err_i);
    }
 
+   /// @brief Returns Max|u_ex - u_h| error for complex-valued H1 or L2 elements
+   ///
+   /// Compute the $L_\infty$ error across the entire domain.
+   ///
+   /// @param[in] exsolr  Coefficient object reproducing the real part of the
+   ///                    anticipated values of the scalar field, Re(u_ex).
+   /// @param[in] exsoli  Coefficient object reproducing the imaginary part of
+   ///                    the anticipated values of the scalar field, Im(u_ex).
+   /// @param[in] irs        Optional pointer to an array of custom integration
+   ///                       rules e.g. higher order than the default rules. If
+   ///                       present the array will be indexed by
+   ///                       Geometry::Type.
+   ///
+   /// @note Uses ComputeLpError internally. See the ComputeLpError
+   ///       documentation for generalizations of this error computation.
+   ///
+   /// @note If an array of integration rules is provided through @a irs, be
+   ///       sure to include valid rules for each element type that may occur
+   ///       in the list of elements.
+   ///
+   virtual real_t ComputeMaxError(Coefficient &exsolr,
+                                  Coefficient &exsoli,
+                                  const IntegrationRule *irs[] = NULL) const
+   {
+      return ComputeLpError(infinity(), exsolr, exsoli, NULL, irs);
+   }
+
+   /// @brief Returns ||u_ex - u_h||_Lp for complex-valued H1 or L2 elements
+   ///
+   /// Computes:
+   ///    $$(\sum_{elems} \int_{elem} w \, |u_{ex} - u_h|^p)^{1/p}$$
+   /// Where:
+   ///    $$|u_{ex} - u_h| = \sqrt{Re(u_{ex} - u_h)^2 + Im(u_{ex} - u_h)^2}$$
+   ///
+   /// @param[in] p       Real value indicating the exponent of the $L^p$ norm.
+   ///                    To avoid domain errors p should have a positive value,
+   ///                    either finite or infinite.
+   /// @param[in] exsolr  Coefficient object reproducing the real part of the
+   ///                    anticipated values of the scalar field, Re(u_ex).
+   /// @param[in] exsoli  Coefficient object reproducing the imaginary part of
+   ///                    the anticipated values of the scalar field, Im(u_ex).
+   /// @param[in] weight  Optional pointer to a Coefficient object reproducing
+   ///                    a weighting function, w.
+   /// @param[in] irs     Optional pointer to an array of custom integration
+   ///                    rules e.g. higher order than the default rules. If
+   ///                    present the array will be indexed by Geometry::Type.
+   /// @param[in] elems   Optional pointer to a marker array, with a length
+   ///                    equal to the number of local elements, indicating
+   ///                    which elements to integrate over. Only those elements
+   ///                    corresponding to non-zero entries in @a elems will
+   ///                    contribute to the computed L2 error.
+   ///
+   /// @note If an array of integration rules is provided through @a irs, be
+   ///       sure to include valid rules for each element type that may occur
+   ///       in the list of elements.
+   ///
+   /// @note Quadratures with negative weights (as in some simplex integration
+   ///       rules in MFEM) can produce negative integrals even with
+   ///       non-negative integrands. To avoid returning negative errors this
+   ///       function uses the absolute values of the element-wise integrals.
+   ///       This may lead to results which are not entirely consistent with
+   ///       such integration rules.
+   virtual real_t ComputeLpError(const real_t p,
+                                 Coefficient &exsolr,
+                                 Coefficient &exsoli,
+                                 Coefficient *weight = NULL,
+                                 const IntegrationRule *irs[] = NULL,
+                                 const Array<int> *elems = NULL) const;
+
    /// Save the ComplexGridFunction to an output stream.
    virtual void Save(std::ostream &out) const;
 
@@ -436,6 +505,186 @@ public:
    virtual ~SesquilinearForm();
 };
 
+/** Class for a mixed sesquilinear form
+
+    A mixed sesquilinear form is a generalization of a mixed bilinear form to
+    complex-valued fields. Mixed sesquilinear forms are linear in the second
+    argument but the first argument involves a complex conjugate in the sense
+    that:
+
+                a(alpha u, beta v) = conj(alpha) beta a(u, v)
+
+    The @a convention argument in the class's constructor is documented in the
+    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
+
+    When supplying integrators to the MixedSesquilinearForm either the real or
+    imaginary integrator can be NULL. This indicates that the corresponding
+    portion of the complex-valued material coefficient is equal to zero.
+*/
+class MixedSesquilinearForm
+{
+private:
+   ComplexOperator::Convention conv;
+
+   MixedBilinearForm * mblfr;
+   MixedBilinearForm * mblfi;
+
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
+   bool RealInteg();
+   bool ImagInteg();
+
+public:
+   MixedSesquilinearForm(
+      FiniteElementSpace * trial_fes,
+      FiniteElementSpace * test_fes,
+      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
+
+   /** @brief Create a MixedSesquilinearForm on the given trial and test
+       FiniteElementSpaces, using the same integrators as the
+       MixedBilinearForms @a bfr and @a bfi.
+
+       The FiniteElementSpace pointers are not owned by the newly constructed
+       object.
+
+       The integrators are copied as pointers and they are not owned by the
+       newly constructed MixedSesquilinearForm. */
+   MixedSesquilinearForm(
+      FiniteElementSpace * trial_fes,
+      FiniteElementSpace * test_fes,
+      MixedBilinearForm * bfr,
+      MixedBilinearForm * bfi,
+      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
+
+   ComplexOperator::Convention GetConvention() const { return conv; }
+   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
+
+   /// Set the desired assembly level.
+   /** Valid choices are:
+
+       - AssemblyLevel::LEGACY (default)
+       - AssemblyLevel::FULL
+       - AssemblyLevel::PARTIAL
+       - AssemblyLevel::ELEMENT
+       - AssemblyLevel::NONE
+
+       This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level)
+   {
+      mblfr->SetAssemblyLevel(assembly_level);
+      mblfi->SetAssemblyLevel(assembly_level);
+   }
+
+   MixedBilinearForm & real() { return *mblfr; }
+   MixedBilinearForm & imag() { return *mblfi; }
+   const MixedBilinearForm & real() const { return *mblfr; }
+   const MixedBilinearForm & imag() const { return *mblfi; }
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                            BilinearFormIntegrator * bfi_imag);
+
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                            BilinearFormIntegrator * bfi_imag,
+                            Array<int> & elem_marker);
+
+   /// Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                              BilinearFormIntegrator * bfi_imag);
+
+   /** @brief Adds new boundary Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The mfem::array @a bdr_marker is stored internally as a pointer to the given
+       mfem::Array<int> object. */
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                              BilinearFormIntegrator * bfi_imag,
+                              Array<int> & bdr_marker);
+
+   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
+   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag);
+
+   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
+   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                             BilinearFormIntegrator * bfi_imag);
+
+   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The mfem::array @a bdr_marker is stored internally as a pointer to the given
+       mfem::Array<int> object. */
+   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                             BilinearFormIntegrator * bfi_imag,
+                             Array<int> & bdr_marker);
+
+   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
+
+       This type of integrator assembles terms over all faces of the mesh using
+       the face FE from the trial space and the two adjacent volume FEs from
+       the test space. */
+   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                               BilinearFormIntegrator * bfi_imag);
+
+   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag);
+
+   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag,
+                                  Array<int> &bdr_marker);
+
+   /// Assemble the local matrix
+   void Assemble(int skip_zeros = 1);
+
+   /// Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Updates the internal mixed forms with the new finite element space.
+   virtual void Update();
+
+   /** @brief Return a ComplexSparseMatrix wrapping the local (L-dof) real
+       and imaginary matrices of the form.
+
+       The returned wrapper has to be deleted by the caller, but it does not
+       own the wrapped real and imaginary matrices, which remain owned by
+       this form. */
+   ComplexSparseMatrix *AssembleComplexSparseMatrix();
+
+   /// Return the trial FE space associated with the MixedSesquilinearForm.
+   FiniteElementSpace *TrialFESpace() { return mblfr->TrialFESpace(); }
+
+   /// Read-only access to the associated trial FiniteElementSpace.
+   const FiniteElementSpace *TrialFESpace() const { return mblfr->TrialFESpace(); }
+
+   /// Return the test FE space associated with the MixedSesquilinearForm.
+   FiniteElementSpace *TestFESpace() { return mblfr->TestFESpace(); }
+
+   /// Read-only access to the associated test FiniteElementSpace.
+   const FiniteElementSpace *TestFESpace() const { return mblfr->TestFESpace(); }
+
+
+   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
+                                    const Array<int> & ess_test_tdof_list,
+                                    Vector & x,
+                                    Vector & b,
+                                    OperatorHandle & A,
+                                    Vector & X,
+                                    Vector & B);
+
+   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
+                                    const Array<int> & ess_test_tdof_list,
+                                    OperatorHandle & A);
+
+   virtual ~MixedSesquilinearForm();
+};
+
 #ifdef MFEM_USE_MPI
 
 /// Class for parallel complex-valued grid function - real + imaginary part
@@ -852,6 +1101,169 @@ public:
    virtual ~ParSesquilinearForm();
 };
 
+/** Class for a parallel mixed sesquilinear form
+
+    A mixed sesquilinear form is a generalization of a mixed bilinear form to
+    complex-valued fields. Mixed sesquilinear forms are linear in the second
+    argument but the first argument involves a complex conjugate in the sense
+    that:
+
+                a(alpha u, beta v) = conj(alpha) beta a(u, v)
+
+    The @a convention argument in the class's constructor is documented in the
+    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
+
+    When supplying integrators to the ParMixedSesquilinearForm either the real
+    or imaginary integrator can be NULL. This indicates that the corresponding
+    portion of the complex-valued material coefficient is equal to zero.
+*/
+class ParMixedSesquilinearForm
+{
+private:
+   ComplexOperator::Convention conv;
+
+   ParMixedBilinearForm * pmblfr;
+   ParMixedBilinearForm * pmblfi;
+
+   /* These methods check if the real/imag parts of the sesqulinear form are
+    not empty */
+   bool RealInteg();
+   bool ImagInteg();
+
+public:
+   ParMixedSesquilinearForm(
+      ParFiniteElementSpace * trial_fes,
+      ParFiniteElementSpace * test_fes,
+      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
+
+   /** @brief Create a ParMixedSesquilinearForm on the given trial and test
+       ParFiniteElementSpaces, using the same integrators as the
+       ParMixedBilinearForms @a pbfr and @a pbfi.
+
+       The ParFiniteElementSpace pointers are not owned by the newly
+       constructed object.
+
+       The integrators are copied as pointers and they are not owned by the
+       newly constructed ParMixedSesquilinearForm. */
+   ParMixedSesquilinearForm(
+      ParFiniteElementSpace * trial_fes,
+      ParFiniteElementSpace * test_fes,
+      ParMixedBilinearForm * pbfr,
+      ParMixedBilinearForm * pbfi,
+      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
+
+   ComplexOperator::Convention GetConvention() const { return conv; }
+   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
+
+   /// Set the desired assembly level.
+   /** Valid choices are:
+
+       - AssemblyLevel::LEGACY (default)
+       - AssemblyLevel::FULL
+       - AssemblyLevel::PARTIAL
+       - AssemblyLevel::ELEMENT
+       - AssemblyLevel::NONE
+
+       This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level)
+   {
+      pmblfr->SetAssemblyLevel(assembly_level);
+      pmblfi->SetAssemblyLevel(assembly_level);
+   }
+
+   ParMixedBilinearForm & real() { return *pmblfr; }
+   ParMixedBilinearForm & imag() { return *pmblfi; }
+   const ParMixedBilinearForm & real() const { return *pmblfr; }
+   const ParMixedBilinearForm & imag() const { return *pmblfi; }
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                            BilinearFormIntegrator * bfi_imag);
+
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                            BilinearFormIntegrator * bfi_imag,
+                            Array<int> & elem_marker);
+
+   /// Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                              BilinearFormIntegrator * bfi_imag);
+
+   /** @brief Adds new boundary Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The mfem::array @a bdr_marker is stored internally as a pointer to the given
+       mfem::Array<int> object. */
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                              BilinearFormIntegrator * bfi_imag,
+                              Array<int> & bdr_marker);
+
+   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
+   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag);
+
+   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
+   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                             BilinearFormIntegrator * bfi_imag);
+
+   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The mfem::array @a bdr_marker is stored internally as a pointer to the given
+       mfem::Array<int> object. */
+   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                             BilinearFormIntegrator * bfi_imag,
+                             Array<int> & bdr_marker);
+
+   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
+
+      This type of integrator assembles terms over all faces of the mesh using
+      the face FE from the trial space and the two adjacent volume FEs from
+      the test space. */
+   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                               BilinearFormIntegrator * bfi_imag);
+
+   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag);
+
+   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag,
+                                  Array<int> &bdr_marker);
+
+   /// Assemble the local matrix
+   void Assemble(int skip_zeros = 1);
+
+   /// Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Updates the internal mixed forms with the new finite element space.
+   virtual void Update();
+
+   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   /** The returned matrix has to be deleted by the caller. */
+   ComplexHypreParMatrix * ParallelAssemble();
+
+   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
+                                    const Array<int> & ess_test_tdof_list,
+                                    Vector & x,
+                                    Vector & b,
+                                    OperatorHandle & A,
+                                    Vector & X,
+                                    Vector & B);
+
+   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
+                                    const Array<int> & ess_test_tdof_list,
+                                    OperatorHandle & A);
+
+   virtual ~ParMixedSesquilinearForm();
+};
+
 #endif // MFEM_USE_MPI
 
 }

fem/datacollection.cpp

@@ -809,7 +809,7 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 
 void ParaViewDataCollectionBase::SetLevelsOfDetail(int levels_of_detail_)
 {
-   levels_of_detail = levels_of_detail_;
+   levels_of_detail = std::max(levels_of_detail_, 1);
 }
 
 void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)
@@ -1181,12 +1181,14 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
    DenseMatrix vval, pmat;
    std::vector<char> buf;
    int vec_dim = it->second->VectorDim();
+   int map_type = it->second->FESpace()->GetTypicalFE()->GetMapType();
    os << "<DataArray type=\"" << GetDataTypeString()
       << "\" Name=\"" << it->first
       << "\" NumberOfComponents=\"" << vec_dim << "\" "
       << VTKComponentLabels(vec_dim) << " "
       << "format=\"" << GetDataFormatString() << "\" >" << '\n';
-   if (vec_dim == 1)
+   if (vec_dim == 1 && (map_type == FiniteElement::VALUE ||
+                        map_type == FiniteElement::INTEGRAL))
    {
       for (int i = 0; i < mesh->GetNE(); i++)
       {

fem/fe/fe_fixed_order.hpp

@@ -307,12 +307,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (4) */
+       them in the vector shape of dimension Dof (6) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (6 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
@@ -336,12 +336,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (4) */
+       them in the vector shape of dimension Dof (5) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (5 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;

fem/fe/fe_h1.cpp

@@ -1757,22 +1757,45 @@ H1_BergotPyramidElement::H1_BergotPyramidElement(const int p, const int btype)
       real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       real_t z = ip.z;
 
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
-
-      o = 0;
-      for (int i = 0; i <= p; i++)
+      if (std::abs(z - 1.0) < apex_tol)
       {
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-
-            for (int k = 0; k <= p - maxij; k++)
+         // Compute the limit of the basis functions as z->1 with x and y on the
+         // line between the center of the base and the apex
+         o = 0;
+         for (int i = 0; i <= p; i++)
+            for (int j = 0; j <= p; j++)
             {
-               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                           pow(1.0 - ip.z, maxij);
+               int maxij = std::max(i, j);
+               for (int k = 0; k <= p - maxij; k++)
+                  if (i == 0 && j == 0)
+                  {
+                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
+                  }
+                  else
+                  {
+                     T(o++, m) = 0.;
+                  }
+            }
+      }
+      else
+      {
+         poly1d.CalcLegendre(p, x, shape_x.GetData());
+         poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+         o = 0;
+         for (int i = 0; i <= p; i++)
+         {
+            for (int j = 0; j <= p; j++)
+            {
+               int maxij = std::max(i, j);
+               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                                z, 1.0, shape_z);
+
+               for (int k = 0; k <= p - maxij; k++)
+               {
+                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                              pow(1.0 - ip.z, maxij);
+               }
             }
          }
       }
@@ -1793,25 +1816,44 @@ void H1_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    Vector u(dof);
 #endif
 
-   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   real_t z = ip.z;
+   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   const real_t z = ip.z;
 
-   poly1d.CalcLegendre(p, x, shape_x.GetData());
-   poly1d.CalcLegendre(p, y, shape_y.GetData());
-
-   int o = 0;
-   for (int i = 0; i <= p; i++)
-      for (int j = 0; j <= p; j++)
-      {
-         int maxij = std::max(i, j);
-         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
-                                          shape_z);
-         for (int k = 0; k <= p - maxij; k++)
-            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                     pow(1.0 - ip.z, maxij);
-      }
+   if (std::abs(z - 1.0) < apex_tol)
+   {
+      // Compute the limit of the basis functions as z->1 with x and y on the
+      // line between the center of the base and the apex
+      u = 0.;
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            for (int k = 0; k <= p - maxij; k++, o++)
+               if (i == 0 && j == 0)
+               {
+                  u(o) = ((k + 3.) * k + 2.) / 2.;
+               }
+         }
+   }
+   else
+   {
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
+            for (int k = 0; k <= p - maxij; k++)
+               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                        pow(1.0 - ip.z, maxij);
+         }
+   }
    Ti.Mult(u, shape);
 }
 
@@ -1830,37 +1872,68 @@ void H1_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    Vector dshape_z(order+1);
    Vector dshape_z_dt(order+1);
 #endif
-   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   real_t z = ip.z;
+   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   const real_t z = ip.z;
 
-   poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-   poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
-
-   int o = 0;
-   for (int i = 0; i <= p; i++)
-      for (int j = 0; j <= p; j++)
-      {
-         int maxij = std::max(i, j);
-         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
-                                          shape_z, dshape_z, dshape_z_dt);
-
-         for (int k = 0; k <= p - maxij; k++, o++)
+   if (std::abs(z - 1.0) < apex_tol)
+   {
+      // Compute the limit of the gradients of the basis functions as
+      // z->1 with x and y on the line between the center of the base and the
+      // apex
+      du = 0.;
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
          {
-            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
-            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
-            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                      pow(1.0 - ip.z, maxij) +
-                      (ip.x * dshape_x(i) * shape_y(j) +
-                       ip.y * shape_x(i) * dshape_y(j)) *
-                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                      maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
+            int maxij = std::max(i, j);
+            for (int k = 0; k <= p - maxij; k++, o++)
+            {
+               if (i == 0 && j == 0)
+               {
+                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
+               }
+               else if (i == 1 && j == 0)
+               {
+                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
+               }
+               else if (i == 0 && j == 1)
+               {
+                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
+               }
+            }
          }
-      }
+   }
+   else
+   {
+      poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0,
+                                             shape_z, dshape_z, dshape_z_dt);
+
+            for (int k = 0; k <= p - maxij; k++, o++)
+            {
+               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                         pow(1.0 - ip.z, maxij - 1);
+               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                         pow(1.0 - ip.z, maxij - 1);
+               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                         pow(1.0 - ip.z, maxij) +
+                         (ip.x * dshape_x(i) * shape_y(j) +
+                          ip.y * shape_x(i) * dshape_y(j)) *
+                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
+            }
+         }
+   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_h1.hpp

@@ -208,6 +208,8 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
+   static constexpr real_t apex_tol = 1e-8;
+
 public:
    H1_BergotPyramidElement(const int p,
                            const int btype = BasisType::GaussLobatto);

fem/fe/fe_l2.cpp

@@ -1106,9 +1106,16 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
          {
             const real_t wik = op[i] + op[k] + op[p-i-k];
             const real_t w = wik * wjk * op[p-k];
-            Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
-                                     op[j] * (op[j] + op[p-j-k]) / w,
-                                     op[k] * op[p-k] / w);
+            if (std::abs(w) < apex_tol)
+            {
+               Nodes.IntPoint(o++).Set3(0.,0.,1.);
+            }
+            else
+            {
+               Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
+                                        op[j] * (op[i] + op[p-i-k]) / w,
+                                        op[k] * op[p-k] / w);
+            }
          }
       }
 
@@ -1125,22 +1132,45 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
       const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       const real_t z = ip.z;
 
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
-
-      o = 0;
-      for (int i = 0; i <= p; i++)
+      if (std::abs(z - 1.0) < apex_tol)
       {
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-
-            for (int k = 0; k <= p - maxij; k++)
+         // Compute the limit of the basis functions as z->1 with x and y on the
+         // line between the center of the base and the apex
+         o = 0;
+         for (int i = 0; i <= p; i++)
+            for (int j = 0; j <= p; j++)
             {
-               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                           pow(1.0 - ip.z, maxij);
+               int maxij = std::max(i, j);
+               for (int k = 0; k <= p - maxij; k++)
+                  if (i == 0 && j == 0)
+                  {
+                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
+                  }
+                  else
+                  {
+                     T(o++, m) = 0.;
+                  }
+            }
+      }
+      else
+      {
+         poly1d.CalcLegendre(p, x, shape_x.GetData());
+         poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+         o = 0;
+         for (int i = 0; i <= p; i++)
+         {
+            for (int j = 0; j <= p; j++)
+            {
+               int maxij = std::max(i, j);
+               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                                z, 1.0, shape_z);
+
+               for (int k = 0; k <= p - maxij; k++)
+               {
+                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                              pow(1.0 - ip.z, maxij);
+               }
             }
          }
       }
@@ -1165,26 +1195,41 @@ void L2_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   poly1d.CalcLegendre(p, x, shape_x.GetData());
-   poly1d.CalcLegendre(p, y, shape_y.GetData());
-
-   int o = 0;
-   for (int i = 0; i <= p; i++)
+   if (std::abs(z - 1.0) < apex_tol)
    {
-      for (int j = 0; j <= p; j++)
-      {
-         int maxij = std::max(i, j);
-         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
-                                          shape_z);
-
-         for (int k = 0; k <= p - maxij; k++)
+      // Compute the limit of the basis functions as z->1 with x and y on the
+      // line between the center of the base and the apex
+      u = 0.;
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
          {
-            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                     pow(1.0 - ip.z, maxij);
+            int maxij = std::max(i, j);
+            for (int k = 0; k <= p - maxij; k++, o++)
+               if (i == 0 && j == 0)
+               {
+                  u(o) = ((k + 3.) * k + 2.) / 2.;
+               }
          }
-      }
    }
+   else
+   {
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
+
+            for (int k = 0; k <= p - maxij; k++)
+               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                        pow(1.0 - ip.z, maxij);
+         }
+   }
    Ti.Mult(u, shape);
 }
 
@@ -1208,35 +1253,64 @@ void L2_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-   Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
-
-   int o = 0;
-   for (int i = 0; i <= p; i++)
+   if (std::abs(z - 1.0) < apex_tol)
    {
-      for (int j = 0; j <= p; j++)
-      {
-         int maxij = std::max(i, j);
-         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
-                                          shape_z, dshape_z, dshape_z_dt);
-
-         for (int k = 0; k <= p - maxij; k++, o++)
+      // Compute the limit of the gradients of the basis functions as
+      // z->1 with x and y on the line between the center of the base and the
+      // apex
+      du = 0.;
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
          {
-            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
-            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
-            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                      pow(1.0 - ip.z, maxij) +
-                      (ip.x * dshape_x(i) * shape_y(j) +
-                       ip.y * shape_x(i) * dshape_y(j)) *
-                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                      ((maxij > 0) ? (maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                                      pow(1.0 - ip.z, maxij - 1)) : 0.0);
+            int maxij = std::max(i, j);
+            for (int k = 0; k <= p - maxij; k++, o++)
+            {
+               if (i == 0 && j == 0)
+               {
+                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
+               }
+               else if (i == 1 && j == 0)
+               {
+                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
+               }
+               else if (i == 0 && j == 1)
+               {
+                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
+               }
+            }
          }
-      }
    }
+   else
+   {
+      Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+      Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0,
+                                             shape_z, dshape_z, dshape_z_dt);
+
+            for (int k = 0; k <= p - maxij; k++, o++)
+            {
+               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                         pow(1.0 - ip.z, maxij - 1);
+               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                         pow(1.0 - ip.z, maxij - 1);
+               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                         pow(1.0 - ip.z, maxij) +
+                         (ip.x * dshape_x(i) * shape_y(j) +
+                          ip.y * shape_x(i) * dshape_y(j)) *
+                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
+            }
+         }
+   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_l2.hpp

@@ -225,6 +225,8 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
+   static constexpr real_t apex_tol = 1e-8;
+
 public:
    /// Construct the L2_PyramidElement of order @a p and BasisType @a btype
    L2_BergotPyramidElement(const int p,

fem/fe/fe_nd.cpp

@@ -1282,12 +1282,49 @@ ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
    }
 }
 
+void ND_SegmentElement::CalcShape(const IntegrationPoint &ip,
+                                  Vector &shape) const
+{
+   if (obasis1d.IsIntegratedType()) { obasis1d.ScaleIntegrated(false); }
+   obasis1d.Eval(ip.x, shape);
+}
+
 void ND_SegmentElement::CalcVShape(const IntegrationPoint &ip,
                                    DenseMatrix &shape) const
 {
    Vector vshape(shape.Data(), dof);
 
-   obasis1d.Eval(ip.x, vshape);
+   CalcShape(ip, vshape);
+}
+
+void ND_SegmentElement::ProjectIntegrated(VectorCoefficient &vc,
+                                          ElementTransformation &Trans,
+                                          Vector &dofs) const
+{
+   MFEM_ASSERT(obasis1d.IsIntegratedType(), "Not integrated type");
+   real_t vk[Geometry::MaxDim];
+   Vector xk(vk, vc.GetVDim());
+
+   const real_t *cp = poly1d.ClosedPoints(dof, BasisType::GaussLobatto);
+   const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, dof);
+   IntegrationPoint ip;
+
+   for (int i = 0; i < dof; i++)
+   {
+      const real_t h = cp[i+1] - cp[i];
+      real_t val = 0.0;
+
+      for (int q = 0; q < ir.GetNPoints(); q++)
+      {
+         const IntegrationPoint &ip1d = ir.IntPoint(q);
+         ip.x = cp[i] + h*ip1d.x;
+         Trans.SetIntPoint(&ip);
+         vc.Eval(xk, Trans, ip);
+         val += ip1d.weight*Trans.Jacobian().InnerProduct(tk, vk);
+      }
+
+      dofs(i) = val*h;
+   }
 }
 
 const real_t ND_WedgeElement::tk[15] =

fem/fe/fe_nd.hpp

@@ -303,8 +303,7 @@ public:
    /** @brief Construct the ND_SegmentElement of order @a p and open
        BasisType @a ob_type */
    ND_SegmentElement(const int p, const int ob_type = BasisType::GaussLegendre);
-   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override
-   { obasis1d.Eval(ip.x, shape); }
+   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
    void CalcVShape(ElementTransformation &Trans,
@@ -325,7 +324,10 @@ public:
    using FiniteElement::Project;
    void Project(VectorCoefficient &vc,
                 ElementTransformation &Trans, Vector &dofs) const override
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   {
+      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
+      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   }
    void ProjectMatrixCoefficient(MatrixCoefficient &mc,
                                  ElementTransformation &T,
                                  Vector &dofs) const override
@@ -338,6 +340,11 @@ public:
                     ElementTransformation &Trans,
                     DenseMatrix &grad) const override
    { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
+
+protected:
+   void ProjectIntegrated(VectorCoefficient &vc,
+                          ElementTransformation &Trans,
+                          Vector &dofs) const;
 };
 
 class ND_WedgeElement : public VectorFiniteElement

fem/fe/fe_pyramid.hpp

@@ -17,6 +17,12 @@
 namespace mfem
 {
 
+struct ScalarPyramid
+{
+   // Default basis type for H1 and L2 pyramids
+   static inline int DefaultType = 1; // Bergot(0) or Fuentes(1)
+};
+
 /** Base class for arbitrary order basis functions on pyramid-shaped elements
 
   This base class provides a common class to store temporary vectors,

fem/fe_coll.cpp

@@ -228,7 +228,19 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1_", 3))
    {
-      fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
+      // Parse pyramid basis type if included in the name
+      const char *pyr = strstr(name, "Pyr");
+      if (pyr == NULL)
+      {
+         // Use default pyramid type elements
+         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
+      }
+      else
+      {
+         // Use specific pyramid type elements
+         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3),
+                                   BasisType::GaussLobatto, atoi(pyr + 3));
+      }
    }
    else if (!strncmp(name, "H1Pos_Trace_", 12))
    {
@@ -245,26 +257,44 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1@", 3))
    {
-      fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                BasisType::GetType(name[3]));
+      // Parse pyramid basis type if included in the name
+      const char *pyr = strstr(name, "Pyr");
+      if (pyr == NULL)
+      {
+         // Use default pyramid type elements
+         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                   BasisType::GetType(name[3]));
+      }
+      else
+      {
+         // Use specific pyramid type elements
+         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                   BasisType::GetType(name[3]),
+                                   atoi(pyr + 3));
+      }
    }
-   else if (!strncmp(name, "L2_T", 4))
-      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
-                                atoi(name + 4));
-   else if (!strncmp(name, "L2_", 3))
+   else if (!strncmp(name, "L2", 2))
    {
-      fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));
-   }
-   else if (!strncmp(name, "L2Int_T", 7))
-   {
-      fec = new L2_FECollection(atoi(name + 13), atoi(name + 9),
-                                atoi(name + 7), FiniteElement::INTEGRAL);
-   }
-   else if (!strncmp(name, "L2Int_", 6))
-   {
-      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
-                                BasisType::GaussLegendre,
-                                FiniteElement::INTEGRAL);
+      // Parse Map Type
+      const int mtype = strstr(name, "Int") == NULL ?
+                        FiniteElement::VALUE : FiniteElement::INTEGRAL;
+
+      // Parse the base order
+      const int p = atoi(strstr(name, "_P") + 2);
+
+      // Parse the mesh dimension
+      const int dim = atoi(strstr(name, "D") - 1);
+
+      // Parse basis type if specified
+      const char *t = strstr(name, "_T");
+      const int btype = t == NULL ? BasisType::GaussLegendre : atoi(t + 2);
+
+      // Parse the pyramid type if specified
+      const char *pyr = strstr(name, "Pyr");
+      const int ptype = pyr == NULL ? 1 : atoi(pyr + 3);
+
+      // Create collection
+      fec = new L2_FECollection(p, dim, btype, mtype, ptype);
    }
    else if (!strncmp(name, "RT_Trace_", 9))
    {
@@ -1709,9 +1739,10 @@ const int *RT1_3DFECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 
 H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
-                                 const int pyrtype)
+                                 const int pyr_type)
    : FiniteElementCollection(p)
    , dim(dim)
+   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 1, "H1_FECollection requires order >= 1.");
    MFEM_VERIFY(dim >= 0 && dim <= 3, "H1_FECollection requires 0 <= dim <= 3.");
@@ -1724,7 +1755,14 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
    {
       case BasisType::GaussLobatto:
       {
-         snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
+         if (pyr_type == ScalarPyramid::DefaultType)
+         {
+            snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
+         }
+         else
+         {
+            snprintf(h1_name, 32, "H1_%dD_P%d_Pyr%d", dim, p, pyr_type);
+         }
          break;
       }
       case BasisType::Positive:
@@ -1910,11 +1948,11 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
-         if (pyrtype == 0 || b_type == BasisType::Positive)
+         if (pyr_type == 0 || b_type == BasisType::Positive)
          {
             H1_dof[Geometry::PYRAMID] = pm2*pm1*(2*p-3)/6; // Bergot (JSC)
          }
-         else if (pyrtype == 1)
+         else if (pyr_type == 1)
          {
             H1_dof[Geometry::PYRAMID] = pm1*pm1*pm1; // Fuentes
          }
@@ -1935,13 +1973,15 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
                new H1_TetrahedronElement(p, btype);
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
-            if (pyrtype == 0)
+            if (pyr_type == 0)
             {
-               H1_Elements[Geometry::PYRAMID] = new H1_BergotPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] =
+                  new H1_BergotPyramidElement(p, btype);
             }
             else
             {
-               H1_Elements[Geometry::PYRAMID] = new H1_FuentesPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] =
+                  new H1_FuentesPyramidElement(p, btype);
             }
          }
 
@@ -2148,6 +2188,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    : FiniteElementCollection(p)
    , dim(dim)
    , m_type(map_type)
+   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 0, "L2_FECollection requires order >= 0.");
 
@@ -2163,10 +2204,25 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    switch (btype)
    {
       case BasisType::GaussLegendre:
-         snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
+         if (pyr_type == ScalarPyramid::DefaultType)
+         {
+            snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
+         }
+         else
+         {
+            snprintf(d_name, 32, "%s_%dD_P%d_Pyr%d", prefix, dim, p, pyr_type);
+         }
          break;
       default:
-         snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
+         if (pyr_type == ScalarPyramid::DefaultType)
+         {
+            snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
+         }
+         else
+         {
+            snprintf(d_name, 32, "%s_T%d_%dD_P%d_Pyr%d",
+                     prefix, btype, dim, p, pyr_type);
+         }
    }
 
    for (int g = 0; g < Geometry::NumGeom; g++)
@@ -2285,11 +2341,13 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
          if (pyr_type == 0)
          {
-            L2_Elements[Geometry::PYRAMID] = new L2_BergotPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] =
+               new L2_BergotPyramidElement(p, btype);
          }
          else
          {
-            L2_Elements[Geometry::PYRAMID] = new L2_FuentesPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] =
+               new L2_FuentesPyramidElement(p, btype);
          }
       }
 

fem/fe_coll.hpp

@@ -100,6 +100,10 @@ public:
       return FiniteElementForGeometry(GeomType);
    }
 
+   /** @brief Returns a collection of the trace elements.
+
+       @note The collection is owned by the caller and is NOT deleted in the
+       destructor. */
    virtual FiniteElementCollection *GetTraceCollection() const;
 
    virtual ~FiniteElementCollection();
@@ -286,7 +290,7 @@ protected:
 class H1_FECollection : public FiniteElementCollection
 {
 protected:
-   int dim, b_type;
+   int dim, b_type, p_type;
    char h1_name[32];
    FiniteElement *H1_Elements[Geometry::NumGeom];
    int H1_dof[Geometry::NumGeom];
@@ -295,7 +299,7 @@ protected:
 public:
    explicit H1_FECollection(const int p, const int dim = 3,
                             const int btype = BasisType::GaussLobatto,
-                            const int pyrtype = 1);
+                            const int pyr_type = ScalarPyramid::DefaultType);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -320,7 +324,7 @@ public:
    const int *GetDofMap(Geometry::Type GeomType, int p) const;
 
    FiniteElementCollection *Clone(int p) const override
-   { return new H1_FECollection(p, dim, b_type); }
+   { return new H1_FECollection(p, dim, b_type, p_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }
@@ -367,6 +371,7 @@ private:
    int dim;
    int b_type; // BasisType
    int m_type; // map type
+   int p_type; // Pyramid type (0 -> Bergot, 1 -> Fuentes)
    char d_name[32];
    ScalarFiniteElement *L2_Elements[Geometry::NumGeom];
    ScalarFiniteElement *Tr_Elements[Geometry::NumGeom];
@@ -379,7 +384,7 @@ public:
    L2_FECollection(const int p, const int dim,
                    const int btype = BasisType::GaussLegendre,
                    const int map_type = FiniteElement::VALUE,
-                   const int pyrtype = 1);
+                   const int pyr_type = ScalarPyramid::DefaultType);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -409,7 +414,7 @@ public:
    int GetBasisType() const { return b_type; }
 
    FiniteElementCollection *Clone(int p) const override
-   { return new L2_FECollection(p, dim, b_type, m_type); }
+   { return new L2_FECollection(p, dim, b_type, m_type, p_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }

fem/gslib.cpp

@@ -76,6 +76,9 @@ namespace gslib
 #ifndef GSLIB_RELEASE_VERSION //gslib v1.0.7
 #define GSLIB_RELEASE_VERSION 10007
 #endif
+static_assert(std::is_same_v<uint,unsigned int>,
+              "GSLIB's integer-type, 'uint', defined in gslib.h, must be the same as 'unsigned int'!");
+
 extern "C" {
    struct hash_data_3
    {

fem/gslib.hpp

@@ -326,22 +326,22 @@ protected:
    void findptsedge_setup_2(DEV_STRUCT &devs,
                             const double *const elx[2],
                             const unsigned n,
-                            const uint nel,
+                            const unsigned int nel,
                             const unsigned m,
                             const double bbox_rel_size_inc,
-                            const uint local_hash_size,
-                            const uint global_hash_size,
+                            const unsigned int local_hash_size,
+                            const unsigned int global_hash_size,
                             const Vector *aabb_sz_inc);
 
    /// Preprocess 3D surface mesh needed for FindPoints.
    void findptssurf_setup_3(DEV_STRUCT &devs,
                             const double *const elx[3],
                             const unsigned n,
-                            const uint nel,
+                            const unsigned int nel,
                             const unsigned m,
                             const double bbox_rel_size_inc,
-                            const uint local_hash_size,
-                            const uint global_hash_size,
+                            const unsigned int local_hash_size,
+                            const unsigned int global_hash_size,
                             const int rD,
                             const Vector *aabb_sz_inc);
 

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -147,18 +147,16 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply2D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y)
+void PAHcurlMassApply2D(const int NE, const bool symmetric,
+                        [[maybe_unused]] const bool scalar_coeff,
+                        const Array<real_t> &bo, const Array<real_t> &bc,
+                        const Array<real_t> &bot, const Array<real_t> &bct,
+                        const Vector &pa_data, const Vector &x, Vector &y,
+                        const int D1D, [[maybe_unused]] const int TestD1D,
+                        const int Q1D)
 {
+   MFEM_ASSERT(D1D == TestD1D,
+               "Trial and Test space must have the same number of dofs");
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -277,18 +275,16 @@ void PAHcurlMassApply2D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply3D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y)
+void PAHcurlMassApply3D(const int NE, const bool symmetric,
+                        [[maybe_unused]] const bool scalar_coeff,
+                        const Array<real_t> &bo, const Array<real_t> &bc,
+                        const Array<real_t> &bot, const Array<real_t> &bct,
+                        const Vector &pa_data, const Vector &x, Vector &y,
+                        const int D1D, [[maybe_unused]] const int TestD1D,
+                        const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -181,228 +181,312 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
 }
 
 // PA H(curl) Mass Apply 2D kernel
-void PAHcurlMassApply2D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
+void PAHcurlMassApply2D(const int NE, const bool symmetric,
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int TrialD1D,
+                        const int TestD1D, const int Q1D);
 
 // PA H(curl) Mass Apply 3D kernel
-void PAHcurlMassApply3D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
+void PAHcurlMassApply3D(const int NE, const bool symmetric,
+                        [[maybe_unused]] const bool scalar_coeff,
+                        const Array<real_t> &bo, const Array<real_t> &bc,
+                        const Array<real_t> &bot, const Array<real_t> &bct,
+                        const Vector &pa_data, const Vector &x, Vector &y,
+                        const int TrialD1D, [[maybe_unused]] const int TestD1D,
+                        const int Q1D);
 
 // Shared memory PA H(curl) Mass Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHcurlMassApply3D(const int d1d,
-                                   const int q1d,
-                                   const int NE,
-                                   const bool symmetric,
-                                   const Array<real_t> &bo,
-                                   const Array<real_t> &bc,
-                                   const Array<real_t> &bot,
-                                   const Array<real_t> &bct,
-                                   const Vector &pa_data,
-                                   const Vector &x,
-                                   Vector &y)
+template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
+inline void SmemPAHcurlMassApply3D(
+   const int NE, const bool symmetric, [[maybe_unused]] const bool scalar_coeff,
+   const Array<real_t> &bo, const Array<real_t> &bc,
+   [[maybe_unused]] const Array<real_t> &bot,
+   [[maybe_unused]] const Array<real_t> &bct, const Vector &pa_data,
+   const Vector &x, Vector &y, const int d1d = 0,
+   [[maybe_unused]] const int test_d1d = 0, const int q1d = 0)
 {
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
    MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: d1d > HCURL_MAX_D1D");
    MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
                "Error: q1d > HCURL_MAX_Q1D");
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
+   MFEM_ASSERT(Q1D >= D1D, "Expected Q1D >= D1D");
    const int dataSize = symmetric ? 6 : 9;
 
-   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
-   auto Bc = Reshape(bc.Read(), Q1D, D1D);
-   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
-   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
-   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
+   // assume trial space == test space
+   auto Bo = bo.Read();
+   auto Bc = bc.Read();
+   auto op =
+      Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
+   auto X_ = Reshape(x.Read(), 3 * (D1D - 1) * D1D * D1D, NE);
+   auto y_ = y.ReadWrite();
 
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   constexpr int MD_ = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
+   constexpr int MQ_ = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+   constexpr int MDQ_ = std::max(MD_, MQ_);
+   constexpr int MB_ = TBATCH ? TBATCH : 1;
+
+   mfem::forall_2D_batch<MDQ_ * MDQ_ * MDQ_ * MB_>(
+      NE, MDQ_ * MDQ_ * MDQ_, 1, MB_, [=] MFEM_HOST_DEVICE(int e)
    {
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = TBATCH ? TBATCH : 1;
+      int tidz = MFEM_THREAD_ID(z);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
       constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MDQ = std::max(MD1D, MQ1D);
 
-      MFEM_SHARED real_t sBo[MQ1D][MD1D];
-      MFEM_SHARED real_t sBc[MQ1D][MD1D];
+      // nvcc limit work-around: can't have Y_ be captured first in
+      // if constexpr, so capture y_ and construct Y_ locally
+      // only works on GPU
+      auto Y = Reshape(y_, VDIM * (D1D - 1) * D1D * D1D, NE);
 
-      real_t op9[9];
-      MFEM_SHARED real_t sop[9*MQ1D*MQ1D];
-      MFEM_SHARED real_t mass[MQ1D][MQ1D][3];
+      MFEM_SHARED real_t sBo[MDQ * (MD1D - 1)];
+      MFEM_SHARED real_t sBc[MDQ * MD1D];
+      auto BO = Reshape(sBo, Q1D, D1D - 1);
+      auto BC = Reshape(sBc, Q1D, D1D);
 
-      MFEM_SHARED real_t sX[MD1D][MD1D][MD1D];
+      MFEM_SHARED real_t sX[nbz * VDIM * (MD1D - 1) * MD1D * MD1D];
+      MFEM_SHARED real_t sm0[nbz * VDIM * MDQ * MDQ * MDQ];
+      MFEM_SHARED real_t sm1[nbz * VDIM * MDQ * MDQ * MDQ];
 
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      real_t(*X)[nbz][(MD1D - 1) * MD1D * MD1D] =
+         (real_t(*)[nbz][(MD1D - 1) * MD1D * MD1D])(sX);
+      // shapes of buffers always use MQ1D to mitigate shared memory bank
+      // conflicts
+      real_t(*DDQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+      real_t(*DQQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
+      real_t(*QQQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+      real_t(*QQD)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
+      real_t(*QDD)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+
+      // load dofs into smem
+      const int offset = (D1D - 1) * D1D * D1D;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         for (int dim = 0; dim < VDIM; ++dim)
          {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               for (int i=0; i<dataSize; ++i)
-               {
-                  op9[i] = op(qx,qy,qz,i,e);
-               }
-            }
+            X[dim][tidz][ix] = X_(ix + dim * offset, e);
          }
       }
-
-      const int tidx = MFEM_THREAD_ID(x);
-      const int tidy = MFEM_THREAD_ID(y);
-      const int tidz = MFEM_THREAD_ID(z);
-
+      // load basis functions data
       if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(ix, x, D1D * Q1D) { sBc[ix] = Bc[ix]; }
+         MFEM_FOREACH_THREAD_DIRECT(ix, x, (D1D - 1) * Q1D)
          {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
+            sBo[ix] = Bo[ix];
+         }
+      }
+
+      for (int dim0 = 0; dim0 < VDIM; ++dim0)
+      {
+         MFEM_SYNC_THREAD;
+         // sum factor to QQQ = Q_{dim0,dim1} B X_{dim1}
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, Q1D, D1Dy, D1Dz,
+                                                 Q1D, Q1D, Q1D)
             {
-               sBc[q][d] = Bc(q,d);
-               if (d < D1D-1)
+               real_t u = 0;
+               for (int dx = 0; dx < D1Dx; ++dx)
                {
-                  sBo[q][d] = Bo(q,d);
+                  real_t b;
+                  if (dim1 == 0)
+                  {
+                     b = BO(qx, dx);
+                  }
+                  else
+                  {
+                     b = BC(qx, dx);
+                  }
+                  u += X[dim1][tidz][dx + (dy + dz * D1Dy) * D1Dx] * b;
+               }
+               DDQ[dim1][tidz][dz][dy][qx] = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, Q1D, Q1D, D1Dz,
+                                                 Q1D, Q1D, Q1D)
+            {
+               real_t u = 0;
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  real_t b;
+                  if (dim1 == 1)
+                  {
+                     b = BO(qy, dy);
+                  }
+                  else
+                  {
+                     b = BC(qy, dy);
+                  }
+                  u += DDQ[dim1][tidz][dz][dy][qx] * b;
+               }
+               DQQ[dim1][tidz][dz][qy][qx] = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            // const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            MFEM_FOREACH_THREAD_DIRECT_3D(qx, qy, qz, x, Q1D, Q1D, Q1D)
+            {
+               real_t u = 0;
+               for (int dz = 0; dz < D1Dz; ++dz)
+               {
+                  real_t b;
+                  if (dim1 == 2)
+                  {
+                     b = BO(qz, dz);
+                  }
+                  else
+                  {
+                     b = BC(qz, dz);
+                  }
+                  u += DQQ[dim1][tidz][dz][qy][qx] * b;
+               }
+               // pa_data is row major
+               int idx;
+               if (symmetric)
+               {
+                  int row;
+                  int col;
+                  if (dim0 > dim1)
+                  {
+                     row = dim1;
+                     col = dim0;
+                  }
+                  else
+                  {
+                     row = dim0;
+                     col = dim1;
+                  }
+                  idx = col + VDIM * row - row * (row + 1) / 2;
+               }
+               else
+               {
+                  idx = dim0 * VDIM + dim1;
+               }
+               QQQ[dim1][tidz][qz][qy][qx] = op(qx, qy, qz, idx, e) * u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         // sum factor back to Y
+         // Assume bot and bct == bo^t and bc^t respectively (i.e. test ==
+         // trial functions), skip loading them again.
+         {
+            const int D1Dz = (dim0 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim0 == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (dim0 == 0) ? D1D - 1 : D1D;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, D1Dz, Q1D, Q1D,
+                                                 Q1D, Q1D, Q1D)
+            {
+               for (int dim1 = 0; dim1 < VDIM; ++dim1)
+               {
+                  real_t u = 0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     real_t b = 0;
+                     if (dim0 == 2)
+                     {
+                        b = BO(qz, dz);
+                     }
+                     else
+                     {
+                        b = BC(qz, dz);
+                     }
+                     u += QQQ[dim1][tidz][qz][qy][qx] * b;
+                  }
+                  QQD[dim1][tidz][qy][qx][dz] = u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, D1Dy, D1Dz, Q1D,
+                                                 Q1D, Q1D, Q1D)
+            {
+               for (int dim1 = 0; dim1 < VDIM; ++dim1)
+               {
+                  real_t u = 0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     real_t b;
+                     if (dim0 == 1)
+                     {
+                        b = BO(qy, dy);
+                     }
+                     else
+                     {
+                        b = BC(qy, dy);
+                     }
+                     u += QQD[dim1][tidz][qy][qx][dz] * b;
+                  }
+                  QDD[dim1][tidz][qx][dz][dy] = u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT_3D(dx, dy, dz, x, D1Dx, D1Dy, D1Dz)
+            {
+               int ix = dx + D1Dx * (dy + D1Dy * dz);
+               real_t u = 0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  real_t b;
+                  if (dim0 == 0)
+                  {
+                     b = BO(qx, dx);
+                  }
+                  else
+                  {
+                     b = BC(qx, dx);
+                  }
+                  for (int dim1 = 0; dim1 < VDIM; ++dim1)
+                  {
+                     u += QDD[dim1][tidz][qx][dz][dy] * b;
+                  }
+               }
+               if constexpr (ACCUMULATE)
+               {
+                  Y(ix + dim0 * offset, e) += u;
+               }
+               else
+               {
+                  Y(ix + dim0 * offset, e) = u;
                }
             }
          }
       }
-      MFEM_SYNC_THREAD;
-
-      for (int qz=0; qz < Q1D; ++qz)
-      {
-         int osc = 0;
-         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
-         {
-            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     sX[dz][dy][dx] = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            if (tidz == qz)
-            {
-               for (int i=0; i<dataSize; ++i)
-               {
-                  sop[i + (dataSize*tidx) + (dataSize*Q1D*tidy)] = op9[i];
-               }
-
-               MFEM_FOREACH_THREAD(qy,y,Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx,x,Q1D)
-                  {
-                     real_t u = 0.0;
-
-                     for (int dz = 0; dz < D1Dz; ++dz)
-                     {
-                        const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
-                        for (int dy = 0; dy < D1Dy; ++dy)
-                        {
-                           const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
-                           for (int dx = 0; dx < D1Dx; ++dx)
-                           {
-                              const real_t t = sX[dz][dy][dx];
-                              const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
-                              u += t * wx * wy * wz;
-                           }
-                        }
-                     }
-
-                     mass[qy][qx][c] = u;
-                  } // qx
-               } // qy
-            } // tidz == qz
-
-            osc += D1Dx * D1Dy * D1Dz;
-            MFEM_SYNC_THREAD;
-         } // c
-
-         MFEM_SYNC_THREAD;  // Sync mass[qy][qx][d] and sop
-
-         osc = 0;
-         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
-         {
-            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
-
-            real_t dxyz = 0.0;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
-
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     for (int qy = 0; qy < Q1D; ++qy)
-                     {
-                        const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
-                        for (int qx = 0; qx < Q1D; ++qx)
-                        {
-                           const int os = (dataSize*qx) + (dataSize*Q1D*qy);
-                           const int id1 = os + ((c == 0) ? 0 : ((c == 1) ? (symmetric ? 1 : 3) :
-                                                                 (symmetric ? 2 : 6))); // O11, O21, O31
-                           const int id2 = os + ((c == 0) ? 1 : ((c == 1) ? (symmetric ? 3 : 4) :
-                                                                 (symmetric ? 4 : 7))); // O12, O22, O32
-                           const int id3 = os + ((c == 0) ? 2 : ((c == 1) ? (symmetric ? 4 : 5) :
-                                                                 (symmetric ? 5 : 8))); // O13, O23, O33
-
-                           const real_t m_c = (sop[id1] * mass[qy][qx][0]) + (sop[id2] * mass[qy][qx][1]) +
-                                              (sop[id3] * mass[qy][qx][2]);
-
-                           const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
-                           dxyz += m_c * wx * wy * wz;
-                        }
-                     }
-                  }
-               }
-            }
-
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += dxyz;
-                  }
-               }
-            }
-
-            osc += D1Dx * D1Dy * D1Dz;
-         } // c loop
-      } // qz
    }); // end of element loop
 }
 

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -62,6 +62,30 @@ void PAHcurlHdivMassApply2D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
+/// H(curl) test, H(div) trial
+inline void
+PAHcurlHdivMassApply2D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false,
+                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
+/// H(div) test, H(curl) trial
+inline void
+PAHdivHcurlMassApply2D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true,
+                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
 // PA H(curl)-H(div) Mass Apply 3D kernel
 void PAHcurlHdivMassApply3D(const int D1D,
                             const int D1Dtest,
@@ -78,6 +102,30 @@ void PAHcurlHdivMassApply3D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
+/// H(curl) test, H(div) trial
+inline void
+PAHcurlHdivMassApply3D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false, false, Bo_,
+                          Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
+/// H(div) test, H(curl) trial
+inline void
+PAHdivHcurlMassApply3D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true, false, Bo_,
+                          Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
 // PA H(curl)-H(div) Curl Apply 3D kernel
 template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
 inline void PAHcurlHdivApply3D(const int d1d,

fem/integ/bilininteg_hdiv_kernels.cpp

@@ -294,61 +294,14 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const bool symmetric,
-                     const Array<real_t> &Bo,
-                     const Array<real_t> &Bc,
-                     const Array<real_t> &Bot,
-                     const Array<real_t> &Bct,
-                     const Vector &op,
-                     const Vector &x,
-                     Vector &y)
-{
-   const int id = (D1D << 4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         default: // fallback
-            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         default: // fallback
-            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-      }
-   }
-}
-
-void PAHdivMassApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_)
+void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int TestD1D, const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -468,18 +421,14 @@ void PAHdivMassApply2D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply3D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_)
+void PAHdivMassApply3D(const int NE, const bool symmetric, const bool,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int TestD1D, const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
                "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -66,58 +66,29 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_);
 
-void PAHdivMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const bool symmetric,
-                     const Array<real_t> &Bo,
-                     const Array<real_t> &Bc,
-                     const Array<real_t> &Bot,
-                     const Array<real_t> &Bct,
-                     const Vector &op,
-                     const Vector &x,
-                     Vector &y);
-
 // PA H(div) Mass Apply 2D kernel
-void PAHdivMassApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
+void PAHdivMassApply2D(const int NE, const bool symmetric,
+                       const bool scalar_coeff, const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_, const Vector &op_,
+                       const Vector &x_, Vector &y_, const int D1D,
+                       const int TestD1D, const int Q1D);
 
 // PA H(div) Mass Apply 3D kernel
-void PAHdivMassApply3D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
+void PAHdivMassApply3D(const int NE, const bool symmetric,
+                       const bool scalar_coeff, const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_, const Vector &op_,
+                       const Vector &x_, Vector &y_, const int D1D,
+                       const int TestD1D, const int Q1D);
 
 // Shared memory PA H(div) Mass Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply2D(const int NE,
-                                  const bool symmetric,
-                                  const Array<real_t> &Bo_,
-                                  const Array<real_t> &Bc_,
-                                  const Array<real_t> &Bot_,
-                                  const Array<real_t> &Bct_,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(
+   const int NE, const bool symmetric, const bool, const Array<real_t> &Bo_,
+   const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+   const Array<real_t> &Bct_, const Vector &op_, const Vector &x_, Vector &y_,
+   const int d1d = 0, const int = 0, const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);
@@ -280,18 +251,13 @@ inline void SmemPAHdivMassApply2D(const int NE,
 }
 
 // Shared memory PA H(div) Mass Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply3D(const int NE,
-                                  const bool symmetric,
-                                  const Array<real_t> &Bo_,
-                                  const Array<real_t> &Bc_,
-                                  const Array<real_t> &Bot_,
-                                  const Array<real_t> &Bct_,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void
+SmemPAHdivMassApply3D(const int NE, const bool symmetric, const bool,
+                      const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                      const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                      const Vector &op_, const Vector &x_, Vector &y_,
+                      const int d1d = 0, const int = 0, const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);

fem/integ/bilininteg_vecdiv_pa.hpp

@@ -0,0 +1,365 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/vector.hpp"
+#include "../bilininteg.hpp"
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+namespace internal
+{
+
+// Shared memory PA Divergence Apply 2D kernel
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+inline void SmemPADivergenceApply2D(const int NE,
+                                    const Array<real_t> &b_,
+                                    const Array<real_t> &g_,
+                                    const Array<real_t> &bt_,
+                                    const Vector &q_,
+                                    const Vector &x_,
+                                    Vector &y_,
+                                    const int tr_d1d = 0,
+                                    const int te_d1d = 0,
+                                    const int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
+   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
+   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, 2, NE);
+   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, 1, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
+      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
+
+      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
+      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
+
+      kernels::internal::LoadDofs2d(e, TR_D1D, X, g0);
+      kernels::internal::Grad2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            r0[0][qy][qx] =
+               g1[0][0][qy][qx] * Q(qx, qy, 0, 0, e) +
+               g1[0][1][qy][qx] * Q(qx, qy, 1, 0, e) +
+               g1[1][0][qy][qx] * Q(qx, qy, 0, 1, e) +
+               g1[1][1][qy][qx] * Q(qx, qy, 1, 1, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::LoadMatrix<MQ1,true>(TE_D1D, Q1D, Bt, sB);
+      kernels::internal::EvalTranspose2d(TE_D1D, Q1D, smem, sB, r0, r1);
+      kernels::internal::WriteDofs2d(e, TE_D1D, r1, Y);
+   });
+}
+
+// Shared memory PA Divergence Apply 2D kernel transpose
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+inline void SmemPADivergenceApplyTranspose2D(const int NE,
+                                             const Array<real_t> &bt,
+                                             const Array<real_t> &gt,
+                                             const Array<real_t> &b,
+                                             const Vector &q_,
+                                             const Vector &x_,
+                                             Vector &y_,
+                                             const int tr_d1d = 0,
+                                             const int te_d1d = 0,
+                                             const int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
+   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
+   const auto X  = Reshape(x_.Read(), TE_D1D, TE_D1D, 1, NE);
+   auto Y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, 2, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
+      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
+
+      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
+      kernels::internal::LoadDofs2d(e, TE_D1D, X, r0);
+      kernels::internal::Eval2d(TE_D1D, Q1D, smem, sB, r0, r1);
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            g0[0][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 0, e);
+            g0[0][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 0, e);
+            g0[1][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 1, e);
+            g0[1][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 1, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
+      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
+      kernels::internal::GradTranspose2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
+      kernels::internal::WriteDofs2d(e, TR_D1D, g1, Y);
+   });
+}
+
+// Shared memory PA Divergence Apply 3D kernel transpose
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+inline void SmemPADivergenceApplyTranspose3D(const int NE,
+                                             const Array<real_t> &bt,
+                                             const Array<real_t> &gt,
+                                             const Array<real_t> &b,
+                                             const Vector &q_,
+                                             const Vector &x_,
+                                             Vector &y_,
+                                             int tr_d1d = 0,
+                                             int te_d1d = 0,
+                                             int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
+   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
+   const auto X = Reshape(x_.Read(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
+   auto Y = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
+      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
+
+      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
+      kernels::internal::LoadDofs3d(e, TE_D1D, X, r0);
+      kernels::internal::Eval3d(TE_D1D, Q1D, smem, sB, r0, r1);
+
+      for (int qz = 0; qz < Q1D; qz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               const auto r = r1[0][qz][qy][qx];
+               g0[0][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 0, e);
+               g0[0][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 0, e);
+               g0[0][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 0, e);
+
+               g0[1][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 1, e);
+               g0[1][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 1, e);
+               g0[1][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 1, e);
+
+               g0[2][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 2, e);
+               g0[2][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 2, e);
+               g0[2][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 2, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
+      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
+      kernels::internal::GradTranspose3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
+      kernels::internal::WriteDofs3d(e, TR_D1D, g1, Y);
+   });
+}
+
+// Shared memory PA Divergence Apply 3D kernel
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+inline void SmemPADivergenceApply3D(const int NE,
+                                    const Array<real_t> &b_,
+                                    const Array<real_t> &g_,
+                                    const Array<real_t> &bt_,
+                                    const Vector &q_,
+                                    const Vector &x_,
+                                    Vector &y_,
+                                    const int tr_d1d = 0,
+                                    const int te_d1d = 0,
+                                    const int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
+   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3,3, NE);
+   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
+   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
+
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
+      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
+
+      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
+      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
+
+      kernels::internal::LoadDofs3d(e, TR_D1D, X, g0);
+      kernels::internal::Grad3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
+
+      for (int qz = 0; qz < Q1D; qz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               r0[0][qz][qy][qx] =
+                  // c = 0
+                  g1[0][0][qz][qy][qx] * Q(qx, qy, qz, 0, 0, e) +
+                  g1[0][1][qz][qy][qx] * Q(qx, qy, qz, 1, 0, e) +
+                  g1[0][2][qz][qy][qx] * Q(qx, qy, qz, 2, 0, e) +
+                  // c = 1
+                  g1[1][0][qz][qy][qx] * Q(qx, qy, qz, 0, 1, e) +
+                  g1[1][1][qz][qy][qx] * Q(qx, qy, qz, 1, 1, e) +
+                  g1[1][2][qz][qy][qx] * Q(qx, qy, qz, 2, 1, e) +
+                  // c = 2
+                  g1[2][0][qz][qy][qx] * Q(qx, qy, qz, 0, 2, e) +
+                  g1[2][1][qz][qy][qx] * Q(qx, qy, qz, 1, 2, e) +
+                  g1[2][2][qz][qy][qx] * Q(qx, qy, qz, 2, 2, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::LoadMatrix<MQ1, true>(TE_D1D, Q1D, Bt, sB);
+      kernels::internal::EvalTranspose3d(TE_D1D, Q1D, smem, sB, r0, r1);
+      kernels::internal::WriteDofs3d(e, TE_D1D, r1, Y);
+   });
+}
+
+} // namespace internal
+
+template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
+VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
+VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Kernel()
+{
+   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
+   if constexpr (DIM == 2)
+   {
+      return internal::SmemPADivergenceApply2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::SmemPADivergenceApply3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+inline VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
+VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Fallback
+(int dim, int tr_d1d, int te_d1d, int q1d)
+{
+   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
+   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   if (dim == 2)
+   {
+      return internal::SmemPADivergenceApply2D;
+   }
+   else if (dim == 3)
+   {
+      return internal::SmemPADivergenceApply3D;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
+VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
+VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Kernel()
+{
+   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
+   if constexpr (DIM == 2)
+   {
+      return internal::SmemPADivergenceApplyTranspose2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::SmemPADivergenceApplyTranspose3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+inline VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
+VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Fallback
+(int dim, int tr_d1d, int te_d1d, int q1d)
+{
+   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
+   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   if (dim == 2)
+   {
+      return internal::SmemPADivergenceApplyTranspose2D;
+   }
+   else if (dim == 3)
+   {
+      return internal::SmemPADivergenceApplyTranspose3D;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/integ/bilininteg_vecmass_pa.hpp

@@ -328,7 +328,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Kernel()
    {
       return internal::SmemPAVectorMassApply3D<T_D1D, T_Q1D>;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType
@@ -342,7 +342,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
    {
       return internal::SmemPAVectorMassApply3D;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 // DiagonalPA kernels
@@ -358,7 +358,7 @@ VectorMassIntegrator::VectorMassAssembleDiagonalPA::Kernel()
    {
       return internal::SmemPAVectorMassAssembleDiagonal3D<T_Q1D>;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 inline VectorMassIntegrator::VectorMassAssembleDiagonalPAType
@@ -372,7 +372,7 @@ VectorMassIntegrator::VectorMassAssembleDiagonalPA::Fallback(int dim, int)
    {
       return internal::SmemPAVectorMassAssembleDiagonal3D;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_vectorfemass_kernels.hpp

@@ -0,0 +1,113 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
+#define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
+
+#include "../../config/config.hpp"
+#include "../bilininteg.hpp"
+
+#include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+#include "bilininteg_hdiv_kernels.hpp"
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
+namespace mfem
+{
+/// \cond DO_NOT_DOCUMENT
+namespace internal
+{
+namespace hcurlmass
+{
+constexpr int NBZ3D(int d1d, int q1d)
+{
+   if (d1d <= 1 || q1d <= 0)
+   {
+      return 1;
+   }
+   // assume q1d >= d1d
+   // z dimension is capped at 64 on nvidia and amd gpus
+   int tmp = std::min((128 + q1d * q1d * q1d - 1) / (q1d * q1d * q1d), 64);
+   int smem_req =
+      sizeof(mfem::real_t) *
+      (3 * ((d1d - 1) * d1d * d1d + 2 * q1d * q1d * q1d) * tmp +
+       q1d * (d1d - 1) + q1d * d1d);
+   // assume GPU has at least 48k shared memory
+   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
+}
+} // namespace hcurlmass
+} // namespace internal
+
+template <FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
+          int DIM, int TrialD1D, int TestD1D, int Q1D>
+VectorFEMassIntegrator::ApplyKernelType
+VectorFEMassIntegrator::ApplyPAKernels::Kernel()
+{
+   constexpr bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
+   constexpr bool trial_div = (TrialType == mfem::FiniteElement::DIV);
+   constexpr bool test_curl = (TestType == mfem::FiniteElement::CURL);
+   constexpr bool test_div = (TestType == mfem::FiniteElement::DIV);
+
+   if constexpr (DIM == 3)
+   {
+      if constexpr (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            // assume TrialD1D == TestD1D
+            return internal::SmemPAHcurlMassApply3D<
+                   TrialD1D, Q1D, internal::hcurlmass::NBZ3D(TrialD1D, Q1D)>;
+         }
+         else
+         {
+            return internal::PAHcurlMassApply3D;
+         }
+      }
+      else if constexpr (trial_div && test_div)
+      {
+         // assumes TrialD1D == TestD1D
+         return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
+      }
+      else if constexpr (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply3D;
+      }
+      else if constexpr (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply3D;
+      }
+   }
+   else if constexpr (DIM == 2) // 2D
+   {
+      if constexpr (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply2D;
+      }
+      else if constexpr (trial_div && test_div)
+      {
+         // assumes TrialD1D == TestD1D
+         return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
+      }
+      else if constexpr (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply2D;
+      }
+      else if constexpr (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply2D;
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+/// \endcond DO_NOT_DOCUMENT
+}
+
+#endif

fem/integ/bilininteg_vectorfemass_pa.cpp

@@ -10,15 +10,123 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
-#include "../gridfunc.hpp"
-#include "../qfunction.hpp"
-#include "bilininteg_diffusion_kernels.hpp"
-#include "bilininteg_hcurl_kernels.hpp"
-#include "bilininteg_hdiv_kernels.hpp"
-#include "bilininteg_hcurlhdiv_kernels.hpp"
+#include "bilininteg_vectorfemass_kernels.hpp"
 
 namespace mfem
 {
+/// \cond DO_NOT_DOCUMENT
+VectorFEMassIntegrator::ApplyKernelType
+VectorFEMassIntegrator::ApplyPAKernels::Fallback(
+   FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
+   int dim, int, int, int)
+{
+   const bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
+   const bool trial_div = (TrialType == mfem::FiniteElement::DIV);
+   const bool test_curl = (TestType == mfem::FiniteElement::CURL);
+   const bool test_div = (TestType == mfem::FiniteElement::DIV);
+
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply3D;
+      }
+      else if (trial_div && test_div)
+      {
+         return internal::PAHdivMassApply3D;
+      }
+      else if (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply3D;
+      }
+      else if (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply3D;
+      }
+   }
+   else if (dim == 2) // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply2D;
+      }
+      else if (trial_div && test_div)
+      {
+         return internal::PAHdivMassApply2D;
+      }
+      else if (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply2D;
+      }
+      else if (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply2D;
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+/// \endcond DO_NOT_DOCUMENT
+
+VectorFEMassIntegrator::Kernels::Kernels()
+{
+   // h(curl), h(curl)
+   // Q = P + 1 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 5>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 6>();
+   // Q = P + 2 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 5>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 6>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 7>();
+   // Q = P + 4 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 6>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 7>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 8>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 9>();
+   // h(div), h(div)
+   // Q = P (2D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 2, 2, 2>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 3, 3, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 4, 4, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 5, 5, 5>();
+
+   // Q = P + 1 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 2, 2, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 3, 3, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 4, 4, 5>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 5, 5, 6>();
+}
+
+void VectorFEMassIntegrator::Init(Coefficient *q, DiagonalMatrixCoefficient *dq,
+                                  MatrixCoefficient *mq)
+{
+   static Kernels kernels{};
+   Q = q;
+   DQ = dq;
+   MQ = mq;
+}
 
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
@@ -67,8 +175,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   trial_fetype = trial_el->GetDerivType();
-   test_fetype = test_el->GetDerivType();
+   trial_fetype = static_cast<FiniteElement::DerivType>(trial_el->GetDerivType());
+   test_fetype = static_cast<FiniteElement::DerivType>(test_el->GetDerivType());
 
    const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
    const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -215,225 +323,34 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 
 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
-
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23:
-                  return internal::SmemPAHcurlMassApply3D<2,3>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x34:
-                  return internal::SmemPAHcurlMassApply3D<3,4>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x45:
-                  return internal::SmemPAHcurlMassApply3D<4,5>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x56:
-                  return internal::SmemPAHcurlMassApply3D<5,6>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               default:
-                  return internal::SmemPAHcurlMassApply3D(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-            }
-         }
-         else
-         {
-            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
-         }
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                   mapsO->Bt, mapsC->Bt, pa_data, x, y);
-      }
-      else if (trial_curl && test_div)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                          mapsCtest->Bt, pa_data, x, y);
-      }
-      else if (trial_div && test_curl)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                          mapsCtest->Bt, pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
-   else // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                      mapsO->Bt, mapsC->Bt, pa_data, x, y);
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                   mapsO->Bt,
-                                   mapsC->Bt, pa_data, x, y);
-      }
-      else if ((trial_curl && test_div) || (trial_div && test_curl))
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          trial_curl, false, mapsO->B, mapsC->B,
-                                          mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
+   const bool scalar_coeff = !(DQ || MQ);
+   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
+                       quad1D, ne, symmetric, scalar_coeff, mapsO->B, mapsC->B,
+                       mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y, dofs1D,
+                       dofs1Dtest, quad1D);
 }
 
 void VectorFEMassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
 {
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+   const bool scalar_coeff = !(DQ || MQ);
 
    Vector abs_pa_data(pa_data);
    abs_pa_data.Abs();
 
    Array<real_t> absBo(mapsO->B);
    Array<real_t> absBc(mapsC->B);
-   Array<real_t> absBto(mapsO->Bt);
-   Array<real_t> absBtc(mapsC->Bt);
    Array<real_t> absBto_t(mapsOtest->Bt);
    Array<real_t> absBtc_t(mapsCtest->Bt);
 
    absBo.Abs();
    absBc.Abs();
-   absBto.Abs();
-   absBtc.Abs();
    absBto_t.Abs();
    absBtc_t.Abs();
 
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23:
-                  return internal::SmemPAHcurlMassApply3D<2,3>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x34:
-                  return internal::SmemPAHcurlMassApply3D<3,4>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x45:
-                  return internal::SmemPAHcurlMassApply3D<4,5>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x56:
-                  return internal::SmemPAHcurlMassApply3D<5,6>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               default:
-                  return internal::SmemPAHcurlMassApply3D(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-            }
-         }
-         else
-         {
-            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric,
-                                         absBo, absBc, absBto, absBtc,
-                                         abs_pa_data, x, y);
-         }
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric,
-                                   absBo, absBc, absBto, absBtc,
-                                   abs_pa_data, x, y);
-      }
-      else if (trial_curl && test_div)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, true, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else if (trial_div && test_curl)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, false, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
-   else // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric,
-                                      absBo, absBc, absBto, absBtc,
-                                      abs_pa_data, x, y);
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric,
-                                   absBo, absBc, absBto, absBtc,
-                                   abs_pa_data, x, y);
-      }
-      else if ((trial_curl && test_div) || (trial_div && test_curl))
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, trial_curl, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
+   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
+                       quad1D, ne, symmetric, scalar_coeff, absBo, absBc,
+                       absBto_t, absBtc_t, abs_pa_data, x, y, dofs1D,
+                       dofs1Dtest, quad1D);
 }
 
 void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,

fem/integ/nonlininteg_vecconvection_pa.cpp

@@ -9,21 +9,51 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../../general/forall.hpp"
-#include "../nonlininteg.hpp"
 #include "../ceed/integrators/nlconvection/nlconvection.hpp"
+#include "./nonlininteg_vecconvection_pa.hpp" // IWYU pragma: keep
+#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
+#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
 
 namespace mfem
 {
 
+VectorConvectionNLFIntegrator::Kernels::Kernels()
+{
+   // 2D
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 2>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 3>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 4>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 6>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 7>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 8>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 6, 8>();
+   // 3D
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 3>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 4>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 4>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 6>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 6>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 7>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 8>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 6>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 7>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 8>();
+}
+
 void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
                "PA Only supports Ordering::byNODES!");
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   ElementTransformation &T = *mesh->GetTypicalElementTransformation();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, T);
+   ElementTransformation &Tr = *mesh->GetTypicalElementTransformation();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Tr);
+
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
@@ -39,769 +69,124 @@ void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
       }
       return;
    }
-   dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
+
+   ne = mesh->GetNE();
    nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 2 || dim == 3, "Dimension not supported");
+
+   const MemoryType mt = pa_mt == MemoryType::DEFAULT
+                         ? Device::GetDeviceMemoryType()
+                         : pa_mt;
+   pa_adj.SetSize(ne * nq * dim * dim, mt);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   pa_data.SetSize(ne * nq * dim * dim, Device::GetMemoryType());
-   real_t COEFF = 1.0;
-   if (Q)
-   {
-      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient *>(Q);
-      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
-      COEFF = cQ->constant;
-   }
-   const int NE = ne;
-   const int NQ = nq;
-   auto W = ir->GetWeights().Read();
-   if (dim == 1)
-   {
-      MFEM_ABORT("dim==1 not supported!");
-   }
+   d1d = maps->ndof;
+   q1d = maps->nqpt;
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
+   const int nq1d = q1d * q1d * (dim==3 ? q1d : 1);
+   MFEM_VERIFY(coeff.Size() == 1 || coeff.Size() == nq1d*ne, "Invalid coeff");
+   MFEM_VERIFY(ir->GetWeights().Size() == nq1d, "Invalid weights size");
+
+   const auto w_r = ir->GetWeights().Read();
+   const bool const_coeff = coeff.Size() == 1;
+
    if (dim == 2)
    {
-      auto J = Reshape(geom->J.Read(), NQ, 2, 2, NE);
-      auto G = Reshape(pa_data.Write(), NQ, 2, 2, NE);
-      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+      const int Q1D = q1d;
+      constexpr int VDIM = 2, DIM = 2;
+      const auto W = Reshape(w_r, Q1D, Q1D);
+      const auto C = const_coeff ?
+                     Reshape(coeff.Read(), 1, 1, 1) :
+                     Reshape(coeff.Read(), Q1D, Q1D, ne);
+      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, VDIM, DIM, ne);
+      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, ne);
+
+      mfem::forall_2D(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
       {
-         for (int q = 0; q < NQ; ++q)
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
          {
-            const real_t J11 = J(q, 0, 0, e);
-            const real_t J12 = J(q, 0, 1, e);
-            const real_t J21 = J(q, 1, 0, e);
-            const real_t J22 = J(q, 1, 1, e);
-            // Store wq * Q * adj(J)
-            G(q, 0, 0, e) = W[q] * COEFF * J22;  // 1,1
-            G(q, 0, 1, e) = W[q] * COEFF * -J12; // 1,2
-            G(q, 1, 0, e) = W[q] * COEFF * -J21; // 2,1
-            G(q, 1, 1, e) = W[q] * COEFF * J11;  // 2,2
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               const real_t J11 = J(qx, qy, 0, 0, e), J12 = J(qx, qy, 0, 1, e);
+               const real_t J21 = J(qx, qy, 1, 0, e), J22 = J(qx, qy, 1, 1, e);
+               // adj(J)
+               const real_t A11 = +J22, A12 = -J12;
+               const real_t A21 = -J21, A22 = +J11;
+               // Store w * coeff * adj(J)
+               const real_t w = W(qx, qy);
+               const real_t c = const_coeff ? C(0, 0, 0) : C(qx, qy, e);
+               A(0, 0, qx, qy, e) = w * c * A11;
+               A(1, 0, qx, qy, e) = w * c * A12;
+               A(0, 1, qx, qy, e) = w * c * A21;
+               A(1, 1, qx, qy, e) = w * c * A22;
+            }
          }
       });
    }
-   if (dim == 3)
+   else if (dim == 3)
    {
-      auto J = Reshape(geom->J.Read(), NQ, 3, 3, NE);
-      auto G = Reshape(pa_data.Write(), NQ, 3, 3, NE);
-      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+      const int Q1D = q1d;
+      constexpr int VDIM = 3, DIM = 3;
+      const auto W = Reshape(w_r, Q1D, Q1D, Q1D);
+      const auto C = const_coeff ?
+                     Reshape(coeff.Read(), 1, 1, 1, 1) :
+                     Reshape(coeff.Read(), Q1D, Q1D, Q1D, ne);
+      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, Q1D, VDIM, DIM, ne);
+      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, Q1D, ne);
+
+      mfem::forall_3D(ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
       {
-         for (int q = 0; q < NQ; ++q)
+         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
          {
-            const real_t J11 = J(q, 0, 0, e);
-            const real_t J21 = J(q, 1, 0, e);
-            const real_t J31 = J(q, 2, 0, e);
-            const real_t J12 = J(q, 0, 1, e);
-            const real_t J22 = J(q, 1, 1, e);
-            const real_t J32 = J(q, 2, 1, e);
-            const real_t J13 = J(q, 0, 2, e);
-            const real_t J23 = J(q, 1, 2, e);
-            const real_t J33 = J(q, 2, 2, e);
-            const real_t cw = W[q] * COEFF;
-            // adj(J)
-            const real_t A11 = (J22 * J33) - (J23 * J32);
-            const real_t A12 = (J32 * J13) - (J12 * J33);
-            const real_t A13 = (J12 * J23) - (J22 * J13);
-            const real_t A21 = (J31 * J23) - (J21 * J33);
-            const real_t A22 = (J11 * J33) - (J13 * J31);
-            const real_t A23 = (J21 * J13) - (J11 * J23);
-            const real_t A31 = (J21 * J32) - (J31 * J22);
-            const real_t A32 = (J31 * J12) - (J11 * J32);
-            const real_t A33 = (J11 * J22) - (J12 * J21);
-            // Store wq * Q * adj(J)
-            G(q, 0, 0, e) = cw * A11; // 1,1
-            G(q, 0, 1, e) = cw * A12; // 1,2
-            G(q, 0, 2, e) = cw * A13; // 1,3
-            G(q, 1, 0, e) = cw * A21; // 2,1
-            G(q, 1, 1, e) = cw * A22; // 2,2
-            G(q, 1, 2, e) = cw * A23; // 2,3
-            G(q, 2, 0, e) = cw * A31; // 3,1
-            G(q, 2, 1, e) = cw * A32; // 3,2
-            G(q, 2, 2, e) = cw * A33; // 3,3
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  const real_t J11 = J(qx, qy, qz, 0, 0, e),
+                               J12 = J(qx, qy, qz, 0, 1, e),
+                               J13 = J(qx, qy, qz, 0, 2, e);
+                  const real_t J21 = J(qx, qy, qz, 1, 0, e),
+                               J22 = J(qx, qy, qz, 1, 1, e),
+                               J23 = J(qx, qy, qz, 1, 2, e);
+                  const real_t J31 = J(qx, qy, qz, 2, 0, e),
+                               J32 = J(qx, qy, qz, 2, 1, e),
+                               J33 = J(qx, qy, qz, 2, 2, e);
+                  const real_t c =
+                     const_coeff ? C(0, 0, 0, 0) : C(qx, qy, qz, e);
+                  const real_t cw = W(qx, qy, qz) * c;
+                  // adj(J)
+                  const real_t A11 = (J22 * J33) - (J23 * J32);
+                  const real_t A12 = (J32 * J13) - (J12 * J33);
+                  const real_t A13 = (J12 * J23) - (J22 * J13);
+                  const real_t A21 = (J31 * J23) - (J21 * J33);
+                  const real_t A22 = (J11 * J33) - (J13 * J31);
+                  const real_t A23 = (J21 * J13) - (J11 * J23);
+                  const real_t A31 = (J21 * J32) - (J31 * J22);
+                  const real_t A32 = (J31 * J12) - (J11 * J32);
+                  const real_t A33 = (J11 * J22) - (J12 * J21);
+                  // Store wq * coeff * adj(J)
+                  A(0, 0, qx, qy, qz, e) = cw * A11;
+                  A(1, 0, qx, qy, qz, e) = cw * A12;
+                  A(2, 0, qx, qy, qz, e) = cw * A13;
+                  A(0, 1, qx, qy, qz, e) = cw * A21;
+                  A(1, 1, qx, qy, qz, e) = cw * A22;
+                  A(2, 1, qx, qy, qz, e) = cw * A23;
+                  A(0, 2, qx, qy, qz, e) = cw * A31;
+                  A(1, 2, qx, qy, qz, e) = cw * A32;
+                  A(2, 2, qx, qy, qz, e) = cw * A33;
+               }
+            }
          }
       });
    }
-}
-
-// PA Convection NL 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAConvectionNLApply2D(const int NE,
-                                  const Array<real_t> &b,
-                                  const Array<real_t> &g,
-                                  const Array<real_t> &bt,
-                                  const Vector &q_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Q = Reshape(q_.Read(), Q1D * Q1D, 2, 2, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, 2, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   else
    {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      real_t data[max_Q1D][max_Q1D][2];
-      real_t grad0[max_Q1D][max_Q1D][2];
-      real_t grad1[max_Q1D][max_Q1D][2];
-      real_t Z[max_Q1D][max_Q1D][2];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            data[qy][qx][0] = 0.0;
-            data[qy][qx][1] = 0.0;
-            grad0[qy][qx][0] = 0.0;
-            grad0[qy][qx][1] = 0.0;
-            grad1[qy][qx][0] = 0.0;
-            grad1[qy][qx][1] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         real_t dataX[max_Q1D][2];
-         real_t gradX0[max_Q1D][2];
-         real_t gradX1[max_Q1D][2];
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            dataX[qx][0] = 0.0;
-            dataX[qx][1] = 0.0;
-            gradX0[qx][0] = 0.0;
-            gradX0[qx][1] = 0.0;
-            gradX1[qx][0] = 0.0;
-            gradX1[qx][1] = 0.0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const real_t s0 = x(dx, dy, 0, e);
-            const real_t s1 = x(dx, dy, 1, e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const real_t Bx = B(qx, dx);
-               const real_t Gx = G(qx, dx);
-               dataX[qx][0] += s0 * Bx;
-               dataX[qx][1] += s1 * Bx;
-               gradX0[qx][0] += s0 * Gx;
-               gradX0[qx][1] += s0 * Bx;
-               gradX1[qx][0] += s1 * Gx;
-               gradX1[qx][1] += s1 * Bx;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const real_t By = B(qy, dy);
-            const real_t Gy = G(qy, dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               data[qy][qx][0] += dataX[qx][0] * By;
-               data[qy][qx][1] += dataX[qx][1] * By;
-               grad0[qy][qx][0] += gradX0[qx][0] * By;
-               grad0[qy][qx][1] += gradX0[qx][1] * Gy;
-               grad1[qy][qx][0] += gradX1[qx][0] * By;
-               grad1[qy][qx][1] += gradX1[qx][1] * Gy;
-            }
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const int q = qx + qy * Q1D;
-            const real_t u1 = data[qy][qx][0];
-            const real_t u2 = data[qy][qx][1];
-            const real_t grad00 = grad0[qy][qx][0];
-            const real_t grad01 = grad0[qy][qx][1];
-            const real_t grad10 = grad1[qy][qx][0];
-            const real_t grad11 = grad1[qy][qx][1];
-            const real_t Dxu1 = grad00 * Q(q, 0, 0, e) + grad01 * Q(q, 1, 0, e);
-            const real_t Dyu1 = grad00 * Q(q, 0, 1, e) + grad01 * Q(q, 1, 1, e);
-            const real_t Dxu2 = grad10 * Q(q, 0, 0, e) + grad11 * Q(q, 1, 0, e);
-            const real_t Dyu2 = grad10 * Q(q, 0, 1, e) + grad11 * Q(q, 1, 1, e);
-            Z[qy][qx][0] = u1 * Dxu1 + u2 * Dyu1;
-            Z[qy][qx][1] = u1 * Dxu2 + u2 * Dyu2;
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         real_t Y[max_D1D][2];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            Y[dx][0] = 0.0;
-            Y[dx][1] = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const real_t Btx = Bt(dx, qx);
-               Y[dx][0] += Btx * Z[qy][qx][0];
-               Y[dx][1] += Btx * Z[qy][qx][1];
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const real_t Bty = Bt(dy, qy);
-               y(dx, dy, 0, e) += Bty * Y[dx][0];
-               y(dx, dy, 1, e) += Bty * Y[dx][1];
-            }
-         }
-      }
-   });
-}
-
-// PA Convection NL 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAConvectionNLApply3D(const int NE,
-                                  const Array<real_t> &b,
-                                  const Array<real_t> &g,
-                                  const Array<real_t> &bt,
-                                  const Vector &q_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
-{
-   constexpr int VDIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Q = Reshape(q_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int VDIM = 3;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      real_t data[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      real_t grad0[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      real_t grad1[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      real_t grad2[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      real_t Z[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               data[qz][qy][qx][0] = 0.0;
-               data[qz][qy][qx][1] = 0.0;
-               data[qz][qy][qx][2] = 0.0;
-
-               grad0[qz][qy][qx][0] = 0.0;
-               grad0[qz][qy][qx][1] = 0.0;
-               grad0[qz][qy][qx][2] = 0.0;
-
-               grad1[qz][qy][qx][0] = 0.0;
-               grad1[qz][qy][qx][1] = 0.0;
-               grad1[qz][qy][qx][2] = 0.0;
-
-               grad2[qz][qy][qx][0] = 0.0;
-               grad2[qz][qy][qx][1] = 0.0;
-               grad2[qz][qy][qx][2] = 0.0;
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         real_t dataXY[max_Q1D][max_Q1D][VDIM];
-         real_t gradXY0[max_Q1D][max_Q1D][VDIM];
-         real_t gradXY1[max_Q1D][max_Q1D][VDIM];
-         real_t gradXY2[max_Q1D][max_Q1D][VDIM];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dataXY[qy][qx][0] = 0.0;
-               dataXY[qy][qx][1] = 0.0;
-               dataXY[qy][qx][2] = 0.0;
-
-               gradXY0[qy][qx][0] = 0.0;
-               gradXY0[qy][qx][1] = 0.0;
-               gradXY0[qy][qx][2] = 0.0;
-
-               gradXY1[qy][qx][0] = 0.0;
-               gradXY1[qy][qx][1] = 0.0;
-               gradXY1[qy][qx][2] = 0.0;
-
-               gradXY2[qy][qx][0] = 0.0;
-               gradXY2[qy][qx][1] = 0.0;
-               gradXY2[qy][qx][2] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            real_t dataX[max_Q1D][VDIM];
-            real_t gradX0[max_Q1D][VDIM];
-            real_t gradX1[max_Q1D][VDIM];
-            real_t gradX2[max_Q1D][VDIM];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dataX[qx][0] = 0.0;
-               dataX[qx][1] = 0.0;
-               dataX[qx][2] = 0.0;
-
-               gradX0[qx][0] = 0.0;
-               gradX0[qx][1] = 0.0;
-               gradX0[qx][2] = 0.0;
-
-               gradX1[qx][0] = 0.0;
-               gradX1[qx][1] = 0.0;
-               gradX1[qx][2] = 0.0;
-
-               gradX2[qx][0] = 0.0;
-               gradX2[qx][1] = 0.0;
-               gradX2[qx][2] = 0.0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const real_t s0 = x(dx, dy, dz, 0, e);
-               const real_t s1 = x(dx, dy, dz, 1, e);
-               const real_t s2 = x(dx, dy, dz, 2, e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t Bx = B(qx, dx);
-                  const real_t Gx = G(qx, dx);
-
-                  dataX[qx][0] += s0 * Bx;
-                  dataX[qx][1] += s1 * Bx;
-                  dataX[qx][2] += s2 * Bx;
-
-                  gradX0[qx][0] += s0 * Gx;
-                  gradX0[qx][1] += s0 * Bx;
-                  gradX0[qx][2] += s0 * Bx;
-
-                  gradX1[qx][0] += s1 * Gx;
-                  gradX1[qx][1] += s1 * Bx;
-                  gradX1[qx][2] += s1 * Bx;
-
-                  gradX2[qx][0] += s2 * Gx;
-                  gradX2[qx][1] += s2 * Bx;
-                  gradX2[qx][2] += s2 * Bx;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const real_t By = B(qy, dy);
-               const real_t Gy = G(qy, dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  dataXY[qy][qx][0] += dataX[qx][0] * By;
-                  dataXY[qy][qx][1] += dataX[qx][1] * By;
-                  dataXY[qy][qx][2] += dataX[qx][2] * By;
-
-                  gradXY0[qy][qx][0] += gradX0[qx][0] * By;
-                  gradXY0[qy][qx][1] += gradX0[qx][1] * Gy;
-                  gradXY0[qy][qx][2] += gradX0[qx][2] * By;
-
-                  gradXY1[qy][qx][0] += gradX1[qx][0] * By;
-                  gradXY1[qy][qx][1] += gradX1[qx][1] * Gy;
-                  gradXY1[qy][qx][2] += gradX1[qx][2] * By;
-
-                  gradXY2[qy][qx][0] += gradX2[qx][0] * By;
-                  gradXY2[qy][qx][1] += gradX2[qx][1] * Gy;
-                  gradXY2[qy][qx][2] += gradX2[qx][2] * By;
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const real_t Bz = B(qz, dz);
-            const real_t Gz = G(qz, dz);
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  data[qz][qy][qx][0] += dataXY[qy][qx][0] * Bz;
-                  data[qz][qy][qx][1] += dataXY[qy][qx][1] * Bz;
-                  data[qz][qy][qx][2] += dataXY[qy][qx][2] * Bz;
-
-                  grad0[qz][qy][qx][0] += gradXY0[qy][qx][0] * Bz;
-                  grad0[qz][qy][qx][1] += gradXY0[qy][qx][1] * Bz;
-                  grad0[qz][qy][qx][2] += gradXY0[qy][qx][2] * Gz;
-
-                  grad1[qz][qy][qx][0] += gradXY1[qy][qx][0] * Bz;
-                  grad1[qz][qy][qx][1] += gradXY1[qy][qx][1] * Bz;
-                  grad1[qz][qy][qx][2] += gradXY1[qy][qx][2] * Gz;
-
-                  grad2[qz][qy][qx][0] += gradXY2[qy][qx][0] * Bz;
-                  grad2[qz][qy][qx][1] += gradXY2[qy][qx][1] * Bz;
-                  grad2[qz][qy][qx][2] += gradXY2[qy][qx][2] * Gz;
-               }
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const int q = qx + Q1D * (qy + qz * Q1D);
-
-               const real_t u1 = data[qz][qy][qx][0];
-               const real_t u2 = data[qz][qy][qx][1];
-               const real_t u3 = data[qz][qy][qx][2];
-
-               const real_t grad00 = grad0[qz][qy][qx][0];
-               const real_t grad01 = grad0[qz][qy][qx][1];
-               const real_t grad02 = grad0[qz][qy][qx][2];
-
-               const real_t grad10 = grad1[qz][qy][qx][0];
-               const real_t grad11 = grad1[qz][qy][qx][1];
-               const real_t grad12 = grad1[qz][qy][qx][2];
-
-               const real_t grad20 = grad2[qz][qy][qx][0];
-               const real_t grad21 = grad2[qz][qy][qx][1];
-               const real_t grad22 = grad2[qz][qy][qx][2];
-
-               const real_t Dxu1 = grad00 * Q(q, 0, 0, e)
-                                   + grad01 * Q(q, 1, 0, e)
-                                   + grad02 * Q(q, 2, 0, e);
-               const real_t Dyu1 = grad00 * Q(q, 0, 1, e)
-                                   + grad01 * Q(q, 1, 1, e)
-                                   + grad02 * Q(q, 2, 1, e);
-               const real_t Dzu1 = grad00 * Q(q, 0, 2, e)
-                                   + grad01 * Q(q, 1, 2, e)
-                                   + grad02 * Q(q, 2, 2, e);
-
-               const real_t Dxu2 = grad10 * Q(q, 0, 0, e)
-                                   + grad11 * Q(q, 1, 0, e)
-                                   + grad12 * Q(q, 2, 0, e);
-               const real_t Dyu2 = grad10 * Q(q, 0, 1, e)
-                                   + grad11 * Q(q, 1, 1, e)
-                                   + grad12 * Q(q, 2, 1, e);
-               const real_t Dzu2 = grad10 * Q(q, 0, 2, e)
-                                   + grad11 * Q(q, 1, 2, e)
-                                   + grad12 * Q(q, 2, 2, e);
-
-               const real_t Dxu3 = grad20 * Q(q, 0, 0, e)
-                                   + grad21 * Q(q, 1, 0, e)
-                                   + grad22 * Q(q, 2, 0, e);
-               const real_t Dyu3 = grad20 * Q(q, 0, 1, e)
-                                   + grad21 * Q(q, 1, 1, e)
-                                   + grad22 * Q(q, 2, 1, e);
-               const real_t Dzu3 = grad20 * Q(q, 0, 2, e)
-                                   + grad21 * Q(q, 1, 2, e)
-                                   + grad22 * Q(q, 2, 2, e);
-
-               Z[qz][qy][qx][0] = u1 * Dxu1 + u2 * Dyu1 + u3 * Dzu1;
-               Z[qz][qy][qx][1] = u1 * Dxu2 + u2 * Dyu2 + u3 * Dzu2;
-               Z[qz][qy][qx][2] = u1 * Dxu3 + u2 * Dyu3 + u3 * Dzu3;
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         real_t opXY[max_D1D][max_D1D][VDIM];
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               opXY[dy][dx][0] = 0.0;
-               opXY[dy][dx][1] = 0.0;
-               opXY[dy][dx][2] = 0.0;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            real_t opX[max_D1D][VDIM];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               opX[dx][0] = 0.0;
-               opX[dx][1] = 0.0;
-               opX[dx][2] = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t Btx = Bt(dx, qx);
-                  opX[dx][0] += Btx * Z[qz][qy][qx][0];
-                  opX[dx][1] += Btx * Z[qz][qy][qx][1];
-                  opX[dx][2] += Btx * Z[qz][qy][qx][2];
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t Bty = Bt(dy, qy);
-                  opXY[dy][dx][0] += Bty * opX[dx][0];
-                  opXY[dy][dx][1] += Bty * opX[dx][1];
-                  opXY[dy][dx][2] += Bty * opX[dx][2];
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t Btz = Bt(dz, qz);
-                  y(dx, dy, dz, 0, e) += Btz * opXY[dy][dx][0];
-                  y(dx, dy, dz, 1, e) += Btz * opXY[dy][dx][1];
-                  y(dx, dy, dz, 2, e) += Btz * opXY[dy][dx][2];
-               }
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_MAX_D1D = 0, int T_MAX_Q1D = 0>
-static void SmemPAConvectionNLApply3D(const int NE,
-                                      const Array<real_t> &b_,
-                                      const Array<real_t> &g_,
-                                      const Vector &d_,
-                                      const Vector &x_,
-                                      Vector &y_,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   constexpr int VDIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
-      MFEM_SHARED real_t BG[2][MQ1 * MD1];
-      real_t(*B)[MD1] = (real_t(*)[MD1])(BG + 0);
-      real_t(*G)[MD1] = (real_t(*)[MD1])(BG + 1);
-      real_t(*Bt)[MQ1] = (real_t(*)[MQ1])(BG + 0);
-      MFEM_SHARED real_t U[2][MQ1][MQ1][MQ1];
-      MFEM_SHARED real_t sm0[3][MQ1 * MQ1 * MQ1];
-      MFEM_SHARED real_t sm1[3][MQ1 * MQ1 * MQ1];
-      real_t(*DDQ0)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 0);
-      real_t(*DDQ1)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 1);
-      real_t(*X)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 2);
-      real_t(*DQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 0);
-      real_t(*DQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 1);
-      real_t(*DQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 2);
-      real_t(*QQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 0);
-      real_t(*QQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 1);
-      real_t(*QQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 2);
-      real_t(*QQD0)[MQ1][MD1] = (real_t(*)[MQ1][MD1])(sm1 + 0);
-      real_t(*QDD0)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 0);
-      MFEM_SHARED real_t Z[MQ1][MQ1][MQ1];
-
-      for (int cy = 0; cy < VDIM; ++cy)
-      {
-         if (tidz == 0)
-         {
-            MFEM_FOREACH_THREAD(q, x, Q1D)
-            {
-               MFEM_FOREACH_THREAD(d, y, D1D)
-               {
-                  B[q][d] = b(q, d);
-                  G[q][d] = g(q, d);
-               }
-            }
-         }
-         MFEM_FOREACH_THREAD(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx, x, Q1D) { Z[qz][qy][qx] = 0.0; }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int c = 0; c < VDIM; ++c)
-         {
-            MFEM_FOREACH_THREAD(dz, z, D1D)
-            {
-               MFEM_FOREACH_THREAD(dy, y, D1D)
-               {
-                  MFEM_FOREACH_THREAD(dx, x, D1D)
-                  {
-                     X[dz][dy][dx] = x(dx, dy, dz, cy, e);
-                     U[0][dz][dy][dx] = x(dx, dy, dz, c, e);
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            MFEM_FOREACH_THREAD(dz, z, D1D)
-            {
-               MFEM_FOREACH_THREAD(dy, y, D1D)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, Q1D)
-                  {
-                     real_t u = 0.0;
-                     real_t v = 0.0;
-                     real_t z = 0.0;
-                     for (int dx = 0; dx < D1D; ++dx)
-                     {
-                        const real_t coord = X[dz][dy][dx];
-                        const real_t value = U[0][dz][dy][dx];
-                        u += coord * B[qx][dx];
-                        v += coord * G[qx][dx];
-                        z += value * B[qx][dx];
-                     }
-                     DDQ0[dz][dy][qx] = u;
-                     DDQ1[dz][dy][qx] = v;
-                     U[1][dz][dy][qx] = z;
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            MFEM_FOREACH_THREAD(dz, z, D1D)
-            {
-               MFEM_FOREACH_THREAD(qy, y, Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, Q1D)
-                  {
-                     real_t u = 0.0;
-                     real_t v = 0.0;
-                     real_t w = 0.0;
-                     real_t z = 0.0;
-                     for (int dy = 0; dy < D1D; ++dy)
-                     {
-                        u += DDQ1[dz][dy][qx] * B[qy][dy];
-                        v += DDQ0[dz][dy][qx] * G[qy][dy];
-                        w += DDQ0[dz][dy][qx] * B[qy][dy];
-                        z += U[1][dz][dy][qx] * B[qy][dy];
-                     }
-                     DQQ0[dz][qy][qx] = u;
-                     DQQ1[dz][qy][qx] = v;
-                     DQQ2[dz][qy][qx] = w;
-                     U[0][dz][qy][qx] = z;
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            MFEM_FOREACH_THREAD(qz, z, Q1D)
-            {
-               MFEM_FOREACH_THREAD(qy, y, Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, Q1D)
-                  {
-                     real_t u = 0.0;
-                     real_t v = 0.0;
-                     real_t w = 0.0;
-                     real_t z = 0.0;
-                     for (int dz = 0; dz < D1D; ++dz)
-                     {
-                        u += DQQ0[dz][qy][qx] * B[qz][dz];
-                        v += DQQ1[dz][qy][qx] * B[qz][dz];
-                        w += DQQ2[dz][qy][qx] * G[qz][dz];
-                        z += U[0][dz][qy][qx] * B[qz][dz];
-                     }
-                     QQQ0[qz][qy][qx] = u;
-                     QQQ1[qz][qy][qx] = v;
-                     QQQ2[qz][qy][qx] = w;
-                     U[1][qz][qy][qx] = z;
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            MFEM_FOREACH_THREAD(qz, z, Q1D)
-            {
-               MFEM_FOREACH_THREAD(qy, y, Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, Q1D)
-                  {
-                     const int q = qx + (qy + qz * Q1D) * Q1D;
-                     const real_t z = U[1][qz][qy][qx];
-                     const real_t gX = QQQ0[qz][qy][qx];
-                     const real_t gY = QQQ1[qz][qy][qx];
-                     const real_t gZ = QQQ2[qz][qy][qx];
-                     const real_t d = gX * D(q, 0, c, e) + gY * D(q, 1, c, e)
-                                      + gZ * D(q, 2, c, e);
-                     Z[qz][qy][qx] += z * d;
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-         } // for each conv component
-         if (tidz == 0)
-         {
-            MFEM_FOREACH_THREAD(d, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(q, x, Q1D) { Bt[d][q] = b(q, d); }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += Z[qz][qy][qx] * Bt[dx][qx];
-                  }
-                  QQD0[qz][qy][dx] = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += QQD0[qz][qy][dx] * Bt[dy][qy];
-                  }
-                  QDD0[qz][dy][dx] = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz, z, D1D)
-         {
-            MFEM_FOREACH_THREAD(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += QDD0[qz][dy][dx] * Bt[dz][qz];
-                  }
-                  Y(dx, dy, dz, cy, e) += u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
+      MFEM_ABORT("dim " << dim << " not supported!");
+   }
 }
 
 void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -812,26 +197,13 @@ void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      const int NE = ne;
-      const int D1D = maps->ndof;
-      const int Q1D = maps->nqpt;
-      const Vector &QV = pa_data;
-      const Array<real_t> &B = maps->B;
-      const Array<real_t> &G = maps->G;
-      const Array<real_t> &Bt = maps->Bt;
-      if (dim == 2)
-      {
-         return PAConvectionNLApply2D(NE, B, G, Bt, QV, x, y, D1D, Q1D);
-      }
-      if (dim == 3)
-      {
-         constexpr int T_MAX_D1D = 8;
-         constexpr int T_MAX_Q1D = 8;
-         MFEM_VERIFY(D1D <= T_MAX_D1D && Q1D <= T_MAX_Q1D, "Not yet implemented!");
-         return SmemPAConvectionNLApply3D<0, 0, T_MAX_D1D, T_MAX_Q1D>
-                (NE, B, G, QV, x, y, D1D, Q1D);
-      }
-      MFEM_ABORT("Not yet implemented!");
+      AddMultPAKernels::Run(dim, d1d, q1d, ne,
+                            maps->B.Read(),
+                            maps->G.Read(),
+                            pa_adj.Read(),
+                            x.Read(),
+                            y.ReadWrite(),
+                            d1d, q1d);
    }
 }
 

fem/integ/nonlininteg_vecconvection_pa.hpp

@@ -0,0 +1,209 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../config/config.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../kernels.hpp"
+#include "../nonlininteg.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+namespace internal
+{
+
+// PA Convection NL 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLApply2D(const int NE,
+                                      const real_t *b,
+                                      const real_t *g,
+                                      const real_t *a,
+                                      const real_t *x,
+                                      real_t *y,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
+{
+   static constexpr int VDIM = 2, DIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
+   const auto X = Reshape(x, D1D, D1D, VDIM, NE);
+   auto Y = Reshape(y, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
+      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
+      kernels::internal::v_regs2d_t<VDIM, MQ1> s0, s1;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
+
+      kernels::internal::LoadDofs2d(e, D1D, X, r0);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
+      kernels::internal::LoadDofs2d(e, D1D, X, g0);
+      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            const future::tensor<real_t, 2> U =
+            {
+               r1[0][qy][qx], r1[1][qy][qx]
+            };
+            const future::tensor<real_t, 2,2> gradU = {{
+                  {g1[0][0][qy][qx], g1[1][0][qy][qx]},
+                  {g1[0][1][qy][qx], g1[1][1][qy][qx]},
+               }
+            };
+            const future::tensor<real_t, 2,2> Q = {{
+                  {A(0,0,qx,qy,e), A(1,0,qx,qy,e)},
+                  {A(0,1,qx,qy,e), A(1,1,qx,qy,e)},
+               }
+            };
+            const future::tensor<real_t, 2> conv = transpose(gradU) * (Q * U);
+            s0[0][qy][qx] = conv[0];
+            s0[1][qy][qx] = conv[1];
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, s0, s1);
+      kernels::internal::WriteDofs2d(e, D1D, s1, Y);
+   });
+}
+
+// PA Convection NL 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLApply3D(const int NE,
+                                      const real_t *b,
+                                      const real_t *g,
+                                      const real_t *a,
+                                      const real_t *x,
+                                      real_t *y,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
+{
+   static constexpr int VDIM = 3, DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x, D1D, D1D, D1D, VDIM, NE);
+   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
+      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
+      kernels::internal::v_regs3d_t<VDIM, MQ1> s0, s1;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
+
+      kernels::internal::LoadDofs3d(e, D1D, X, r0);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
+      kernels::internal::LoadDofs3d(e, D1D, X, g0);
+      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
+
+      for (int qz = 0; qz < Q1D; qz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               const future::tensor<real_t, 3> U =
+               {
+                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
+               };
+               const future::tensor<real_t, 3,3> gradU = {{
+                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
+                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
+                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
+                  }
+               };
+               const future::tensor<real_t, 3,3> Q = {{
+                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
+                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
+                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
+                  }
+               };
+               const future::tensor<real_t, 3> conv = transpose(gradU) * (Q * U);
+               s0[0][qz][qy][qx] = conv[0];
+               s0[1][qz][qy][qx] = conv[1];
+               s0[2][qz][qy][qx] = conv[2];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, s0, s1);
+      kernels::internal::WriteDofs3d(e, D1D, s1, Y);
+   });
+}
+
+} // namespace internal
+
+template<int DIM, int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::AddMultPAType
+VectorConvectionNLFIntegrator::AddMultPAKernels::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   if constexpr (DIM == 2)
+   {
+      return internal::SmemPAConvectionNLApply2D<T_D1D, T_Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::SmemPAConvectionNLApply3D<T_D1D, T_Q1D>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+inline VectorConvectionNLFIntegrator::AddMultPAType
+VectorConvectionNLFIntegrator::AddMultPAKernels::Fallback
+(int dim, int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   if (dim == 2)
+   {
+      return internal::SmemPAConvectionNLApply2D<>;
+   }
+   else if (dim == 3)
+   {
+      return internal::SmemPAConvectionNLApply3D<>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../ceed/interface/util.hpp"
+#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
+
+namespace mfem
+{
+
+void VectorConvectionNLFIntegrator::AssembleGradDiagonalPA(Vector &de) const
+{
+   MFEM_VERIFY(!DeviceCanUseCeed(),
+               "VectorConvectionNLFIntegrator PA gradients are not supported "
+               "with the libCEED backend");
+
+   if (dim == 2)
+   {
+      GradDiagPA2D::Run(d1d, q1d, ne,
+                        maps->B.Read(),
+                        maps->G.Read(),
+                        pa_adj.Read(),
+                        pa_u.Read(),
+                        de.ReadWrite(),
+                        d1d, q1d);
+   }
+   else if (dim == 3)
+   {
+      GradDiagPA3D::Run(d1d, q1d, ne,
+                        maps->B.Read(),
+                        maps->G.Read(),
+                        pa_adj.Read(),
+                        pa_u.Read(),
+                        de.ReadWrite(),
+                        d1d, q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+   }
+}
+
+} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.hpp

@@ -0,0 +1,302 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../config/config.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../kernels.hpp"
+#include "../nonlininteg.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+namespace internal
+{
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLGradDiagonal2D(const int NE,
+                                             const real_t *b,
+                                             const real_t *g,
+                                             const real_t *a,
+                                             const real_t *u,
+                                             real_t *de,
+                                             const int d1d,
+                                             const int q1d)
+{
+   static constexpr int VDIM = 2, DIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
+   const auto U = Reshape(u, D1D, D1D, VDIM, NE);
+   auto D = Reshape(de, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t sM[3][MQ1][MQ1], sQ[3][MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
+      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
+
+      kernels::internal::LoadDofs2d(e, D1D, U, r0);
+      kernels::internal::Eval2d(D1D, Q1D, sM[0], sB, r0, r1);
+
+      kernels::internal::LoadDofs2d(e, D1D, U, g0);
+      kernels::internal::Grad2d(D1D, Q1D, sM[0], sB, sG, g0, g1);
+
+      for (int v = 0; v < VDIM; ++v)
+      {
+         future::tensor<real_t, VDIM> e_v = {};
+         e_v[v] = real_t(1);
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               const future::tensor<real_t, VDIM> u_val =
+               {
+                  r1[0][qy][qx], r1[1][qy][qx]
+               };
+               const future::tensor<real_t, VDIM, DIM> Q_adj =
+               {
+                  {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
+                     { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
+                  }
+               };
+               const future::tensor<real_t, VDIM, DIM> grad_U =
+               {
+                  {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
+                     { g1[0][1][qy][qx], g1[1][1][qy][qx] }
+                  }
+               };
+               const auto one = Q_adj * u_val;
+               const auto two = transpose(grad_U) * (Q_adj * e_v);
+               sQ[0][qx][qy] = one[0];
+               sQ[1][qx][qy] = one[1];
+               sQ[2][qx][qy] = two[v];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               real_t s[3] = {};
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const real_t By = sB[dy][qy], Gy = sG[dy][qy];
+                  s[0] += By * By * sQ[0][qx][qy];
+                  s[1] += Gy * By * sQ[1][qx][qy];
+                  s[2] += By * By * sQ[2][qx][qy];
+               }
+               sM[0][qx][dy] = s[0];
+               sM[1][qx][dy] = s[1];
+               sM[2][qx][dy] = s[2];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               real_t d = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
+                  d += Gx * Bx * sM[0][qx][dy] +
+                       Bx * Bx * sM[1][qx][dy] +
+                       Bx * Bx * sM[2][qx][dy];
+               }
+               D(dx, dy, v, e) += d;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLGradDiagonal3D(const int NE,
+                                             const real_t *b,
+                                             const real_t *g,
+                                             const real_t *a,
+                                             const real_t *u,
+                                             real_t *de,
+                                             const int d1d,
+                                             const int q1d)
+{
+   static constexpr int VDIM = 3, DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, NE);
+   auto D = Reshape(de, D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t sM[4][MQ1][MQ1], sQ[4][MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
+      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
+
+      kernels::internal::LoadDofs3d(e, D1D, U, r0);
+      kernels::internal::Eval3d(D1D, Q1D, sM[0], sB, r0, r1);
+
+      kernels::internal::LoadDofs3d(e, D1D, U, g0);
+      kernels::internal::Grad3d(D1D, Q1D, sM[0], sB, sG, g0, g1);
+
+      for (int v = 0; v < VDIM; ++v)
+      {
+         future::tensor<real_t, VDIM> e_v = {};
+         e_v[v] = real_t(1);
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  real_t s[4] = {};
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     const future::tensor<real_t, VDIM> u_val =
+                     {
+                        r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
+                     };
+                     const future::tensor<real_t, VDIM, DIM> Q_adj = {{
+                           {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
+                           {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
+                           {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
+                        }
+                     };
+                     const future::tensor<real_t, VDIM, DIM> grad_U = {{
+                           {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
+                           {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
+                           {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
+                        }
+                     };
+                     const auto one = Q_adj * u_val;
+                     const auto two = transpose(grad_U) * (Q_adj * e_v);
+
+                     const real_t Bz = sB[dz][qz], Gz = sG[dz][qz];
+                     s[0] += one[0] * Bz * Bz;
+                     s[1] += one[1] * Bz * Bz;
+                     s[2] += one[2] * Bz * Gz;
+                     s[3] += two[v] * Bz * Bz;
+                  }
+                  sQ[0][qx][qy] = s[0];
+                  sQ[1][qx][qy] = s[1];
+                  sQ[2][qx][qy] = s[2];
+                  sQ[3][qx][qy] = s[3];
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  real_t s[4] = {};
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     const real_t By = sB[dy][qy], Gy = sG[dy][qy];
+                     s[0] += By * By * sQ[0][qx][qy];
+                     s[1] += Gy * By * sQ[1][qx][qy];
+                     s[2] += By * By * sQ[2][qx][qy];
+                     s[3] += By * By * sQ[3][qx][qy];
+                  }
+                  sM[0][dy][qx] = s[0];
+                  sM[1][dy][qx] = s[1];
+                  sM[2][dy][qx] = s[2];
+                  sM[3][dy][qx] = s[3];
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  real_t d = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
+                     d += Gx * Bx * sM[0][dy][qx];
+                     d += Bx * Bx * sM[1][dy][qx];
+                     d += Bx * Bx * sM[2][dy][qx];
+                     d += Bx * Bx * sM[3][dy][qx];
+                  }
+                  D(dx, dy, dz, v, e) += d;
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   });
+}
+
+} // namespace internal
+
+template<int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::GradDiagPAType
+VectorConvectionNLFIntegrator::GradDiagPA2D::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   return internal::SmemPAConvectionNLGradDiagonal2D<T_D1D, T_Q1D>;
+}
+
+inline VectorConvectionNLFIntegrator::GradDiagPAType
+VectorConvectionNLFIntegrator::GradDiagPA2D::Fallback(int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   return internal::SmemPAConvectionNLGradDiagonal2D<>;
+}
+
+template<int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::GradDiagPAType
+VectorConvectionNLFIntegrator::GradDiagPA3D::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   return internal::SmemPAConvectionNLGradDiagonal3D<T_D1D, T_Q1D>;
+}
+
+inline VectorConvectionNLFIntegrator::GradDiagPAType
+VectorConvectionNLFIntegrator::GradDiagPA3D::Fallback(int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   return internal::SmemPAConvectionNLGradDiagonal3D<>;
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.cpp

@@ -0,0 +1,64 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../ceed/interface/util.hpp"
+#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
+
+namespace mfem
+{
+
+void VectorConvectionNLFIntegrator::AssembleGradPA(
+   const Vector &u, const FiniteElementSpace &fes)
+{
+   MFEM_VERIFY(!DeviceCanUseCeed(),
+               "VectorConvectionNLFIntegrator PA gradients are not supported "
+               "with the libCEED backend");
+
+   this->pa_u = u;
+   AssemblePA(fes);
+}
+
+void VectorConvectionNLFIntegrator::AddMultGradPA(const Vector &x,
+                                                  Vector &y) const
+{
+   MFEM_VERIFY(!DeviceCanUseCeed(),
+               "VectorConvectionNLFIntegrator PA gradients are not supported "
+               "with the libCEED backend");
+
+   if (dim == 2)
+   {
+      AddMultGradPA2D::Run(d1d, q1d, ne,
+                           maps->B.Read(),
+                           maps->G.Read(),
+                           pa_adj.Read(),
+                           pa_u.Read(),
+                           x.Read(),
+                           y.ReadWrite(),
+                           d1d, q1d);
+   }
+   else if (dim == 3)
+   {
+      AddMultGradPA3D::Run(d1d, q1d, ne,
+                           maps->B.Read(),
+                           maps->G.Read(),
+                           pa_adj.Read(),
+                           pa_u.Read(),
+                           x.Read(),
+                           y.ReadWrite(),
+                           d1d, q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+   }
+}
+
+} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.hpp

@@ -0,0 +1,257 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../config/config.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../kernels.hpp"
+#include "../nonlininteg.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+namespace internal
+{
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLGradApply2D(const int ne,
+                                          const real_t *b,
+                                          const real_t *g,
+                                          const real_t *a,
+                                          const real_t *u,
+                                          const real_t *du,
+                                          real_t *y,
+                                          const int d1d,
+                                          const int q1d)
+{
+   static constexpr int VDIM = 2, DIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, ne);
+   const auto U = Reshape(u, D1D, D1D, VDIM, ne);
+   const auto dU = Reshape(du, D1D, D1D, VDIM, ne);
+   auto Y = Reshape(y, D1D, D1D, VDIM, ne);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1, g2;
+      kernels::internal::v_regs2d_t<DIM, MQ1> r0, r1, r2;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
+
+      kernels::internal::LoadDofs2d(e, D1D, dU, g0);
+      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
+
+      kernels::internal::LoadDofs2d(e, D1D, U, r0);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r2); // u value
+
+      kernels::internal::LoadDofs2d(e, D1D, dU, r0);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // δu value
+
+      kernels::internal::LoadDofs2d(e, D1D, U, g0);
+      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            // First part of the Jacobian: u·∇δu
+            const future::tensor<real_t, DIM> u_val =
+            {
+               r2[0][qy][qx], r2[1][qy][qx]
+            };
+            const future::tensor<real_t, VDIM, DIM> Q_adj =
+            {
+               {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
+                  { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
+               }
+            };
+            const future::tensor<real_t, VDIM, DIM> grad_dU =
+            {
+               {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
+                  { g1[0][1][qy][qx], g1[1][1][qy][qx] }
+               }
+            };
+            const auto one = transpose(grad_dU) * (Q_adj * u_val);
+
+            // Second part of the Jacobian: δu·∇u
+            const future::tensor<real_t, DIM> du_val =
+            {
+               r1[0][qy][qx], r1[1][qy][qx]
+            };
+            const future::tensor<real_t, VDIM, DIM> grad_U =
+            {
+               {  { g2[0][0][qy][qx], g2[1][0][qy][qx] },
+                  { g2[0][1][qy][qx], g2[1][1][qy][qx] }
+               }
+            };
+            const auto two = transpose(grad_U) * (Q_adj * du_val);
+
+            // u⋅∇δu + δu⋅∇u
+            r0[0][qy][qx] = one[0] + two[0];
+            r0[1][qy][qx] = one[1] + two[1];
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r0, r1);
+      kernels::internal::WriteDofs2d(e, D1D, r1, Y);
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLGradApply3D(const int ne,
+                                          const real_t *b,
+                                          const real_t *g,
+                                          const real_t *a,
+                                          const real_t *u,
+                                          const real_t *du,
+                                          real_t *y,
+                                          const int d1d,
+                                          const int q1d)
+{
+   static constexpr int VDIM = 3, DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, ne);
+   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, ne);
+   const auto dU = Reshape(du, D1D, D1D, D1D, VDIM, ne);
+   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, ne);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1, r2;
+      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1, g2;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
+
+      kernels::internal::LoadDofs3d(e, D1D, dU, g0);
+      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
+
+      kernels::internal::LoadDofs3d(e, D1D, U, r0);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r2); // u value
+
+      kernels::internal::LoadDofs3d(e, D1D, dU, r0);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // δu value
+
+      kernels::internal::LoadDofs3d(e, D1D, U, g0);
+      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
+
+      for (int qz = 0; qz < Q1D; qz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               // First part of the Jacobian: u·∇δu
+               const future::tensor<real_t, DIM> u_val =
+               {
+                  r2[0][qz][qy][qx],
+                  r2[1][qz][qy][qx],
+                  r2[2][qz][qy][qx]
+               };
+               const future::tensor<real_t, VDIM, DIM> Q_adj = {{
+                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
+                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
+                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
+                  }
+               };
+               const future::tensor<real_t, DIM, DIM> grad_dU = {{
+                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
+                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
+                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
+                  }
+               };
+               const auto one = transpose(grad_dU) * (Q_adj * u_val);
+
+               // Second part of the Jacobian: δu·∇u
+               const future::tensor<real_t, DIM> du_val =
+               {
+                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
+               };
+               const future::tensor<real_t, VDIM, DIM> grad_U = {{
+                     {g2[0][0][qz][qy][qx], g2[1][0][qz][qy][qx], g2[2][0][qz][qy][qx]},
+                     {g2[0][1][qz][qy][qx], g2[1][1][qz][qy][qx], g2[2][1][qz][qy][qx]},
+                     {g2[0][2][qz][qy][qx], g2[1][2][qz][qy][qx], g2[2][2][qz][qy][qx]}
+                  }
+               };
+               const auto two = transpose(grad_U) * (Q_adj * du_val);
+
+               // u⋅∇δu + δu⋅∇u
+               r0[0][qz][qy][qx] = one[0] + two[0];
+               r0[1][qz][qy][qx] = one[1] + two[1];
+               r0[2][qz][qy][qx] = one[2] + two[2];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, r0, r1);
+      kernels::internal::WriteDofs3d(e, D1D, r1, Y);
+   });
+}
+
+} // namespace internal
+
+template<int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::AddMultGradPAType
+VectorConvectionNLFIntegrator::AddMultGradPA2D::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   return internal::SmemPAConvectionNLGradApply2D<T_D1D, T_Q1D>;
+}
+
+inline VectorConvectionNLFIntegrator::AddMultGradPAType
+VectorConvectionNLFIntegrator::AddMultGradPA2D::Fallback(int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   return internal::SmemPAConvectionNLGradApply2D<>;
+}
+
+template<int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::AddMultGradPAType
+VectorConvectionNLFIntegrator::AddMultGradPA3D::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   return internal::SmemPAConvectionNLGradApply3D<T_D1D, T_Q1D>;
+}
+
+inline VectorConvectionNLFIntegrator::AddMultGradPAType
+VectorConvectionNLFIntegrator::AddMultGradPA3D::Fallback(int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   return internal::SmemPAConvectionNLGradApply3D<>;
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/intrules.cpp

@@ -542,7 +542,10 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
          return;
    }
 
-#ifndef MFEM_USE_MPFR
+#ifdef MFEM_USE_MPFR
+   MFEM_WARNING("MPFR implementation of Gauss-Jacobi quadrature not implemented yet. Falling "
+                "back to double precision implementation...");
+#endif
 
    const int n = np;
    // common constants for Jacobi polynomials
@@ -611,13 +614,6 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
                                                 ab + 1) / ((1.0 - xi*xi)*pp*pp) / pow(2, ab);
       // map nodes and weights to the interval [0,1]
    }
-
-#else // MFEM_USE_MPFR is defined
-
-   MFEM_ABORT("MPFR implementation of Gauss-Jacobi quadrature not defined yet");
-
-#endif // MFEM_USE_MPFR
-
 }
 
 

fem/kernel_dispatch.hpp

@@ -61,7 +61,7 @@ namespace mfem
 #define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
 
-// Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
+// Version of MFEM_REGISTER_KERNELS with optional (non-dispatch)
 // parameters (e.g. NBZ).
 #define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \

fem/kernels.hpp

@@ -83,7 +83,7 @@ constexpr int SetMaxOf(int n) { return NextMultipleOf<4>(n); }
 #endif // CUDA/HIP && DEVICE_COMPILE
 
 /// Load 2D matrix into shared memory
-template <int MQ1>
+template <int MQ1, bool TRANSPOSE = false>
 inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
                                         const real_t *M, real_t (*N)[MQ1])
 {
@@ -91,7 +91,14 @@ inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
    {
       MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
       {
-         N[dy][qx] = M[dy * q1d + qx];
+         if constexpr (TRANSPOSE)
+         {
+            N[dy][qx] = M[qx * d1d + dy];
+         }
+         else
+         {
+            N[dy][qx] = M[dy * q1d + qx];
+         }
       }
    }
    MFEM_SYNC_THREAD;

fem/nonlinearform_ext.cpp

@@ -100,6 +100,17 @@ PANonlinearFormExtension::Gradient::Gradient(const PANonlinearFormExtension &e):
 
 void PANonlinearFormExtension::Gradient::AssembleGrad(const Vector &g)
 {
+   if (DeviceCanUseCeed())
+   {
+      for (int i = 0; i < ext.dnfi.Size(); ++i)
+      {
+         MFEM_VERIFY(dynamic_cast<VectorConvectionNLFIntegrator *>
+                     (ext.dnfi[i]) == nullptr,
+                     "VectorConvectionNLFIntegrator PA gradients are not supported "
+                     "with the libCEED backend");
+      }
+   }
+
    ext.elemR->Mult(g, ext.xe);
    for (int i = 0; i < ext.dnfi.Size(); ++i)
    {

fem/nonlininteg.cpp

@@ -954,4 +954,74 @@ void SkewSymmetricVectorConvectionNLFIntegrator::AssembleElementGrad(
    }
 }
 
+void ConvectiveVectorConvectionNLFIntegrator::AssemblePA(
+   const FiniteElementSpace &)
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void ConvectiveVectorConvectionNLFIntegrator::AssembleGradPA(
+   const Vector &, const FiniteElementSpace &)
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void ConvectiveVectorConvectionNLFIntegrator::AddMultPA(
+   const Vector &, Vector &) const
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void ConvectiveVectorConvectionNLFIntegrator::AddMultGradPA(
+   const Vector &, Vector &) const
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void ConvectiveVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
+   Vector &) const
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AssemblePA(
+   const FiniteElementSpace &)
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradPA(
+   const Vector &, const FiniteElementSpace &)
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AddMultPA(
+   const Vector &, Vector &) const
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AddMultGradPA(
+   const Vector &, Vector &) const
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
+   Vector &) const
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
 }

fem/nonlininteg.hpp

@@ -18,6 +18,7 @@
 #include "fespace.hpp"
 #include "ceed/interface/operator.hpp"
 #include "integrator.hpp"
+#include "kernel_dispatch.hpp"
 
 namespace mfem
 {
@@ -384,15 +385,17 @@ private:
    DenseMatrix dshape, dshapex, EF, gradEF, ELV, elmat_comp;
    Vector shape;
    // PA extension
-   Vector pa_data;
+   int dim, ne, nq, d1d, q1d;
+   Vector pa_adj, pa_u;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq;
 
 public:
-   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { }
+   struct Kernels { Kernels(); };
 
-   VectorConvectionNLFIntegrator() = default;
+   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { static Kernels kernels; }
+
+   VectorConvectionNLFIntegrator() { static Kernels kernels; }
 
    static const IntegrationRule &GetRule(const FiniteElement &fe,
                                          const ElementTransformation &T);
@@ -411,12 +414,55 @@ public:
 
    void AssemblePA(const FiniteElementSpace &fes) override;
 
-   void AssembleMF(const FiniteElementSpace &fes) override;
+   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
 
    void AddMultPA(const Vector &x, Vector &y) const override;
 
-   void AddMultMF(const Vector &x, Vector &y) const override;
+   using AddMultPAType =
+      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
+              const real_t *x, real_t *y,
+              const int d1d, const int q1d);
+   MFEM_REGISTER_KERNELS(AddMultPAKernels, AddMultPAType, (int, int, int));
 
+   void AddMultGradPA(const Vector &x, Vector &y) const override;
+
+   using AddMultGradPAType =
+      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
+              const real_t *u, const real_t *x, real_t *y,
+              const int d1d, const int q1d);
+
+   MFEM_REGISTER_KERNELS(AddMultGradPA2D, AddMultGradPAType, (int, int));
+   MFEM_REGISTER_KERNELS(AddMultGradPA3D, AddMultGradPAType, (int, int));
+
+   void AssembleGradDiagonalPA(Vector &) const override;
+
+   using GradDiagPAType =
+      void (*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
+               const real_t *u, real_t *y,
+               const int d1d, const int q1d);
+
+   MFEM_REGISTER_KERNELS(GradDiagPA2D, GradDiagPAType, (int, int));
+   MFEM_REGISTER_KERNELS(GradDiagPA3D, GradDiagPAType, (int, int));
+
+   template <int DIM, int D1D, int Q1D>
+   static void AddSpecialization()
+   {
+      AddMultPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+      if constexpr (DIM == 2)
+      {
+         AddMultGradPA2D::Specialization<D1D, Q1D>::Add();
+         GradDiagPA2D::Specialization<D1D, Q1D>::Add();
+      }
+      else if constexpr (DIM == 3)
+      {
+         AddMultGradPA3D::Specialization<D1D, Q1D>::Add();
+         GradDiagPA3D::Specialization<D1D, Q1D>::Add();
+      }
+   }
+
+   void AssembleMF(const FiniteElementSpace &fes) override;
+
+   void AddMultMF(const Vector &x, Vector &y) const override;
 
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
@@ -430,7 +476,8 @@ protected:
 
 
 /** This class is used to assemble the convective form of the nonlinear term
-    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$ */
+    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$.
+    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
 class ConvectiveVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -448,12 +495,20 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
+
+   using NonlinearFormIntegrator::AssemblePA;
+   void AssemblePA(const FiniteElementSpace &fes) override;
+   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
+   void AddMultPA(const Vector &x, Vector &y) const override;
+   void AddMultGradPA(const Vector &x, Vector &y) const override;
+   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 
 /** This class is used to assemble the skew-symmetric form of the nonlinear term
     arising in the Navier-Stokes equations
-    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$ */
+    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$.
+    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
 class SkewSymmetricVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -471,6 +526,13 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
+
+   using NonlinearFormIntegrator::AssemblePA;
+   void AssemblePA(const FiniteElementSpace &fes) override;
+   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
+   void AddMultPA(const Vector &x, Vector &y) const override;
+   void AddMultGradPA(const Vector &x, Vector &y) const override;
+   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 }

fem/pgridfunc.cpp

@@ -22,10 +22,20 @@ using namespace std;
 namespace mfem
 {
 
-ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf)
+ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
+                                 bool preserve)
 {
    fes = pfes = pf;
    SetDataAndSize(gf->GetData(), gf->Size());
+
+   if (pfes->HaveDofSigns())
+   {
+      MFEM_VERIFY(!preserve, "Differing sign conventions for the serial and "
+                  "parallel grid functions will prevent preserving the serial "
+                  "GridFunctions in this context.");
+
+      pfes->ApplyDofSigns(HostReadWrite());
+   }
 }
 
 ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, HypreParVector *tv)

fem/pgridfunc.hpp

@@ -100,8 +100,12 @@ public:
    /// Construct a ParGridFunction using a GridFunction as external data.
    /** The parallel space @a *pf and the space used by @a *gf should match. The
        data from @a *gf is used as the local data of the ParGridFunction on each
-       processor. The ParGridFunction does not assume ownership of the data. */
-   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf);
+       processor. The ParGridFunction does not assume ownership of the data.
+       The boolean, @a preserve, indicates that the data stored in @a *gf should
+       remain unchanged. An error will occur if @a preserve is true and
+       construction of a valid ParGridFunction requires the data to change. */
+   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
+                   bool preserve = true);
 
    /** @brief Creates grid function on (all) dofs from a given vector on the
        true dofs, i.e. P tv. */

fem/qinterp/det.cpp

@@ -21,24 +21,23 @@ namespace quadrature_interpolator
 
 void InitDetKernels()
 {
-   using k = QuadratureInterpolator::DetKernels;
    // 2D
-   k::Specialization<2,2,2,2>::Add();
-   k::Specialization<2,2,2,3>::Add();
-   k::Specialization<2,2,2,4>::Add();
-   k::Specialization<2,2,2,6>::Add();
-   k::Specialization<2,2,3,4>::Add();
-   k::Specialization<2,2,3,6>::Add();
-   k::Specialization<2,2,4,4>::Add();
-   k::Specialization<2,2,4,6>::Add();
-   k::Specialization<2,2,5,6>::Add();
+   QuadratureInterpolator::AddDetSpecializations<2,2,2,2>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,2,3>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,2,4>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,2,6>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,3,4>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,3,6>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,4,4>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,4,6>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,5,6>();
    // 3D
-   k::Specialization<3,3,2,4>::Add();
-   k::Specialization<3,3,3,3>::Add();
-   k::Specialization<3,3,3,5>::Add();
-   k::Specialization<3,3,3,6>::Add();
-   k::Specialization<3,3,4,6>::Add();
-   k::Specialization<3,3,3,4>::Add();
+   QuadratureInterpolator::AddDetSpecializations<3,3,2,4>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,3,3>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,3,5>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,3,6>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,4,6>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,3,4>();
 }
 
 } // namespace quadrature_interpolator
@@ -47,8 +46,8 @@ void InitDetKernels()
 /// @cond Suppress_Doxygen_warnings
 
 QuadratureInterpolator::DetKernelType
-QuadratureInterpolator::DetKernels::Fallback(
-   int DIM, int SDIM, int D1D, int Q1D)
+QuadratureInterpolator::DetKernels::Fallback(int DIM, int SDIM, int D1D,
+                                             int Q1D)
 {
    if (DIM == 1)
    {

fem/qinterp/eval.hpp

@@ -30,23 +30,18 @@ namespace internal
 namespace quadrature_interpolator
 {
 
-template<QVectorLayout Q_LAYOUT>
-static void Values1D(const int NE,
-                     const real_t *b_,
-                     const real_t *x_,
-                     real_t *y_,
-                     const int vdim,
-                     const int d1d,
-                     const int q1d)
+template <QVectorLayout Q_LAYOUT, bool Integral>
+static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
+                         const real_t *x_, real_t *y_, const int vdim,
+                         const int d1d, const int q1d)
 {
-   const auto b = Reshape(b_, q1d, d1d);
-   const auto x = Reshape(x_, d1d, vdim, NE);
-   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
-            Reshape(y_, q1d, vdim, NE):
-            Reshape(y_, vdim, q1d, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
    {
+      const auto b = Reshape(b_, q1d, d1d);
+      const auto x = Reshape(x_, d1d, vdim, NE);
+      const auto detJ = Reshape(detJ_, q1d, NE);
+      auto y = Q_LAYOUT == QVectorLayout::byNODES ? Reshape(y_, q1d, vdim, NE)
+               : Reshape(y_, vdim, q1d, NE);
       for (int c = 0; c < vdim; c++)
       {
          for (int q = 0; q < q1d; q++)
@@ -56,24 +51,36 @@ static void Values1D(const int NE,
             {
                u += b(q, d) * x(d, c, e);
             }
-            if (Q_LAYOUT == QVectorLayout::byVDIM)  { y(c, q, e) = u; }
-            if (Q_LAYOUT == QVectorLayout::byNODES) { y(q, c, e) = u; }
+            if constexpr (Integral)
+            {
+               u /= detJ(q, e);
+            }
+            if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
+            {
+               y(c, q, e) = u;
+            }
+            if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
+            {
+               y(q, c, e) = u;
+            }
          }
       }
    });
 }
 
+template <QVectorLayout Q_LAYOUT>
+static void Values1D(const int NE, const real_t *b_, const real_t *x_,
+                     real_t *y_, const int vdim, const int d1d, const int q1d)
+{
+   ImplValues1D<Q_LAYOUT, false>(NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
+}
+
 // Template compute kernel for Values in 2D: tensor product version.
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1>
-static void Values2D(const int NE,
-                     const real_t *b_,
-                     const real_t *x_,
-                     real_t *y_,
-                     const int vdim = 0,
-                     const int d1d = 0,
-                     const int q1d = 0)
+template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
+          int T_Q1D = 0, int T_NBZ = 1>
+static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
+                         const real_t *x_, real_t *y_, const int vdim = 0,
+                         const int d1d = 0, const int q1d = 0)
 {
    static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
 
@@ -82,13 +89,14 @@ static void Values2D(const int NE,
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
-   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
-            Reshape(y_, Q1D, Q1D, VDIM, NE):
-            Reshape(y_, VDIM, Q1D, Q1D, NE);
 
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int e)
    {
+      const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+      const auto detJ = Reshape(detJ_, Q1D, Q1D, NE);
+      auto y = Q_LAYOUT == QVectorLayout::byNODES
+               ? Reshape(y_, Q1D, Q1D, VDIM, NE)
+               : Reshape(y_, VDIM, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -110,16 +118,33 @@ static void Values2D(const int NE,
 
       for (int c = 0; c < VDIM; c++)
       {
-         kernels::internal::LoadX(e,D1D,c,x,DD);
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx, x, D1D)
+            {
+               DD(dx, dy) = x(dx, dy, c, e);
+            }
+         }
+         MFEM_SYNC_THREAD;
          kernels::internal::EvalX(D1D,Q1D,B,DD,DQ);
          kernels::internal::EvalY(D1D,Q1D,B,DQ,QQ);
          MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
             MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               real_t u = QQ(qx,qy);
-               if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,e) = u; }
-               if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,c,e) = u; }
+               real_t u = QQ(qx, qy);
+               if constexpr (Integral)
+               {
+                  u /= detJ(qx, qy, e);
+               }
+               if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
+               {
+                  y(c, qx, qy, e) = u;
+               }
+               if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
+               {
+                  y(qx, qy, c, e) = u;
+               }
             }
          }
          MFEM_SYNC_THREAD;
@@ -127,29 +152,37 @@ static void Values2D(const int NE,
    });
 }
 
-// Template compute kernel for Values in 3D: tensor product version.
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void Values3D(const int NE,
-                     const real_t *b_,
-                     const real_t *x_,
-                     real_t *y_,
-                     const int vdim = 0,
-                     const int d1d = 0,
+// Template compute kernel for Values in 2D: tensor product version.
+template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+          int T_NBZ = 1>
+static void Values2D(const int NE, const real_t *b_, const real_t *x_,
+                     real_t *y_, const int vdim = 0, const int d1d = 0,
                      const int q1d = 0)
+{
+   return ImplValues2D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D, T_NBZ>(
+             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
+}
+
+// Template compute kernel for Values in 3D: tensor product version.
+template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
+          int T_Q1D = 0>
+static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
+                         const real_t *x_, real_t *y_, const int vdim = 0,
+                         const int d1d = 0, const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
-   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
-            Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE):
-            Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
 
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
    {
+      const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+      const auto detJ = Reshape(detJ_, Q1D, Q1D, Q1D, NE);
+      auto y = Q_LAYOUT == QVectorLayout::byNODES
+               ? Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE)
+               : Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -171,7 +204,17 @@ static void Values3D(const int NE,
 
       for (int c = 0; c < VDIM; c++)
       {
-         kernels::internal::LoadX(e,D1D,c,x,DDD);
+         MFEM_FOREACH_THREAD(dz, z, D1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  DDD(dx, dy, dz) = x(dx, dy, dz, c, e);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
          kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
          kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
          kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);
@@ -181,9 +224,19 @@ static void Values3D(const int NE,
             {
                MFEM_FOREACH_THREAD(qx,x,Q1D)
                {
-                  const real_t u = QQQ(qz,qy,qx);
-                  if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,qz,e) = u; }
-                  if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,qz,c,e) = u; }
+                  real_t u = QQQ(qz,qy,qx);
+                  if constexpr (Integral)
+                  {
+                     u /= detJ(qx, qy, qz, e);
+                  }
+                  if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     y(c, qx, qy, qz, e) = u;
+                  }
+                  if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
+                  {
+                     y(qx, qy, qz, c, e) = u;
+                  }
                }
             }
          }
@@ -192,14 +245,431 @@ static void Values3D(const int NE,
    });
 }
 
+// Template compute kernel for Values in 3D: tensor product version.
+template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void Values3D(const int NE, const real_t *b_, const real_t *x_,
+                     real_t *y_, const int vdim = 0, const int d1d = 0,
+                     const int q1d = 0)
+{
+   return ImplValues3D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D>(
+             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
+}
+
+template <bool Integral>
+void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
+                const real_t *detJ, const GeometricFactors *geom,
+                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                Vector &q_der, Vector &q_det, const int eval_flags);
+
+inline void Eval1D(const int NE, const int vdim, const QVectorLayout q_layout,
+                   const GeometricFactors *geom, const DofToQuad &maps,
+                   const Vector &e_vec, Vector &q_val, Vector &q_der, Vector &q_det,
+                   const int eval_flags)
+{
+   ImplEval1D<false>(NE, vdim, q_layout, nullptr, geom, maps, e_vec, q_val,
+                     q_der, q_det, eval_flags);
+}
+
+// Template compute kernel for 2D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
+static void ImplEval2D(const int NE, const int vdim,
+                       const QVectorLayout q_layout, const real_t *detJ_,
+                       const GeometricFactors *geom, const DofToQuad &maps,
+                       const Vector &e_vec, Vector &q_val, Vector &q_der,
+                       Vector &q_det, const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
+
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
+   if constexpr(Integral)
+   {
+      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
+                                  QI::DETERMINANTS)),
+                  "Integral FE does not support computing derivatives");
+   }
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
+   const auto E_ = e_vec.Read();
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
+              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const auto E = Reshape(E_, ND, VDIM, NE);
+      const auto detJ = Reshape(detJ_, NQ, NE);
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c + d * VDIM] = E(d, c, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++)
+            {
+               ed[c] = 0.0;
+            }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  ed[c] += b * s_E[c + d * VDIM];
+               }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if constexpr (Integral)
+               {
+                  ed[c] /= detJ(q, e);
+               }
+               if (q_layout == QVectorLayout::byVDIM)
+               {
+                  val(c, q, e) = ed[c];
+               }
+               if (q_layout == QVectorLayout::byNODES)
+               {
+                  val(q, c, e) = ed[c];
+               }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM2D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM2D*2];
+            for (int i = 0; i < 2*VDIM; i++)
+            {
+               D[i] = 0.0;
+            }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[4], Jinv[4];
+               Jloc[0] = J(q,0,0,e);
+               Jloc[1] = J(q,1,0,e);
+               Jloc[2] = J(q,0,1,e);
+               Jloc[3] = J(q,1,1,e);
+               kernels::CalcInverse<2>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
+                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                  }
+               }
+            }
+            if (eval_flags & QI::DETERMINANTS)
+            {
+               if (VDIM == 2)
+               {
+                  det(q, e) = kernels::Det<2>(D);
+               }
+               else
+               {
+                  DeviceTensor<2> j(D, 3, 2);
+                  const real_t dE = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
+                  const real_t dF = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
+                  const real_t dG = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
+                  det(q,e) = std::sqrt(dE*dG - dF*dF);
+               }
+            }
+         }
+      }
+   });
+}
+
+// Template compute kernel for 2D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template <const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval2D(const int NE, const int vdim, const QVectorLayout q_layout,
+                   const GeometricFactors *geom, const DofToQuad &maps,
+                   const Vector &e_vec, Vector &q_val, Vector &q_der,
+                   Vector &q_det, const int eval_flags)
+{
+   ImplEval2D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
+                                         maps, e_vec, q_val, q_der, q_det,
+                                         eval_flags);
+}
+
+// Template compute kernel for 3D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
+static void ImplEval3D(const int NE, const int vdim,
+                       const QVectorLayout q_layout, const real_t *detJ_,
+                       const GeometricFactors *geom, const DofToQuad &maps,
+                       const Vector &e_vec, Vector &q_val, Vector &q_der,
+                       Vector &q_det, const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
+
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
+   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
+   if constexpr(Integral)
+   {
+      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
+                                  QI::DETERMINANTS)),
+                  "Integral FE does not support computing derivatives");
+   }
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
+   auto E_ = e_vec.Read();
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
+              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const auto E = Reshape(E_, ND, VDIM, NE);
+      const auto detJ = Reshape(detJ_, NQ, NE);
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c + d * VDIM] = E(d, c, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++)
+            {
+               ed[c] = 0.0;
+            }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  ed[c] += b * s_E[c + d * VDIM];
+               }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if constexpr (Integral)
+               {
+                  ed[c] /= detJ(q, e);
+               }
+               if (q_layout == QVectorLayout::byVDIM)
+               {
+                  val(c, q, e) = ed[c];
+               }
+               if (q_layout == QVectorLayout::byNODES)
+               {
+                  val(q, c, e) = ed[c];
+               }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM3D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM3D*3];
+            for (int i = 0; i < 3*VDIM; i++)
+            {
+               D[i] = 0.0;
+            }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               const real_t wz = G(q,2,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+                  D[c+VDIM*2] += s_e * wz;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                     der(c,2,q,e) = D[c+VDIM*2];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                     der(q,c,2,e) = D[c+VDIM*2];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[9], Jinv[9];
+               for (int col = 0; col < 3; col++)
+               {
+                  for (int row = 0; row < 3; row++)
+                  {
+                     Jloc[row+3*col] = J(q,row,col,e);
+                  }
+               }
+               kernels::CalcInverse<3>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t w = D[c+VDIM*2];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
+                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
+                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                     der(c,2,q,e) = JiW;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                     der(q,c,2,e) = JiW;
+                  }
+               }
+            }
+            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
+            {
+               // The check (VDIM == 3) should eliminate this block when VDIM is
+               // known at compile time and (VDIM != 3).
+               det(q,e) = kernels::Det<3>(D);
+            }
+         }
+      }
+   });
+}
+
+// Template compute kernel for 3D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template <const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval3D(const int NE, const int vdim, const QVectorLayout q_layout,
+                   const GeometricFactors *geom, const DofToQuad &maps,
+                   const Vector &e_vec, Vector &q_val, Vector &q_der,
+                   Vector &q_det, const int eval_flags)
+{
+   ImplEval3D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
+                                         maps, e_vec, q_val, q_der, q_det,
+                                         eval_flags);
+}
+
 } // namespace quadrature_interpolator
 
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-template<int DIM, QVectorLayout Q_LAYOUT,
-         int VDIM, int D1D, int Q1D, int NBZ>
+template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
+QuadratureInterpolator::IntTensorEvalKernelType
+QuadratureInterpolator::IntTensorEvalKernels::Kernel()
+{
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::ImplValues1D<Q_LAYOUT, true>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::ImplValues2D<Q_LAYOUT, true, VDIM, D1D, Q1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::ImplValues3D<Q_LAYOUT, true, VDIM, D1D, Q1D>; }
+   MFEM_ABORT("");
+}
+
+template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
 QuadratureInterpolator::TensorEvalKernelType
 QuadratureInterpolator::TensorEvalKernels::Kernel()
 {
@@ -209,6 +679,28 @@ QuadratureInterpolator::TensorEvalKernels::Kernel()
    MFEM_ABORT("");
 }
 
+template <int DIM, int VDIM, int ND, int NQ>
+QuadratureInterpolator::IntEvalKernelType
+QuadratureInterpolator::IntEvalKernels::Kernel()
+{
+   using namespace internal::quadrature_interpolator;
+   if constexpr (DIM == 1) { return ImplEval1D<true>; }
+   else if constexpr (DIM == 2) { return ImplEval2D<true,VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return ImplEval3D<true,VDIM,ND,NQ>; }
+   MFEM_ABORT("");
+}
+
+template <int DIM, int VDIM, int ND, int NQ>
+QuadratureInterpolator::EvalKernelType
+QuadratureInterpolator::EvalKernels::Kernel()
+{
+   using namespace internal::quadrature_interpolator;
+   if constexpr (DIM == 1) { return Eval1D; }
+   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
+   MFEM_ABORT("");
+}
+
 /// @endcond
 
 } // namespace mfem

fem/qinterp/eval_by_nodes.cpp

@@ -21,63 +21,105 @@ namespace quadrature_interpolator
 
 void InitEvalByNodesKernels()
 {
-   using k = QuadratureInterpolator::TensorEvalKernels;
-
    // 2D
-   k::Specialization<2,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,3,2>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 3, 2, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 3, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 3, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 4, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 4, 4, 1>();
 
-   k::Specialization<2,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,2,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,2,5>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,2,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 2, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 6, 1>();
 
-   k::Specialization<2,QVectorLayout::byNODES,2,3,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,3,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 3, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 3, 6, 1>();
 
-   k::Specialization<2,QVectorLayout::byNODES,2,4,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,4,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,4,5>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,4,6>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,4,7>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 7, 1>();
 
-   k::Specialization<2,QVectorLayout::byNODES,2,5,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 5, 6, 1>();
 
    // 3D
-   k::Specialization<3,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,4,8>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 3, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 3, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 4, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 4, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 4, 8, 1>();
 
-   k::Specialization<3,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 2, 2, 2, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 2, 2, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 2, 3, 4, 1>();
 
-   k::Specialization<3,QVectorLayout::byNODES,3,2,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,2,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,2,5>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,2,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 2, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 2, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 2, 6, 1>();
 
-   k::Specialization<3,QVectorLayout::byNODES,3,3,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,3,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,3,5>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,3,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 3, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 3, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 3, 6, 1>();
 
-   k::Specialization<3,QVectorLayout::byNODES,3,4,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,4,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,4,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,4,7>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,4,8>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 7, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 8, 1>();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/eval_by_vdim.cpp

@@ -21,36 +21,59 @@ namespace quadrature_interpolator
 
 void InitEvalByVDimKernels()
 {
-   using k = QuadratureInterpolator::TensorEvalKernels;
    // 2D
-   k::Specialization<2,QVectorLayout::byVDIM,1,2,4>::Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,1,3,6>::Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,1,4,8>::Opt<2>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 1, 2, 4, 8>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 1, 3, 6, 4>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 1, 4, 8, 2>();
 
-   k::Specialization<2,QVectorLayout::byVDIM,2,2,4>::Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,2,3,4>::Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,2,3,6>::Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,2,4,6>::Opt<2>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,2,4,8>::Opt<2>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 2, 4, 8>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 3, 4, 8>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 3, 6, 4>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 4, 6, 2>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 4, 8, 2>();
    // 3D
-   k::Specialization<3,QVectorLayout::byVDIM,1,2,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,1,3,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,1,4,8>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,2,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,3,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,4,8>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 1, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 1, 3, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 1, 4, 8, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 3, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 4, 8, 1>();
 
-   k::Specialization<3,QVectorLayout::byVDIM,3,2,2>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,3,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,4,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,5,5>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,6,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,7,7>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,8,8>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,9,9>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 2, 2, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 4, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 5, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 6, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 7, 7, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 8, 8, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 9, 9, 1>();
 
-   k::Specialization<3,QVectorLayout::byVDIM,3,4,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,3,4>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 4, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 3, 4, 1>();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/grad.hpp

@@ -268,8 +268,9 @@ static void Derivatives3D(const int NE,
       DeviceMatrix B(BG[0], D1D, Q1D);
       DeviceMatrix G(BG[1], D1D, Q1D);
 
-      MFEM_SHARED real_t sm0[3][MQ1*MQ1*MQ1];
-      MFEM_SHARED real_t sm1[3][MQ1*MQ1*MQ1];
+      constexpr int MDQ = MD1 > MQ1 ? MD1 : MQ1;
+      MFEM_SHARED real_t sm0[3][MD1*MD1*MDQ];
+      MFEM_SHARED real_t sm1[3][MD1*MQ1*MQ1];
       DeviceTensor<3> X(sm0[2], D1D, D1D, D1D);
       DeviceTensor<3> DDQ0(sm0[0], D1D, D1D, Q1D);
       DeviceTensor<3> DDQ1(sm0[1], D1D, D1D, Q1D);

fem/qinterp/grad_by_nodes.cpp

@@ -22,74 +22,71 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByNodesKernels()
 {
-   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byNODES;
    // 2D
-   k::Specialization<2,L,P,1,3,3>::template Opt<16>::Add();
-   k::Specialization<2,L,P,1,3,4>::template Opt<16>::Add();
-   k::Specialization<2,L,P,1,4,3>::template Opt<16>::Add();
-   k::Specialization<2,L,P,1,4,4>::template Opt<16>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,3,16>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,16>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,3,16>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,4,16>();
 
-   k::Specialization<2,L,P,2,2,2>::template Opt<16>::Add();
-   k::Specialization<2,L,P,2,2,3>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,2,4>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,2,5>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,2,6>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,2,16>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,3,8>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,4,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,5,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,6,2>();
 
-   k::Specialization<2,L,P,2,3,3>::template Opt<2>::Add();
-   k::Specialization<2,L,P,2,3,4>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,4,3>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,3,6>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,2>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,6,2>();
 
-   k::Specialization<2,L,P,2,4,4>::template Opt<2>::Add();
-   k::Specialization<2,L,P,2,4,5>::template Opt<2>::Add();
-   k::Specialization<2,L,P,2,4,6>::template Opt<2>::Add();
-   k::Specialization<2,L,P,2,4,7>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,4,2>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,5,2>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,2>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,7,2>();
 
-   k::Specialization<2,L,P,2,5,6>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,6,2>();
    // 3D
-   k::Specialization<3,L,P,1,2,4>::Add();
-   k::Specialization<3,L,P,1,3,3>::Add();
-   k::Specialization<3,L,P,1,3,4>::Add();
-   k::Specialization<3,L,P,1,3,6>::Add();
-   k::Specialization<3,L,P,1,4,4>::Add();
-   k::Specialization<3,L,P,1,4,8>::Add();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,2,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,3>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,8>();
 
-   k::Specialization<3,L,P,3,2,3>::Add();
-   k::Specialization<3,L,P,3,2,4>::Add();
-   k::Specialization<3,L,P,3,2,5>::Add();
-   k::Specialization<3,L,P,3,2,6>::Add();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,3>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,5>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,6>();
 
-   k::Specialization<3,L,P,3,3,3>::Add();
-   k::Specialization<3,L,P,3,3,4>::Add();
-   k::Specialization<3,L,P,3,3,5>::Add();
-   k::Specialization<3,L,P,3,3,6>::Add();
-   k::Specialization<3,L,P,3,4,4>::Add();
-   k::Specialization<3,L,P,3,4,6>::Add();
-   k::Specialization<3,L,P,3,4,7>::Add();
-   k::Specialization<3,L,P,3,4,8>::Add();
-
-   using k2 = QuadratureInterpolator::CollocatedGradKernels;
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,3>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,5>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,7>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,8>();
 
    // 2D
-   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
-   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
 
-   k2::Specialization<3,L,P,1,2>::Add();
-   k2::Specialization<3,L,P,1,3>::Add();
-   k2::Specialization<3,L,P,1,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
 
-   k2::Specialization<3,L,P,2,2>::Add();
-   k2::Specialization<3,L,P,2,3>::Add();
-   k2::Specialization<3,L,P,2,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
 
-   k2::Specialization<3,L,P,3,2>::Add();
-   k2::Specialization<3,L,P,3,3>::Add();
-   k2::Specialization<3,L,P,3,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
 }
 
 template void InitGradByNodesKernels<true>();

fem/qinterp/grad_by_vdim.cpp

@@ -22,47 +22,45 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByVDimKernels()
 {
-   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byVDIM;
    // 2D
-   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,8>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,6,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,5,8,2>();
 
-   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,8>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,8>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,8,2>();
    // 3D
-   k::Specialization<3,L,P,1,3,4>::Add();
-   k::Specialization<3,L,P,1,4,6>::Add();
-   k::Specialization<3,L,P,1,5,8>::Add();
-   k::Specialization<3,L,P,3,3,4>::Add();
-   k::Specialization<3,L,P,3,4,6>::Add();
-   k::Specialization<3,L,P,3,5,8>::Add();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,5,8>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,5,8>();
 
-   using k2 = QuadratureInterpolator::CollocatedGradKernels;
    // 2D
-   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
 
-   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
-   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
 
    // 3D
-   k2::Specialization<3,L,P,1,2>::Add();
-   k2::Specialization<3,L,P,1,3>::Add();
-   k2::Specialization<3,L,P,1,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
 
-   k2::Specialization<3,L,P,2,2>::Add();
-   k2::Specialization<3,L,P,2,3>::Add();
-   k2::Specialization<3,L,P,2,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
 
-   k2::Specialization<3,L,P,3,2>::Add();
-   k2::Specialization<3,L,P,3,3>::Add();
-   k2::Specialization<3,L,P,3,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
 }
 
 template void InitGradByVDimKernels<true>();

fem/quadinterpolator.cpp

@@ -69,8 +69,9 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(SupportsFESpace(fes),
-               "Only elements with MapType VALUE and H_DIV are supported!");
+   MFEM_VERIFY(
+      SupportsFESpace(fes),
+      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
 }
 
 QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
@@ -84,8 +85,9 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 {
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(SupportsFESpace(fes),
-               "Only elements with MapType VALUE and H_DIV are supported!");
+   MFEM_VERIFY(
+      SupportsFESpace(fes),
+      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
 }
 
 bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
@@ -93,9 +95,9 @@ bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
    const FiniteElement *fe = fespace.GetTypicalFE();
    const Mesh &mesh = *fespace.GetMesh();
    return (fe->GetMapType() == FiniteElement::MapType::VALUE ||
-           fe->GetMapType() == FiniteElement::MapType::H_DIV)
-          && (!fespace.IsVariableOrder())
-          && (!mesh.IsMixedMesh());
+           fe->GetMapType() == FiniteElement::MapType::INTEGRAL ||
+           fe->GetMapType() == FiniteElement::MapType::H_DIV) &&
+          (!fespace.IsVariableOrder()) && (!mesh.IsMixedMesh());
 }
 
 namespace internal
@@ -108,16 +110,11 @@ namespace quadrature_interpolator
 // * non-tensor product version,
 // * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
 // * assumes 'maps.mode == FULL'.
-static void Eval1D(const int NE,
-                   const int vdim,
-                   const QVectorLayout q_layout,
-                   const GeometricFactors *geom,
-                   const DofToQuad &maps,
-                   const Vector &e_vec,
-                   Vector &q_val,
-                   Vector &q_der,
-                   Vector &q_det,
-                   const int eval_flags)
+template <bool Integral>
+void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
+                const real_t *detJ_, const GeometricFactors *geom,
+                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                Vector &q_der, Vector &q_det, const int eval_flags)
 {
    using QI = QuadratureInterpolator;
 
@@ -126,13 +123,16 @@ static void Eval1D(const int NE,
    MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
    MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 1, "");
    MFEM_VERIFY(vdim == 1 || !(eval_flags & QI::DETERMINANTS), "");
-   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
-               "'geom' must be given (non-null) only when evaluating physical"
-               " derivatives");
-   const auto B = Reshape(maps.B.Read(), nq, nd);
-   const auto G = Reshape(maps.G.Read(), nq, nd);
+   if constexpr(Integral)
+   {
+      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
+                                  QI::DETERMINANTS)),
+                  "Integral FE does not support computing derivatives");
+   }
+   const auto B_ = maps.B.Read();
+   const auto G_ = maps.G.Read();
    const auto J = Reshape(geom ? geom->J.Read() : nullptr, nq, NE);
-   const auto E = Reshape(e_vec.Read(), nd, vdim, NE);
+   const auto E_ = e_vec.Read();
    auto val = q_layout == QVectorLayout::byNODES ?
               Reshape(q_val.Write(), nq, vdim, NE):
               Reshape(q_val.Write(), vdim, nq, NE);
@@ -140,8 +140,12 @@ static void Eval1D(const int NE,
               Reshape(q_der.Write(), nq, vdim, NE):
               Reshape(q_der.Write(), vdim, nq, NE);
    auto det = Reshape(q_det.Write(), nq, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
    {
+      const auto B = Reshape(B_, nq, nd);
+      const auto G = Reshape(G_, nq, nd);
+      const auto E = Reshape(E_, nd, vdim, NE);
+      const auto detJ = Reshape(detJ_, nq, NE);
       for (int q = 0; q < nq; ++q)
       {
          if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
@@ -151,10 +155,20 @@ static void Eval1D(const int NE,
                real_t q_val = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_val += B(q,d)*E(d,c,e);
+                  q_val += B(q, d) * E(d, c, e);
+               }
+               if constexpr (Integral)
+               {
+                  q_val /= detJ(q, e);
+               }
+               if (q_layout == QVectorLayout::byVDIM)
+               {
+                  val(c, q, e) = q_val;
+               }
+               if (q_layout == QVectorLayout::byNODES)
+               {
+                  val(q, c, e) = q_val;
                }
-               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = q_val; }
-               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = q_val; }
             }
          }
          if ((eval_flags & QI::DERIVATIVES) ||
@@ -166,7 +180,7 @@ static void Eval1D(const int NE,
                real_t q_d = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_d += G(q,d)*E(d,c,e);
+                  q_d += G(q, d) * E(d, c, e);
                }
                if (eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
@@ -174,8 +188,14 @@ static void Eval1D(const int NE,
                }
                if (eval_flags & QI::DERIVATIVES || eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
-                  if (q_layout == QVectorLayout::byVDIM) { der(c,q,e) = q_d; }
-                  if (q_layout == QVectorLayout::byNODES) { der(q,c,e) = q_d; }
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c, q, e) = q_d;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q, c, e) = q_d;
+                  }
                }
                if (vdim == 1 && (eval_flags & QI::DETERMINANTS))
                {
@@ -187,317 +207,17 @@ static void Eval1D(const int NE,
    });
 }
 
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template<const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval2D(const int NE,
-                   const int vdim,
-                   const QVectorLayout q_layout,
-                   const GeometricFactors *geom,
-                   const DofToQuad &maps,
-                   const Vector &e_vec,
-                   Vector &q_val,
-                   Vector &q_der,
-                   Vector &q_det,
-                   const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
+template void
+ImplEval1D<true>(const int NE, const int vdim, const QVectorLayout q_layout,
+                 const real_t *detJ, const GeometricFactors *geom,
+                 const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                 Vector &q_der, Vector &q_det, const int eval_flags);
 
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
-   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
-               "'geom' must be given (non-null) only when evaluating physical"
-               " derivatives");
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
-   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
-              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c+d*VDIM] = E(d,c,e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
-               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM2D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM2D*2];
-            for (int i = 0; i < 2*VDIM; i++) { D[i] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[4], Jinv[4];
-               Jloc[0] = J(q,0,0,e);
-               Jloc[1] = J(q,1,0,e);
-               Jloc[2] = J(q,0,1,e);
-               Jloc[3] = J(q,1,1,e);
-               kernels::CalcInverse<2>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
-                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                  }
-               }
-            }
-            if (eval_flags & QI::DETERMINANTS)
-            {
-               if (VDIM == 2) { det(q,e) = kernels::Det<2>(D); }
-               else
-               {
-                  DeviceTensor<2> j(D, 3, 2);
-                  const double E = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
-                  const double F = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
-                  const double G = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
-                  det(q,e) = std::sqrt(E*G - F*F);
-               }
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template<const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval3D(const int NE,
-                   const int vdim,
-                   const QVectorLayout q_layout,
-                   const GeometricFactors *geom,
-                   const DofToQuad &maps,
-                   const Vector &e_vec,
-                   Vector &q_val,
-                   Vector &q_der,
-                   Vector &q_det,
-                   const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
-   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
-   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
-               "'geom' must be given (non-null) only when evaluating physical"
-               " derivatives");
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
-   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
-              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c+d*VDIM] = E(d,c,e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
-               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM3D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM3D*3];
-            for (int i = 0; i < 3*VDIM; i++) { D[i] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               const real_t wz = G(q,2,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-                  D[c+VDIM*2] += s_e * wz;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                     der(c,2,q,e) = D[c+VDIM*2];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                     der(q,c,2,e) = D[c+VDIM*2];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[9], Jinv[9];
-               for (int col = 0; col < 3; col++)
-               {
-                  for (int row = 0; row < 3; row++)
-                  {
-                     Jloc[row+3*col] = J(q,row,col,e);
-                  }
-               }
-               kernels::CalcInverse<3>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t w = D[c+VDIM*2];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
-                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
-                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                     der(c,2,q,e) = JiW;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                     der(q,c,2,e) = JiW;
-                  }
-               }
-            }
-            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
-            {
-               // The check (VDIM == 3) should eliminate this block when VDIM is
-               // known at compile time and (VDIM != 3).
-               det(q,e) = kernels::Det<3>(D);
-            }
-         }
-      }
-   });
-}
+template void
+ImplEval1D<false>(const int NE, const int vdim, const QVectorLayout q_layout,
+                  const real_t *detJ, const GeometricFactors *geom,
+                  const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                  Vector &q_der, Vector &q_det, const int eval_flags);
 
 } // namespace quadrature_interpolator
 
@@ -535,10 +255,20 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    const int nd = maps.ndof;
    const int nq = maps.nqpt;
    const GeometricFactors *geom = nullptr;
-   if (eval_flags & PHYSICAL_DERIVATIVES)
    {
-      const int jacobians = GeometricFactors::JACOBIANS;
-      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
+      int jac_factors = 0;
+      if (eval_flags & PHYSICAL_DERIVATIVES)
+      {
+         jac_factors = GeometricFactors::JACOBIANS;
+      }
+      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+      {
+         jac_factors |= GeometricFactors::DETERMINANTS;
+      }
+      if (jac_factors)
+      {
+         geom = fespace->GetMesh()->GetGeometricFactors(*ir, jac_factors);
+      }
    }
 
    MFEM_ASSERT(!(eval_flags & DETERMINANTS) || dim == vdim ||
@@ -552,29 +282,61 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    {
       if (eval_flags & (VALUES | PHYSICAL_VALUES))
       {
-         TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne, maps.B.Read(),
-                                e_vec.Read(), q_val.Write(), vdim, nd, nq);
+         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+         {
+            IntTensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
+                                      maps.B.Read(), geom->detJ.Read(),
+                                      e_vec.Read(), q_val.Write(), vdim, nd, nq);
+         }
+         else
+         {
+            TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
+                                   maps.B.Read(), e_vec.Read(), q_val.Write(),
+                                   vdim, nd, nq);
+         }
       }
       if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
       {
          const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
          const real_t *J = phys ? geom->J.Read() : nullptr;
          const int s_dim = phys ? sdim : dim;
-         GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
-                          maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
-                          q_der.Write(), s_dim, vdim, nd, nq);
+         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+         {
+            MFEM_ABORT("");
+         }
+         else
+         {
+            GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
+                             maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
+                             q_der.Write(), s_dim, vdim, nd, nq);
+         }
       }
       if (eval_flags & DETERMINANTS)
       {
-         DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(),
-                         maps.G.Read(), e_vec.Read(), q_det.Write(), nd,
-                         nq, &d_buffer);
+         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+         {
+            MFEM_ABORT("");
+         }
+         else
+         {
+            DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(), maps.G.Read(),
+                            e_vec.Read(), q_det.Write(), nd, nq, &d_buffer);
+         }
       }
    }
    else // use_tensor_eval == false
    {
-      EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne,vdim, q_layout,
-                       geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
+      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+      {
+         IntEvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim,
+                             q_layout, geom->detJ.Read(), geom, maps, e_vec,
+                             q_val, q_der, q_det, eval_flags);
+      }
+      else
+      {
+         EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim, q_layout,
+                          geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
+      }
    }
 }
 
@@ -700,22 +462,41 @@ namespace
 
 using namespace internal::quadrature_interpolator;
 
-using EvalKernel = QuadratureInterpolator::EvalKernelType;
-using TensorEvalKernel = QuadratureInterpolator::TensorEvalKernelType;
-using GradKernel = QuadratureInterpolator::GradKernelType;
-using CollocatedGradKernel = QuadratureInterpolator::CollocatedGradKernelType;
-
-template <QVectorLayout Q_LAYOUT>
-TensorEvalKernel FallbackTensorEvalKernel(int DIM)
+template <QVectorLayout Q_LAYOUT> auto IntFallbackTensorEvalKernel(int DIM)
 {
-   if (DIM == 1) { return Values1D<Q_LAYOUT>; }
-   else if (DIM == 2) { return Values2D<Q_LAYOUT>; }
-   else if (DIM == 3) { return Values3D<Q_LAYOUT>; }
-   else { MFEM_ABORT(""); }
+   if (DIM == 1)
+   {
+      return ImplValues1D<Q_LAYOUT, true>;
+   }
+   else if (DIM == 2)
+   {
+      return ImplValues2D<Q_LAYOUT, true>;
+   }
+   else if (DIM == 3)
+   {
+      return ImplValues3D<Q_LAYOUT, true>;
+   }
+   MFEM_ABORT("");
 }
 
-template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-GradKernel GetGradKernel(int DIM)
+template <QVectorLayout Q_LAYOUT> auto FallbackTensorEvalKernel(int DIM)
+{
+   if (DIM == 1)
+   {
+      return Values1D<Q_LAYOUT>;
+   }
+   else if (DIM == 2)
+   {
+      return Values2D<Q_LAYOUT>;
+   }
+   else if (DIM == 3)
+   {
+      return Values3D<Q_LAYOUT>;
+   }
+   MFEM_ABORT("");
+}
+
+template <QVectorLayout Q_LAYOUT, bool GRAD_PHYS> auto GetGradKernel(int DIM)
 {
    if (DIM == 1) { return Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
    else if (DIM == 2) { return Derivatives2D<Q_LAYOUT, GRAD_PHYS>; }
@@ -723,79 +504,185 @@ GradKernel GetGradKernel(int DIM)
    else { MFEM_ABORT(""); }
 }
 
-
-template<QVectorLayout Q_LAYOUT>
-GradKernel GetGradKernel(int DIM, bool GRAD_PHYS)
+template <QVectorLayout Q_LAYOUT> auto GetGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetGradKernel<Q_LAYOUT, false>(DIM); }
 }
 
 template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-CollocatedGradKernel GetCollocatedGradKernel(int DIM)
+auto GetCollocatedGradKernel(int DIM)
 {
-   if (DIM == 1) { return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 3) { return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>; }
-   else { MFEM_ABORT(""); }
+   if (DIM == 1)
+   {
+      return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>;
+   }
+   else if (DIM == 2)
+   {
+      return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>;
+   }
+   else if (DIM == 3)
+   {
+      return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>;
+   }
+   MFEM_ABORT("");
 }
 
-template<QVectorLayout Q_LAYOUT>
-CollocatedGradKernel GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
+template <QVectorLayout Q_LAYOUT>
+auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetCollocatedGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetCollocatedGradKernel<Q_LAYOUT, false>(DIM); }
 }
+
+auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS, QVectorLayout Q_LAYOUT)
+{
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      return GetCollocatedGradKernel<QVectorLayout::byNODES>(
+                DIM, GRAD_PHYS);
+   }
+   else
+   {
+      return GetCollocatedGradKernel<QVectorLayout::byVDIM>(
+                DIM, GRAD_PHYS);
+   }
+}
 } // namespace
 
-template <int DIM, int VDIM, int ND, int NQ>
-EvalKernel QuadratureInterpolator::EvalKernels::Kernel()
+template <int DIM, bool Integral>
+auto GetEvalKernelVDimFallback(int VDIM)
 {
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return Eval1D; }
-   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
+   if constexpr (Integral)
+   {
+      using EvalKernels = QuadratureInterpolator::IntEvalKernels;
+      if (VDIM == 1)
+      {
+         return EvalKernels::Kernel<DIM, 1, 0, 0>();
+      }
+      else if (VDIM == 2)
+      {
+         return EvalKernels::Kernel<DIM, 2, 0, 0>();
+      }
+      else if (VDIM == 3)
+      {
+         return EvalKernels::Kernel<DIM, 3, 0, 0>();
+      }
+   }
+   if constexpr (!Integral)
+   {
+      using EvalKernels = QuadratureInterpolator::EvalKernels;
+      if (VDIM == 1)
+      {
+         return EvalKernels::Kernel<DIM, 1, 0, 0>();
+      }
+      else if (VDIM == 2)
+      {
+         return EvalKernels::Kernel<DIM, 2, 0, 0>();
+      }
+      else if (VDIM == 3)
+      {
+         return EvalKernels::Kernel<DIM, 3, 0, 0>();
+      }
+   }
    MFEM_ABORT("");
 }
 
-template <int DIM>
-EvalKernel GetEvalKernelVDimFallback(int VDIM)
+template auto GetEvalKernelVDimFallback<1, true>(int VDIM);
+template auto GetEvalKernelVDimFallback<1, false>(int VDIM);
+template auto GetEvalKernelVDimFallback<2, true>(int VDIM);
+template auto GetEvalKernelVDimFallback<2, false>(int VDIM);
+template auto GetEvalKernelVDimFallback<3, true>(int VDIM);
+template auto GetEvalKernelVDimFallback<3, false>(int VDIM);
+
+QuadratureInterpolator::IntEvalKernelType
+QuadratureInterpolator::IntEvalKernels::Fallback(int DIM, int VDIM, int ND,
+                                                 int NQ)
 {
-   using EvalKernels = QuadratureInterpolator::EvalKernels;
-   if (VDIM == 1) { return EvalKernels::Kernel<DIM,1,0,0>(); }
-   else if (VDIM == 2) { return EvalKernels::Kernel<DIM,2,0,0>(); }
-   else if (VDIM == 3) { return EvalKernels::Kernel<DIM,3,0,0>(); }
-   else { MFEM_ABORT(""); }
+   if (DIM == 1)
+   {
+      return GetEvalKernelVDimFallback<1, true>(VDIM);
+   }
+   else if (DIM == 2)
+   {
+      return GetEvalKernelVDimFallback<2, true>(VDIM);
+   }
+   else if (DIM == 3)
+   {
+      return GetEvalKernelVDimFallback<3, true>(VDIM);
+   }
+   else
+   {
+      MFEM_ABORT("");
+   }
 }
 
-EvalKernel QuadratureInterpolator::EvalKernels::Fallback(
-   int DIM, int VDIM, int ND, int NQ)
+QuadratureInterpolator::EvalKernelType
+QuadratureInterpolator::EvalKernels::Fallback(int DIM, int VDIM, int ND, int NQ)
 {
-   if (DIM == 1) { return GetEvalKernelVDimFallback<1>(VDIM); }
-   else if (DIM == 2) { return GetEvalKernelVDimFallback<2>(VDIM); }
-   else if (DIM == 3) { return GetEvalKernelVDimFallback<3>(VDIM); }
-   else { MFEM_ABORT(""); }
+   if (DIM == 1)
+   {
+      return GetEvalKernelVDimFallback<1, false>(VDIM);
+   }
+   else if (DIM == 2)
+   {
+      return GetEvalKernelVDimFallback<2, false>(VDIM);
+   }
+   else if (DIM == 3)
+   {
+      return GetEvalKernelVDimFallback<3, false>(VDIM);
+   }
+   else
+   {
+      MFEM_ABORT("");
+   }
 }
 
-TensorEvalKernel QuadratureInterpolator::TensorEvalKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
+QuadratureInterpolator::IntTensorEvalKernelType
+QuadratureInterpolator::IntTensorEvalKernels::Fallback(int DIM,
+                                                       QVectorLayout Q_LAYOUT,
+                                                       int, int, int)
 {
-   if (Q_LAYOUT == QVectorLayout::byNODES) { return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM); }
-   else { return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM); }
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      return IntFallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
+   }
+   else
+   {
+      return IntFallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
+   }
 }
 
-GradKernel QuadratureInterpolator::GradKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
+QuadratureInterpolator::TensorEvalKernelType
+QuadratureInterpolator::TensorEvalKernels::Fallback(int DIM,
+                                                    QVectorLayout Q_LAYOUT, int,
+                                                    int, int)
+{
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
+   }
+   else
+   {
+      return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
+   }
+}
+
+QuadratureInterpolator::GradKernelType
+QuadratureInterpolator::GradKernels::Fallback(int DIM, QVectorLayout Q_LAYOUT,
+                                              bool GRAD_PHYS, int, int, int)
 {
    if (Q_LAYOUT == QVectorLayout::byNODES) { return GetGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
    else { return GetGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
-CollocatedGradKernel QuadratureInterpolator::CollocatedGradKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int)
+QuadratureInterpolator::CollocatedGradKernelType
+QuadratureInterpolator::CollocatedGradKernels::Fallback(int DIM,
+                                                        QVectorLayout Q_LAYOUT,
+                                                        bool GRAD_PHYS, int,
+                                                        int)
 {
-   if (Q_LAYOUT == QVectorLayout::byNODES) { return GetCollocatedGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
-   else { return GetCollocatedGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
+   return GetCollocatedGradKernel(DIM, GRAD_PHYS, Q_LAYOUT);
 }
 
 /// @endcond
@@ -806,98 +693,97 @@ namespace quadrature_interpolator
 {
 void InitEvalKernels()
 {
-   using k = QuadratureInterpolator::EvalKernels;
    // 2D, VDIM = 1
-   k::Specialization<2,1,1,1>::Add();
-   k::Specialization<2,1,1,4>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,1,1>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,1,4>();
    // Q1
-   k::Specialization<2,1,4,4>::Add();
-   k::Specialization<2,1,4,9>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,4,4>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,4,9>();
    // Q2
-   k::Specialization<2,1,9,9>::Add();
-   k::Specialization<2,1,9,16>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,9,9>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,9,16>();
    // Q3
-   k::Specialization<2,1,16,16>::Add();
-   k::Specialization<2,1,16,25>::Add();
-   k::Specialization<2,1,16,36>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,16,16>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,16,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,16,36>();
    // Q4
-   k::Specialization<2,1,25,25>::Add();
-   k::Specialization<2,1,25,36>::Add();
-   k::Specialization<2,1,25,49>::Add();
-   k::Specialization<2,1,25,64>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,25,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,25,36>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,25,49>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,25,64>();
 
    // 3D, VDIM = 1
    // Q0
-   k::Specialization<3,1,1,1>::Add();
-   k::Specialization<3,1,1,8>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,1,1>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,1,8>();
    // Q1
-   k::Specialization<3,1,8,8>::Add();
-   k::Specialization<3,1,8,27>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,8,8>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,8,27>();
    // Q2
-   k::Specialization<3,1,27,27>::Add();
-   k::Specialization<3,1,27,64>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,27,27>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,27,64>();
    // Q3
-   k::Specialization<3,1,64,64>::Add();
-   k::Specialization<3,1,64,125>::Add();
-   k::Specialization<3,1,64,216>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,64,64>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,64,125>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,64,216>();
    // Q4
-   k::Specialization<3,1,125,125>::Add();
-   k::Specialization<3,1,125,216>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,125,125>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,125,216>();
 
    // 2D, VDIM = 3
    // Q0
-   k::Specialization<2,3,1,1>::Add();
-   k::Specialization<2,3,1,4>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,1,1>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,1,4>();
    // Q1
-   k::Specialization<2,3,4,4>::Add();
-   k::Specialization<2,3,4,9>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,4,4>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,4,9>();
    // Q2
-   k::Specialization<2,3,9,4>::Add();
-   k::Specialization<2,3,9,9>::Add();
-   k::Specialization<2,3,9,16>::Add();
-   k::Specialization<2,3,9,25>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,9,4>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,9,9>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,9,16>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,9,25>();
    // Q3
-   k::Specialization<2,3,16,16>::Add();
-   k::Specialization<2,3,16,25>::Add();
-   k::Specialization<2,3,16,36>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,16,16>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,16,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,16,36>();
    // Q4
-   k::Specialization<2,3,25,25>::Add();
-   k::Specialization<2,3,25,36>::Add();
-   k::Specialization<2,3,25,49>::Add();
-   k::Specialization<2,3,25,64>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,25,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,25,36>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,25,49>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,25,64>();
 
    // 2D, VDIM = 2
    // Q1
-   k::Specialization<2,2,4,4>::Add();
-   k::Specialization<2,2,4,9>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,4,4>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,4,9>();
    // Q2
-   k::Specialization<2,2,9,9>::Add();
-   k::Specialization<2,2,9,16>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,9,9>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,9,16>();
    // Q3
-   k::Specialization<2,2,16,16>::Add();
-   k::Specialization<2,2,16,25>::Add();
-   k::Specialization<2,2,16,36>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,16,16>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,16,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,16,36>();
    // Q4
-   k::Specialization<2,2,25,25>::Add();
-   k::Specialization<2,2,25,36>::Add();
-   k::Specialization<2,2,25,49>::Add();
-   k::Specialization<2,2,25,64>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,25,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,25,36>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,25,49>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,25,64>();
 
    // 3D, VDIM = 3
    // Q1
-   k::Specialization<3,3,8,8>::Add();
-   k::Specialization<3,3,8,27>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,8,8>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,8,27>();
    // Q2
-   k::Specialization<3,3,27,27>::Add();
-   k::Specialization<3,3,27,64>::Add();
-   k::Specialization<3,3,27,125>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,27,27>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,27,64>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,27,125>();
    // Q3
-   k::Specialization<3,3,64,64>::Add();
-   k::Specialization<3,3,64,125>::Add();
-   k::Specialization<3,3,64,216>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,64,64>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,64,125>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,64,216>();
    // Q4
-   k::Specialization<3,3,125,125>::Add();
-   k::Specialization<3,3,125,216>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,125,125>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,125,216>();
 }
 
 } // namespace quadrature_Interpolator

fem/quadinterpolator.hpp

@@ -117,6 +117,10 @@ public:
        FiniteElementSpace is a vector space) and their determinants are computed
        and stored in @a q_det.
 
+       For Integral spaces, the flags VALUES requests the computation of the
+       scalar field values. The result is stored in @a q_val. Derivative types
+       are not supported.
+
        For H(div)-conforming spaces, the flags VALUES / PHYSICAL_VALUES request
        the computation of the vector field values in reference or physical
        space, respectively. The flag PHYSICAL_MAGNITUDES requests the
@@ -159,26 +163,49 @@ public:
    /// QuadratureInterpolator.
    static bool SupportsFESpace(const FiniteElementSpace &fespace);
 
-   using TensorEvalKernelType = void(*)(const int, const real_t *, const real_t *,
-                                        real_t *, const int, const int, const int);
-   using GradKernelType = void(*)(const int, const real_t *, const real_t *,
-                                  const real_t *, const real_t *, real_t *,
-                                  const int, const int, const int, const int);
-   using CollocatedGradKernelType = void(*)(const int, const real_t *,
-                                            const real_t *, const real_t *,
-                                            real_t *, const int, const int,
-                                            const int);
-   using DetKernelType = void(*)(const int NE, const real_t *, const real_t *,
-                                 const real_t *, real_t *, const int, const int,
-                                 Vector *);
-   using EvalKernelType = void(*)(const int, const int, const QVectorLayout,
-                                  const GeometricFactors *, const DofToQuad &,
-                                  const Vector &, Vector &, Vector &, Vector &,
-                                  const int);
+   // value map types
+   using TensorEvalKernelType = void (*)(const int ne, const real_t *B,
+                                         const real_t *e_vec, real_t *q_val,
+                                         const int vdim, const int nd,
+                                         const int nq);
+   using GradKernelType = void (*)(const int ne, const real_t *B,
+                                   const real_t *G, const real_t *J,
+                                   const real_t *e_vec, real_t *q_der,
+                                   const int s_dim, const int v_dim,
+                                   const int nd, const int nq);
+   using CollocatedGradKernelType = void (*)(const int ne, const real_t *G,
+                                             const real_t *J,
+                                             const real_t *e_vec, real_t *q_der,
+                                             const int sdim, const int vdim,
+                                             const int d1d);
+   using DetKernelType = void (*)(const int NE, const real_t *B,
+                                  const real_t *G, const real_t *e_vec,
+                                  real_t *q_det, const int nd, const int nq,
+                                  Vector *d_buffer);
+   using EvalKernelType = void (*)(const int NE, const int vdim,
+                                   const QVectorLayout q_layout,
+                                   const GeometricFactors *geom,
+                                   const DofToQuad &maps, const Vector &e_vec,
+                                   Vector &q_val, Vector &q_der, Vector &q_det,
+                                   const int eval_flags);
+
+   // integral map types
+   using IntTensorEvalKernelType = void (*)(const int ne, const real_t *B,
+                                            const real_t *detJ,
+                                            const real_t *e_vec, real_t *q_val,
+                                            const int vdim, const int nd,
+                                            const int nq);
+   using IntEvalKernelType =
+      void (*)(const int NE, const int vdim, const QVectorLayout q_layout,
+               const real_t *detJ, const GeometricFactors *geom,
+               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+               Vector &q_der, Vector &q_det, const int eval_flags);
+
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
 
+   // value-type mapping
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
                          (int, QVectorLayout, int, int, int), (int));
    MFEM_REGISTER_KERNELS(GradKernels, GradKernelType,
@@ -187,8 +214,84 @@ public:
    MFEM_REGISTER_KERNELS(EvalKernels, EvalKernelType, (int, int, int, int));
    MFEM_REGISTER_KERNELS(CollocatedGradKernels, CollocatedGradKernelType,
                          (int, QVectorLayout, bool, int, int), (int));
+
+   // integral-type mapping
+   MFEM_REGISTER_KERNELS(IntTensorEvalKernels, IntTensorEvalKernelType,
+                         (int, QVectorLayout, int, int, int), (int));
+   MFEM_REGISTER_KERNELS(IntEvalKernels, IntEvalKernelType, (int, int, int, int));
+
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
+
+   /// Adds specializations for TensorEvalKernels
+   template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D,
+             int NBZ = 0>
+   static void AddTensorEvalSpecializations()
+   {
+      if constexpr (NBZ)
+      {
+         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
+                              Q1D>::template Opt<NBZ>::Add();
+         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
+                           Q1D>::template Opt<NBZ>::Add();
+      }
+      else if constexpr (NBZ == 0)
+      {
+         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
+                              Q1D>::Add();
+         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
+                           Q1D>::Add();
+      }
+   }
+
+   /// Adds specializations for EvalKernels
+   template <int DIM, int VDIM, int ND, int NQ>
+   static void AddEvalSpecializations()
+   {
+      IntEvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
+      EvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
+   }
+
+   /// Adds specializations for GradKernels
+   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
+             int Q1D, int NBZ = 0>
+   static void AddGradSpecializations()
+   {
+      if constexpr (NBZ)
+      {
+         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
+                     Q1D>::template Opt<NBZ>::Add();
+      }
+      else if constexpr (NBZ == 0)
+      {
+         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
+                     Q1D>::Add();
+      }
+   }
+
+   /// Adds specializations for CollocatedGradKernels
+   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
+             int NBZ = 0>
+   static void AddCollocatedGradSpecializations()
+   {
+      if constexpr (NBZ)
+      {
+         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
+                               D1D>::template Opt<NBZ>::Add();
+      }
+      else if constexpr (NBZ == 0)
+      {
+         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
+                               D1D>::Add();
+      }
+   }
+
+   /// Adds specializations for DetKernels
+   template <int DIM, int SDIM, int D1D, int Q1D>
+   static void AddDetSpecializations()
+   {
+      DetKernels::Specialization<DIM, SDIM, D1D, Q1D>::Add();
+   }
 };
 
 }

general/backends.hpp

@@ -66,6 +66,23 @@ constexpr bool mfem_use_gpu = false;
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
+// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
+// Example (3,2,1) block:
+// 0 (0,0), 1 (1,0), 2 (2,0)
+// 3 (1,0), 4 (1,1), 5 (2,1)
+#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
+   for (int iz = 0; iz < SZ; ++iz)                                             \
+      for (int iy = 0; iy < SY; ++iy)                                          \
+         for (int ix = 0; ix < SX; ++ix)
+// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
+// contiguous in x. This intentionally offsets threads within the block to avoid
+// shared memory bank conflicts.
+// Example (3,2,1) block assigned to work on (2,2,1) items:
+// 0 (0,0), 1 (1,0), 2 (N/A)
+// 3 (1,0), 4 (1,1), 5 (N/A)
+#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
+                                             OY, OZ)                           \
+   MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)
 #endif
 
 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA

general/cuda.hpp

@@ -49,6 +49,23 @@ constexpr bool mfem_use_gpu = true;
 #define MFEM_THREAD_SIZE(k) blockDim.k
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
+// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
+// Example (3,2,1) block:
+// 0 (0,0), 1 (1,0), 2 (2,0)
+// 3 (1,0), 4 (1,1), 5 (2,1)
+#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
+   if (int ix = threadIdx.k % (SX), iy = threadIdx.k / (SX), iz = iy / (SY);   \
+       (iy %= (SY)), (threadIdx.k < (SX) * (SY) * (SZ)))
+// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
+// contiguous in x. This intentionally offsets threads within the block to avoid
+// shared memory bank conflicts.
+// Example (3,2,1) block assigned to work on (2,2,1) items:
+// 0 (0,0), 1 (1,0), 2 (N/A)
+// 3 (1,0), 4 (1,1), 5 (N/A)
+#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
+                                             OY, OZ)                           \
+   if (int ix = threadIdx.k % (OX), iy = threadIdx.k / (OX), iz = iy / (OY);   \
+       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__CUDA_ARCH__)
 #endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 

general/forall.hpp

@@ -480,8 +480,8 @@ template <typename DBODY>
 void RajaHipWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS,true> >(RAJA::RangeSegment(0,N),
-                                                       d_body);
+   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS, true> >(
+      Device::GetRajaResource(), RAJA::RangeSegment(0, N), d_body);
 }
 
 template <typename DBODY>

general/hip.hpp

@@ -51,6 +51,25 @@ constexpr bool mfem_use_gpu = true;
    for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) \
    if(const int i=hipThreadIdx_ ##k; i<N)
+// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
+// Example (3,2,1) block:
+// 0 (0,0), 1 (1,0), 2 (2,0)
+// 3 (1,0), 4 (1,1), 5 (2,1)
+#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
+   if (int ix = hipThreadIdx_##k % (SX), iy = hipThreadIdx_##k / (SX),         \
+       iz = iy / (SY);                                                         \
+       (iy %= (SY)), (hipThreadIdx_##k < (SX) * (SY) * (SZ)))
+// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
+// contiguous in x. This intentionally offsets threads within the block to avoid
+// shared memory bank conflicts.
+// Example (3,2,1) block assigned to work on (2,2,1) items:
+// 0 (0,0), 1 (1,0), 2 (N/A)
+// 3 (1,0), 4 (1,1), 5 (N/A)
+#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
+                                             OY, OZ)                           \
+   if (int ix = hipThreadIdx_##k % (OX), iy = hipThreadIdx_##k / (OX),         \
+       iz = iy / (OY);                                                         \
+       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__HIP_DEVICE_COMPILE__)
 #endif // defined(MFEM_USE_HIP) && defined(__HIP__)
 

linalg/batched/batched.cpp

@@ -13,6 +13,7 @@
 #include "native.hpp"
 #include "gpu_blas.hpp"
 #include "magma.hpp"
+#include "../../general/reducers.hpp"
 
 namespace mfem
 {
@@ -119,4 +120,16 @@ void BatchedLinAlgBase::MultTranspose(const DenseTensor &A, const Vector &x,
    AddMult(A, x, y, 1.0, 0.0, Op::T);
 }
 
+void VerifyBatchedLUInfo(const Array<int> &info_array, const char *message)
+{
+   static Array<int> workspace;
+   int status = 0;
+   const int *d_info = info_array.Read();
+   mfem::reduce(
+      info_array.Size(), status,
+   [=] MFEM_HOST_DEVICE (int i, int &r) { r |= d_info[i]; },
+   BOrReducer<int> {}, true, workspace);
+   MFEM_VERIFY(status == 0, message);
+}
+
 }

linalg/batched/batched.hpp

@@ -141,6 +141,9 @@ public:
    virtual ~BatchedLinAlgBase() { }
 };
 
+/// Check that all batched LU info values are zero.
+void VerifyBatchedLUInfo(const Array<int> &info_array, const char *message);
+
 } // namespace mfem
 
 #endif

linalg/batched/gpu_blas.cpp

@@ -126,7 +126,8 @@ void GPUBlasBatchedLinAlg::LUFactor(DenseTensor &A, Array<int> &P) const
    const blasStatus_t status = MFEM_GPUBLAS_PREFIX(getrfBatched)(
                                   GPUBlas::Handle(), n, d_A_ptrs, n, P.Write(),
                                   info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
+   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
 }
 
 void GPUBlasBatchedLinAlg::LUSolve(
@@ -189,12 +190,14 @@ void GPUBlasBatchedLinAlg::Invert(DenseTensor &A) const
    status = MFEM_GPUBLAS_PREFIX(getrfBatched)(
                GPUBlas::Handle(), n, d_LU_ptrs, n, P.Write(),
                info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
+   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
 
    status = MFEM_GPUBLAS_PREFIX(getriBatched)(
                GPUBlas::Handle(), n, d_LU_ptrs, n, P.ReadWrite(), d_A_ptrs, n,
                info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
+   VerifyBatchedLUInfo(info_array, "Batch matrix inversion failed");
 }
 
 #endif

linalg/batched/magma.cpp

@@ -99,7 +99,8 @@ void MagmaBatchedLinAlg::LUFactor(DenseTensor &A, Array<int> &P) const
    const magma_int_t status = MFEM_MAGMA_PREFIX(getrf_batched)(
                                  n, n, d_A_ptrs, n, d_P_ptrs,
                                  info_array.Write(), n_mat, Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
+   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
 }
 
 void MagmaBatchedLinAlg::LUSolve(
@@ -169,12 +170,14 @@ void MagmaBatchedLinAlg::Invert(DenseTensor &A) const
    status = MFEM_MAGMA_PREFIX(getrf_batched)(
                n, n, d_LU_ptrs, n, d_P_ptrs, info_array.Write(), n_mat,
                Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
+   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
 
    status = MFEM_MAGMA_PREFIX(getri_outofplace_batched)(
                n, d_LU_ptrs, n, d_P_ptrs, d_A_ptrs, n, info_array.Write(),
                n_mat, Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
+   VerifyBatchedLUInfo(info_array, "Batch matrix inversion failed");
 }
 
 } // namespace mfem

linalg/complex_operator.cpp

@@ -246,6 +246,10 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
    const int  nrows_i = (A_i)?A_i->Height():0;
    const int    nrows = std::max(nrows_r, nrows_i);
 
+   const int  ncols_r = (A_r)?A_r->Width():0;
+   const int  ncols_i = (A_i)?A_i->Width():0;
+   const int    ncols = std::max(ncols_r, ncols_i);
+
    const int     *I_r = (A_r)?A_r->GetI():NULL;
    const int     *I_i = (A_i)?A_i->GetI():NULL;
 
@@ -280,7 +284,7 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
             J[I[i] + j] = J_r[I_r[i] + j];
             D[I[i] + j] = D_r[I_r[i] + j];
 
-            J[I[i+nrows] + off_i + j] = J_r[I_r[i] + j] + nrows;
+            J[I[i+nrows] + off_i + j] = J_r[I_r[i] + j] + ncols;
             D[I[i+nrows] + off_i + j] = factor*D_r[I_r[i] + j];
          }
       }
@@ -289,7 +293,7 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
          const int off_r = (I_r)?(I_r[i+1] - I_r[i]):0;
          for (int j=0; j<I_i[i+1] - I_i[i]; j++)
          {
-            J[I[i] + off_r + j] =  J_i[I_i[i] + j] + nrows;
+            J[I[i] + off_r + j] =  J_i[I_i[i] + j] + ncols;
             D[I[i] + off_r + j] = -D_i[I_i[i] + j];
 
             J[I[i+nrows] + j] = J_i[I_i[i] + j];
@@ -892,12 +896,12 @@ ComplexHypreParMatrix::getColStartStop(const HypreParMatrix * A_r,
    HYPRE_BigInt loc_start_stop[2];
    offd_col_start_stop = new HYPRE_BigInt[2 * num_recv_procs];
 
-   const HYPRE_BigInt * row_part = (A_r) ? A_r->RowPart() :
-                                   ((A_i) ? A_i->RowPart() : NULL);
+   const HYPRE_BigInt * col_part = (A_r) ? A_r->ColPart() :
+                                   ((A_i) ? A_i->ColPart() : NULL);
 
-   int row_part_ind = (HYPRE_AssumedPartitionCheck()) ? 0 : myid_;
-   loc_start_stop[0] = row_part[row_part_ind];
-   loc_start_stop[1] = row_part[row_part_ind+1];
+   int col_part_ind = (HYPRE_AssumedPartitionCheck()) ? 0 : myid_;
+   loc_start_stop[0] = col_part[col_part_ind];
+   loc_start_stop[1] = col_part[col_part_ind+1];
 
    MPI_Request * req = new MPI_Request[send_procs.size()+recv_procs.size()];
    MPI_Status * stat = new MPI_Status[send_procs.size()+recv_procs.size()];

linalg/cudss.cpp

@@ -15,10 +15,20 @@
 
 #ifdef MFEM_USE_CUDSS
 
+#if CUDSS_VERSION >= 800
 #ifdef MFEM_USE_SINGLE
-#define CUDA_REAL_T CUDA_R_32F
+#define CUDSS_REAL_T CUDSS_R_32F
 #else
-#define CUDA_REAL_T CUDA_R_64F
+#define CUDSS_REAL_T CUDSS_R_64F
+#endif
+#define CUDSS_INT_T CUDSS_R_32I
+#else
+#ifdef MFEM_USE_SINGLE
+#define CUDSS_REAL_T CUDA_R_32F
+#else
+#define CUDSS_REAL_T CUDA_R_64F
+#endif
+#define CUDSS_INT_T CUDA_R_32I
 #endif
 
 // Define a cuDSS error check macro, MFEM_CUDSS_CHECK(x), where x returns/is of
@@ -65,8 +75,13 @@ CuDSSSolver::CuDSSSolver(MPI_Comm comm_) : mpi_comm(comm_)
 #endif
    MFEM_CUDSS_CHECK(cudssSetCommLayer(handle, comm_lib));
 
+#if CUDSS_VERSION >= 800
+   MFEM_CUDSS_CHECK(cudssDataSet(handle, solverData, CUDSS_DATA_COMM_HOST,
+                                 &mpi_comm, sizeof(MPI_Comm *)));
+#else
    MFEM_CUDSS_CHECK(cudssDataSet(handle, solverData, CUDSS_DATA_COMM,
                                  &mpi_comm, sizeof(MPI_Comm *)));
+#endif
 }
 #endif // MFEM_USE_MPI
 
@@ -257,11 +272,19 @@ void CuDSSSolver::SetMatrixCuDSS(int *csr_offsets, int *csr_columns,
          CuMemcpyDtoD(csr_offsets_d, csr_offsets, (n_loc + 1) * sizeof(int));
          CuMemcpyDtoD(csr_columns_d, csr_columns, nnz * sizeof(int));
 
+#if CUDSS_VERSION >= 800
          MFEM_CUDSS_CHECK(
             cudssMatrixCreateCsr(
                Ac.get(), n_global, n_global, nnz, csr_offsets_d, NULL,
-               csr_columns_d, csr_values_d, CUDA_R_32I, CUDA_REAL_T, mat_type, mview,
-               CUDSS_BASE_ZERO));
+               csr_columns_d, csr_values_d, CUDSS_INT_T, CUDSS_INT_T, CUDSS_REAL_T,
+               mat_type, mview, CUDSS_BASE_ZERO));
+#else
+         MFEM_CUDSS_CHECK(
+            cudssMatrixCreateCsr(
+               Ac.get(), n_global, n_global, nnz, csr_offsets_d, NULL,
+               csr_columns_d, csr_values_d, CUDSS_INT_T, CUDSS_REAL_T,
+               mat_type, mview, CUDSS_BASE_ZERO));
+#endif
       }
       else    // !reorder_reuse
       {
@@ -269,11 +292,19 @@ void CuDSSSolver::SetMatrixCuDSS(int *csr_offsets, int *csr_columns,
          {
             MFEM_CUDSS_CHECK(cudssMatrixDestroy(*Ac));
          }
+#if CUDSS_VERSION >= 800
          MFEM_CUDSS_CHECK(
             cudssMatrixCreateCsr(
-               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL, csr_columns,
-               csr_values_d, CUDA_R_32I, CUDA_REAL_T, mat_type, mview,
-               CUDSS_BASE_ZERO));
+               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL,
+               csr_columns, csr_values_d, CUDSS_INT_T, CUDSS_INT_T, CUDSS_REAL_T,
+               mat_type, mview, CUDSS_BASE_ZERO));
+#else
+         MFEM_CUDSS_CHECK(
+            cudssMatrixCreateCsr(
+               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL,
+               csr_columns, csr_values_d, CUDSS_INT_T, CUDSS_REAL_T,
+               mat_type, mview, CUDSS_BASE_ZERO));
+#endif
       }
 #ifdef MFEM_USE_MPI
       if (Mpi::IsInitialized())
@@ -334,10 +365,10 @@ void CuDSSSolver::SetNumRHS(int nrhs_) const
       }
       // Create empty RHS and solution vectors
       MFEM_CUDSS_CHECK(cudssMatrixCreateDn(&xc, n_global, nrhs_, n_global, NULL,
-                                           CUDA_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
+                                           CUDSS_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
 
       MFEM_CUDSS_CHECK(cudssMatrixCreateDn(&yc, n_global, nrhs_, n_global, NULL,
-                                           CUDA_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
+                                           CUDSS_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
 
 #ifdef MFEM_USE_MPI
       MFEM_CUDSS_CHECK(cudssMatrixSetDistributionRow1d(xc, row_start, row_end));

makefile

@@ -810,6 +810,7 @@ MINIAPPS_SUBDIRS = dpg/util hooke/operators hooke/preconditioners \
    hooke/materials hooke/kernels
 FORMAT_FILES += $(foreach dir,$(TESTS_SUBDIRS),tests/$(dir)/*.?pp)
 FORMAT_FILES += $(foreach dir,$(UNIT_TESTS_SUBDIRS),tests/unit/$(dir)/*.?pp)
+FORMAT_FILES += tests/unit/fem/specializations/*.?pp
 FORMAT_FILES += $(foreach dir,$(MINIAPPS_SUBDIRS),miniapps/$(dir)/*.?pp)
 FORMAT_FILES += config/cmake/config.hpp.in config/config.hpp.in mfem*.hpp
 FORMAT_EXCLUDE = general/tinyxml2.cpp tests/unit/catch.hpp

mesh/mesh.cpp

@@ -667,9 +667,84 @@ void Mesh::GetEdgeTransformation(int EdgeNo,
          }
          EdTr->SetFE(edge_el);
       }
-      else
+      else // L2 Nodes (e.g., periodic mesh), go through the face containing the edge
       {
-         MFEM_ABORT("Not implemented.");
+         // Search for a face that contains this edge
+         GetEdgeFaceTable();
+
+         Array<int> faces_e;
+         edge_face->GetRow(EdgeNo, faces_e);
+
+         MFEM_VERIFY(faces_e.Size() > 0, "Edge not found in any face!");
+         const int face_no = faces_e[0];
+
+         // Get edge local index and orientation
+         Array<int> edges_f, oris_f;
+         GetFaceEdges(face_no, edges_f, oris_f);
+         const int local_idx = edges_f.Find(EdgeNo);
+         MFEM_ASSERT(local_idx >= 0, "Edge not found on the face!");
+         const int edge_ori = oris_f[local_idx] > 0 ? 0 : 1;
+
+         // Get face information
+         const FaceInfo &face_info = faces_info[face_no];
+
+         // Get transformation from face to edge
+         IntegrationPointTransformation LocEdge;
+         int edge_info = EncodeFaceInfo(local_idx, edge_ori);
+         Element::Type face_type = GetFaceElementType(face_no);
+
+         switch (face_type)
+         {
+            case Element::TRIANGLE:
+               GetLocalSegToTriTransformation(LocEdge.Transf, edge_info);
+               break;
+            case Element::QUADRILATERAL:
+               GetLocalSegToQuadTransformation(LocEdge.Transf, edge_info);
+               break;
+            default:
+               MFEM_ABORT("Unsupported face type for edge transformation!");
+         }
+
+         // Get edge element
+         const int order = Nodes->FESpace()->GetElementOrder(face_info.Elem1No);
+         const L2_FECollection *l2_fec = dynamic_cast<const L2_FECollection*>
+                                         (Nodes->FESpace()->FEColl());
+         if (l2_fec)
+         {
+            // L2 elements do not have a defined trace space
+            if (!EdgeTransfElement || EdgeTransfElement->GetOrder() != order
+                || EdgeTransfElement->GetBasisType() != l2_fec->GetBasisType())
+            {
+               EdgeTransfElement = make_unique<L2_SegmentElement>(
+                                      order, l2_fec->GetBasisType());
+            }
+            edge_el = EdgeTransfElement.get();
+         }
+         else
+         {
+            MFEM_ABORT("Unsupported finite element collection.");
+         }
+
+         // Map edge nodes to face reference space
+         IntegrationRule face_ir(edge_el->GetDof());
+         LocEdge.Transform(edge_el->GetNodes(), face_ir);
+
+         // Then, map from face to element
+         IntegrationPointTransformation Loc1;
+         GetLocalFaceTransformation(face_type,
+                                    GetElementType(face_info.Elem1No),
+                                    Loc1.Transf, face_info.Elem1Inf);
+
+         IntegrationRule elem_ir(edge_el->GetDof());
+         Loc1.Transf.ElementNo = face_info.Elem1No;
+         Loc1.Transf.ElementType = ElementTransformation::ELEMENT;
+         Loc1.Transf.mesh = this;
+         Loc1.Transform(face_ir, elem_ir);
+
+         // Finally, get the physical coordinates
+         Nodes->GetVectorValues(Loc1.Transf, elem_ir, pm);
+
+         EdTr->SetFE(edge_el);
       }
    }
 }
@@ -1824,8 +1899,8 @@ void Mesh::Init()
 
 void Mesh::InitTables()
 {
-   el_to_edge =
-      el_to_face = el_to_el = bel_to_edge = face_edge = edge_vertex = NULL;
+   el_to_edge = el_to_face = el_to_el = bel_to_edge = NULL;
+   face_edge = edge_face = edge_vertex = NULL;
    face_to_elem = NULL;
 }
 
@@ -1848,6 +1923,7 @@ void Mesh::DestroyTables()
    }
 
    delete face_edge;
+   delete edge_face;
    delete edge_vertex;
 
    delete face_to_elem;
@@ -1921,6 +1997,7 @@ void Mesh::ResetLazyData()
 {
    delete el_to_el;     el_to_el = NULL;
    delete face_edge;    face_edge = NULL;
+   delete edge_face;    edge_face = NULL;
    delete face_to_elem;    face_to_elem = NULL;
    delete edge_vertex;  edge_vertex = NULL;
    DeleteGeometricFactors();
@@ -2845,6 +2922,7 @@ void Mesh::ReorderElements(const Array<int> &ordering, bool reorder_vertices)
    //                 boundary element ordering
    // - el_to_el    - no need to rebuild
    // - face_edge   - no need to rebuild
+   // - edge_face   - no need to rebuild
    // - edge_vertex - no need to rebuild
    // - geom_factors - no need to rebuild
 
@@ -3327,11 +3405,25 @@ void Mesh::DoNodeReorder(DSTable *old_v_to_v, Table *old_elem_vert)
       // loop over all elements
       for (int i = 0; i < GetNE(); i++)
       {
+         fes->GetElementInteriorDofs(i, old_dofs);
+         // No need to permute the dofs if there are fewer than two
+         if (old_dofs.Size() < 2)
+         {
+            offset += old_dofs.Size();
+            continue;
+         }
+
          const int *old_v = old_elem_vert->GetRow(i);
          const int *new_v = elements[i]->GetVertices();
          const int *dof_ord;
          int new_or;
          const Geometry::Type geom = elements[i]->GetGeometryType();
+         if (geom == Geometry::CUBE || geom == Geometry::PRISM ||
+             geom == Geometry::PYRAMID)
+         {
+            offset += old_dofs.Size();
+            continue;
+         }
          switch (geom)
          {
             case Geometry::SEGMENT:
@@ -3355,9 +3447,8 @@ void Mesh::DoNodeReorder(DSTable *old_v_to_v, Table *old_elem_vert)
          dof_ord = fec->DofOrderForOrientation(geom, new_or);
          MFEM_VERIFY(dof_ord != NULL,
                      "FE collection '" << fec->Name()
-                     << "' does not define reordering for "
+                     << "' does not define reordering (" << new_or << ") for "
                      << Geometry::Name[geom] << " elements!");
-         fes->GetElementInteriorDofs(i, old_dofs);
          new_dofs.SetSize(old_dofs.Size());
          for (int j = 0; j < new_dofs.Size(); j++)
          {
@@ -4585,8 +4676,9 @@ Mesh::Mesh(const Mesh &mesh, bool copy_nodes)
    // Do NOT copy the element-to-element Table, el_to_el
    el_to_el = NULL;
 
-   // Do NOT copy the face-to-edge Table, face_edge
+   // Do NOT copy the face-to-edge Table, face_edge and edge_face
    face_edge = NULL;
+   edge_face = NULL;
    face_to_elem = NULL;
 
    // Copy the edge-to-vertex Table, edge_vertex
@@ -7116,7 +7208,8 @@ const FiniteElementSpace *Mesh::GetNodalFESpace() const
    return ((Nodes) ? Nodes->FESpace() : NULL);
 }
 
-void Mesh::SetCurvature(int order, bool discont, int space_dim, int ordering)
+void Mesh::SetCurvature(int order, bool discont, int space_dim, int ordering,
+                        int pyr_type)
 {
    if (order <= 0)
    {
@@ -7129,11 +7222,12 @@ void Mesh::SetCurvature(int order, bool discont, int space_dim, int ordering)
    if (discont)
    {
       const int type = 1; // Gauss-Lobatto points
-      nfec = new L2_FECollection(order, Dim, type);
+      nfec = new L2_FECollection(order, Dim, type, FiniteElement::VALUE,
+                                 pyr_type);
    }
    else
    {
-      nfec = new H1_FECollection(order, Dim);
+      nfec = new H1_FECollection(order, Dim, BasisType::GaussLobatto, pyr_type);
    }
    FiniteElementSpace* nfes = new FiniteElementSpace(this, nfec, space_dim,
                                                      ordering);
@@ -8079,6 +8173,22 @@ Table *Mesh::GetFaceEdgeTable() const
    return (face_edge);
 }
 
+Table *Mesh::GetEdgeFaceTable() const
+{
+   if (edge_face)
+   {
+      return edge_face;
+   }
+
+   if (Dim != 3)
+   {
+      return NULL;
+   }
+
+   edge_face = Transpose(*GetFaceEdgeTable());
+   return edge_face;
+}
+
 Table *Mesh::GetEdgeVertexTable() const
 {
    if (edge_vertex)
@@ -11437,6 +11547,7 @@ void Mesh::Swap(Mesh& other, bool non_geometry)
    mfem::Swap(bel_to_edge, other.bel_to_edge);
    mfem::Swap(be_to_face, other.be_to_face);
    mfem::Swap(face_edge, other.face_edge);
+   mfem::Swap(edge_face, other.edge_face);
    mfem::Swap(face_to_elem, other.face_to_elem);
    mfem::Swap(edge_vertex, other.edge_vertex);
 

mesh/mesh.hpp

@@ -250,16 +250,18 @@ protected:
    Table *bel_to_edge;    // for 3D only
 
    // Note that the following tables are owned by this class and should not be
-   // deleted by the caller. Of these three tables, only face_edge and
+   // deleted by the caller. Of these four tables, only face_edge, edge_face and
    // edge_vertex are returned by access functions.
    mutable Table *face_to_elem;  // Used by FindFaceNeighbors, not returned.
    mutable Table *face_edge;     // Returned by GetFaceEdgeTable().
+   mutable Table *edge_face;     // Returned by GetEdgeFaceTable().
    mutable Table *edge_vertex;   // Returned by GetEdgeVertexTable().
 
    IsoparametricTransformation Transformation, Transformation2;
    IsoparametricTransformation BdrTransformation;
    IsoparametricTransformation FaceTransformation, EdgeTransformation;
    FaceElementTransformations FaceElemTr;
+   mutable std::unique_ptr<L2_SegmentElement> EdgeTransfElement;
 
    // refinement embeddings for forward compatibility with NCMesh
    mutable CoarseFineTransformations CoarseFineTr;
@@ -1731,6 +1733,11 @@ public:
    /// @note The returned object should NOT be deleted by the caller.
    Table *GetFaceEdgeTable() const;
 
+   /// Returns the edge-to-face Table (3D)
+   ///
+   /// @note The returned object should NOT be deleted by the caller.
+   Table *GetEdgeFaceTable() const;
+
    /// Returns the edge-to-vertex Table (3D)
    ///
    /// @note The returned object should NOT be deleted by the caller.
@@ -2425,9 +2432,13 @@ public:
                                finite element space (continuous is default).
        @param[in]  space_dim   The space dimension (optional).
        @param[in]  ordering    The Ordering of the finite element space
-                               (Ordering::byVDIM is the default). */
-   virtual void SetCurvature(int order, bool discont = false, int space_dim = -1,
-                             int ordering = 1);
+                               (Ordering::byVDIM is the default).
+       @param[in]  pyr_type    Select Bergot (pyr_type = 0) or Fuentes
+                               (pyr_type = 1) basis functions for pyramid
+                               shaped elements. */
+   virtual void SetCurvature(int order, bool discont = false,
+                             int space_dim = -1, int ordering = 1,
+                             int pyr_type = 1);
 
    /// @}
 

mesh/nurbs.cpp

@@ -1354,22 +1354,27 @@ NURBSPatch::NURBSPatch(std::istream &input)
    int pdim, dim, size = 1;
    string ident;
 
+   skip_comment_lines(input, '#');
    input >> ws >> ident >> pdim; // knotvectors
    kv.SetSize(pdim);
    for (int i = 0; i < pdim; i++)
    {
+      skip_comment_lines(input, '#');
       kv[i] = new KnotVector(input);
       size *= kv[i]->GetNCP();
    }
 
+   skip_comment_lines(input, '#');
    input >> ws >> ident >> dim; // dimension
    init(dim + 1);
 
+   skip_comment_lines(input, '#');
    input >> ws >> ident; // controlpoints (homogeneous coordinates)
    if (ident == "controlpoints" || ident == "controlpoints_homogeneous")
    {
       for (int j = 0, i = 0; i < size; i++)
       {
+         skip_comment_lines(input, '#');
          for (int d = 0; d <= dim; d++, j++)
          {
             input >> data[j];
@@ -1380,6 +1385,7 @@ NURBSPatch::NURBSPatch(std::istream &input)
    {
       for (int j = 0, i = 0; i < size; i++)
       {
+         skip_comment_lines(input, '#');
          for (int d = 0; d <= dim; d++)
          {
             input >> data[j+d];

mesh/pmesh.cpp

@@ -2031,18 +2031,20 @@ std::unique_ptr<ParGridFunction> ParMesh::GetJacobianDeterminantGF() const
    return detgf;
 }
 
-void ParMesh::SetCurvature(int order, bool discont, int space_dim, int ordering)
+void ParMesh::SetCurvature(int order, bool discont, int space_dim, int ordering,
+                           int pyrtype)
 {
    DeleteFaceNbrData();
    space_dim = (space_dim == -1) ? spaceDim : space_dim;
    FiniteElementCollection* nfec;
    if (discont)
    {
-      nfec = new L2_FECollection(order, Dim, BasisType::GaussLobatto);
+      nfec = new L2_FECollection(order, Dim, BasisType::GaussLobatto,
+                                 FiniteElement::VALUE, pyrtype);
    }
    else
    {
-      nfec = new H1_FECollection(order, Dim);
+      nfec = new H1_FECollection(order, Dim, BasisType::GaussLobatto, pyrtype);
    }
    ParFiniteElementSpace* nfes = new ParFiniteElementSpace(this, nfec, space_dim,
                                                            ordering);
@@ -4864,6 +4866,13 @@ void ParMesh::Print(std::ostream &os, const std::string &comments) const
       return;
    }
 
+   if (pncmesh && pncmesh->using_scaling)
+   {
+      // For nodes scaling, we write the file in the format MFEM NC mesh v1.1.
+      Printer(os, "", comments);
+      return;
+   }
+
    const Array<int>* s2l_face;
    if (!pncmesh)
    {

mesh/pmesh.hpp

@@ -563,7 +563,7 @@ public:
    void ExchangeFaceNbrNodes();
 
    void SetCurvature(int order, bool discont = false, int space_dim = -1,
-                     int ordering = 1) override;
+                     int ordering = 1, int pyrtype = 1) override;
 
    std::unique_ptr<ParGridFunction> GetJacobianDeterminantGF() const;
 

mesh/pncmesh.cpp

@@ -28,6 +28,48 @@ namespace mfem
 
 using namespace bin_io;
 
+static int GetHexEdgeSplit(const int* nodes, int v1, int v2);
+
+static bool SameSplitScale(real_t a, real_t b)
+{
+#ifdef MFEM_USE_DOUBLE
+   constexpr real_t rel_tol = 1.0e-8;
+#else
+   constexpr real_t rel_tol = 1.0e-5;
+#endif
+   return std::abs(a - b) <= rel_tol *
+          std::max(real_t(1.0), std::max(std::abs(a), std::abs(b)));
+}
+
+static real_t DirectedHexEdgeScale(const int* nodes, const Refinement &ref,
+                                   int v0, int v1)
+{
+   const int dir = GetHexEdgeSplit(nodes, v0, v1);
+   static const int split_edges[3][4][2] =
+   {
+      {{0, 1}, {3, 2}, {4, 5}, {7, 6}},
+      {{1, 2}, {0, 3}, {5, 6}, {4, 7}},
+      {{0, 4}, {1, 5}, {2, 6}, {3, 7}}
+   };
+
+   for (int i = 0; i < 4; i++)
+   {
+      const int a = nodes[split_edges[dir][i][0]];
+      const int b = nodes[split_edges[dir][i][1]];
+      if (a == v0 && b == v1)
+      {
+         return ref.s[dir];
+      }
+      if (a == v1 && b == v0)
+      {
+         return 1.0 - ref.s[dir];
+      }
+   }
+
+   MFEM_ABORT("Shared face edge does not match the refinement direction.");
+   return 0.0;
+}
+
 ParNCMesh::ParNCMesh(MPI_Comm comm, const NCMesh &ncmesh,
                      const int *partitioning)
    : NCMesh(ncmesh)
@@ -1555,7 +1597,7 @@ bool ParNCMesh::AnisotropicConflict(const Array<Refinement> &refinements,
       ElementNeighborProcessors(elem, ranks);
       for (int j = 0; j < ranks.Size(); j++)
       {
-         send_ref[ranks[j]].AddRefinement(elem, ref.GetType());
+         send_ref[ranks[j]].AddRefinement(elem, ref);
       }
    }
 
@@ -1576,8 +1618,8 @@ bool ParNCMesh::AnisotropicConflict(const Array<Refinement> &refinements,
    for (int i = 0; i < refinements.Size(); i++)
    {
       const Refinement &ref = refinements[i];
-      CheckRefinement(leaf_elements[ref.index], ref.GetType(), refinements,
-                      elemToRef, conflicts);
+      CheckRefinement(leaf_elements[ref.index], ref, refinements, elemToRef,
+                      conflicts);
    }
 
    // Receive (ghost layer) refinements from all neighbors
@@ -1593,7 +1635,9 @@ bool ParNCMesh::AnisotropicConflict(const Array<Refinement> &refinements,
       // check the ghost refinements
       for (int i = 0; i < msg.Size(); i++)
       {
-         CheckRefinement(msg.elements[i], msg.values[i], refinements, elemToRef,
+         Refinement ghost_ref(msg.elements[i], msg.values[i].ref_type);
+         ghost_ref.SetScaleForType(msg.values[i].scale);
+         CheckRefinement(msg.elements[i], ghost_ref, refinements, elemToRef,
                          conflicts);
       }
    }
@@ -1749,7 +1793,7 @@ int FindHexFace(const int* no, int vn1, int vn2, int vn3, int vn4)
 
 // Assumption: v1 and v2 are indices of hex vertices connected by an edge.
 // The return value is {0,1,2} denoting split {X,Y,Z}.
-int GetHexEdgeSplit(const int* nodes, int v1, int v2)
+static int GetHexEdgeSplit(const int* nodes, int v1, int v2)
 {
    Array<int> v(2);
    v[0] = v1;
@@ -1780,7 +1824,8 @@ int GetHexEdgeSplit(const int* nodes, int v1, int v2)
    return edgeDir[edge];
 }
 
-void ParNCMesh::CheckRefAnisoFace(int elem, int vn1, int vn2, int vn3, int vn4,
+void ParNCMesh::CheckRefAnisoFace(const Refinement &ref, int elem,
+                                  int vn1, int vn2, int vn3, int vn4,
                                   const Array<Refinement> &refinements,
                                   const std::map<int, int> &elemToRef,
                                   std::set<int> &conflicts)
@@ -1798,11 +1843,11 @@ void ParNCMesh::CheckRefAnisoFace(int elem, int vn1, int vn2, int vn3, int vn4,
    if (elemToRef.count(nghbIndex) > 0)
    {
       const int refIndex = elemToRef.at(nghbIndex);
-      const Refinement& ref = refinements[refIndex];
+      const Refinement& nghb_ref = refinements[refIndex];
 
       bool refDir[3];
       for (int i=0; i<3; ++i)
-         refDir[i] = ref.s[i] > real_t{0};
+         refDir[i] = nghb_ref.s[i] > real_t{0};
 
       const int localFace = FindHexFace(nghb.node, vn1, vn2, vn3, vn4);
       const int faceDir = GetHexFaceDir(localFace);
@@ -1834,30 +1879,50 @@ void ParNCMesh::CheckRefAnisoFace(int elem, int vn1, int vn2, int vn3, int vn4,
          MFEM_ASSERT(cnt == 2 && hexSplitOnFace >= 0, "");
 
          const int edgeSplit = GetHexEdgeSplit(nghb.node, vn1, vn2);
-         if (edgeSplit != hexSplitOnFace) { conflicts.insert(refIndex); }
+         if (edgeSplit != hexSplitOnFace)
+         {
+            conflicts.insert(refIndex);
+         }
+         else
+         {
+            const real_t elem_scale =
+               DirectedHexEdgeScale(elements[elem].node, ref, vn1, vn2);
+            const real_t nghb_scale =
+               DirectedHexEdgeScale(nghb.node, nghb_ref, vn1, vn2);
+            if (!SameSplitScale(elem_scale, nghb_scale))
+            {
+               conflicts.insert(refIndex);
+            }
+         }
       }
    }
    // The else case is that the neighbor is not refined, so there is no need to
    // check for conflicts.
 }
 
-void ParNCMesh::CheckRefIsoFace(int elem, int vn1, int vn2, int vn3, int vn4,
+void ParNCMesh::CheckRefIsoFace(const Refinement &ref, int elem,
+                                int vn1, int vn2, int vn3, int vn4,
                                 int en1, int en2, int en3, int en4,
                                 const Array<Refinement> &refinements,
                                 const std::map<int, int> &elemToRef,
                                 std::set<int> &conflicts)
 {
-   CheckRefAnisoFace(elem, vn1, vn2, en2, en4, refinements, elemToRef, conflicts);
-   CheckRefAnisoFace(elem, en4, en2, vn3, vn4, refinements, elemToRef, conflicts);
-   CheckRefAnisoFace(elem, vn4, vn1, en1, en3, refinements, elemToRef, conflicts);
-   CheckRefAnisoFace(elem, en3, en1, vn2, vn3, refinements, elemToRef, conflicts);
+   CheckRefAnisoFace(ref, elem, vn1, vn2, en2, en4, refinements, elemToRef,
+                     conflicts);
+   CheckRefAnisoFace(ref, elem, en4, en2, vn3, vn4, refinements, elemToRef,
+                     conflicts);
+   CheckRefAnisoFace(ref, elem, vn4, vn1, en1, en3, refinements, elemToRef,
+                     conflicts);
+   CheckRefAnisoFace(ref, elem, en3, en1, vn2, vn3, refinements, elemToRef,
+                     conflicts);
 }
 
-void ParNCMesh::CheckRefinement(int elem, char ref_type,
+void ParNCMesh::CheckRefinement(int elem, const Refinement &ref,
                                 const Array<Refinement> &refinements,
                                 const std::map<int, int> &elemToRef,
                                 std::set<int> &conflicts)
 {
+   const char ref_type = ref.GetType();
    const Element &el = elements[elem];
    MFEM_ASSERT(el.geom == Geometry::CUBE && el.ref_type == 0,
                "Element must be an unrefined hexahedron");
@@ -1868,46 +1933,46 @@ void ParNCMesh::CheckRefinement(int elem, char ref_type,
    // This follows the logic of NCMesh::RefineElement().
    if (ref_type == Refinement::X) // split along X axis
    {
-      CheckRefAnisoFace(elem, no[0], no[1], no[5], no[4], refinements,
+      CheckRefAnisoFace(ref, elem, no[0], no[1], no[5], no[4], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[2], no[3], no[7], no[6], refinements,
+      CheckRefAnisoFace(ref, elem, no[2], no[3], no[7], no[6], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[4], no[5], no[6], no[7], refinements,
+      CheckRefAnisoFace(ref, elem, no[4], no[5], no[6], no[7], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[3], no[2], no[1], no[0], refinements,
+      CheckRefAnisoFace(ref, elem, no[3], no[2], no[1], no[0], refinements,
                         elemToRef, conflicts);
    }
    else if (ref_type == Refinement::Y) // split along Y axis
    {
-      CheckRefAnisoFace(elem, no[1], no[2], no[6], no[5], refinements,
+      CheckRefAnisoFace(ref, elem, no[1], no[2], no[6], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[3], no[0], no[4], no[7], refinements,
+      CheckRefAnisoFace(ref, elem, no[3], no[0], no[4], no[7], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[5], no[6], no[7], no[4], refinements,
+      CheckRefAnisoFace(ref, elem, no[5], no[6], no[7], no[4], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[0], no[3], no[2], no[1], refinements,
+      CheckRefAnisoFace(ref, elem, no[0], no[3], no[2], no[1], refinements,
                         elemToRef, conflicts);
    }
    else if (ref_type == Refinement::Z) // split along Z axis
    {
-      CheckRefAnisoFace(elem, no[4], no[0], no[1], no[5], refinements,
+      CheckRefAnisoFace(ref, elem, no[4], no[0], no[1], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[5], no[1], no[2], no[6], refinements,
+      CheckRefAnisoFace(ref, elem, no[5], no[1], no[2], no[6], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[6], no[2], no[3], no[7], refinements,
+      CheckRefAnisoFace(ref, elem, no[6], no[2], no[3], no[7], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[7], no[3], no[0], no[4], refinements,
+      CheckRefAnisoFace(ref, elem, no[7], no[3], no[0], no[4], refinements,
                         elemToRef, conflicts);
    }
    else if (ref_type == Refinement::XY) // XY split
    {
-      CheckRefAnisoFace(elem, no[0], no[1], no[5], no[4], refinements,
+      CheckRefAnisoFace(ref, elem, no[0], no[1], no[5], no[4], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[1], no[2], no[6], no[5], refinements,
+      CheckRefAnisoFace(ref, elem, no[1], no[2], no[6], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[2], no[3], no[7], no[6], refinements,
+      CheckRefAnisoFace(ref, elem, no[2], no[3], no[7], no[6], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[3], no[0], no[4], no[7], refinements,
+      CheckRefAnisoFace(ref, elem, no[3], no[0], no[4], no[7], refinements,
                         elemToRef, conflicts);
 
       const int mid01 = GetMidEdgeNode(no[0], no[1]);
@@ -1920,20 +1985,20 @@ void ParNCMesh::CheckRefinement(int elem, char ref_type,
       const int mid67 = GetMidEdgeNode(no[6], no[7]);
       const int mid74 = GetMidEdgeNode(no[7], no[4]);
 
-      CheckRefIsoFace(elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01,
+      CheckRefIsoFace(ref, elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01,
                       mid30, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67,
+      CheckRefIsoFace(ref, elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67,
                       mid74, refinements, elemToRef, conflicts);
    }
    else if (ref_type == Refinement::XZ) // XZ split
    {
-      CheckRefAnisoFace(elem, no[3], no[2], no[1], no[0], refinements,
+      CheckRefAnisoFace(ref, elem, no[3], no[2], no[1], no[0], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[2], no[6], no[5], no[1], refinements,
+      CheckRefAnisoFace(ref, elem, no[2], no[6], no[5], no[1], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[6], no[7], no[4], no[5], refinements,
+      CheckRefAnisoFace(ref, elem, no[6], no[7], no[4], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[7], no[3], no[0], no[4], refinements,
+      CheckRefAnisoFace(ref, elem, no[7], no[3], no[0], no[4], refinements,
                         elemToRef, conflicts);
 
       const int mid01 = GetMidEdgeNode(no[0], no[1]);
@@ -1946,9 +2011,9 @@ void ParNCMesh::CheckRefinement(int elem, char ref_type,
       const int mid26 = GetMidEdgeNode(no[2], no[6]);
       const int mid37 = GetMidEdgeNode(no[3], no[7]);
 
-      CheckRefIsoFace(elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45,
+      CheckRefIsoFace(ref, elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45,
                       mid04, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67,
+      CheckRefIsoFace(ref, elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67,
                       mid26, refinements, elemToRef, conflicts);
    }
    else if (ref_type == Refinement::YZ) // YZ split
@@ -1963,18 +2028,18 @@ void ParNCMesh::CheckRefinement(int elem, char ref_type,
       const int mid26 = GetMidEdgeNode(no[2], no[6]);
       const int mid37 = GetMidEdgeNode(no[3], no[7]);
 
-      CheckRefAnisoFace(elem, no[4], no[0], no[1], no[5], refinements,
+      CheckRefAnisoFace(ref, elem, no[4], no[0], no[1], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[0], no[3], no[2], no[1], refinements,
+      CheckRefAnisoFace(ref, elem, no[0], no[3], no[2], no[1], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[3], no[7], no[6], no[2], refinements,
+      CheckRefAnisoFace(ref, elem, no[3], no[7], no[6], no[2], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(elem, no[7], no[4], no[5], no[6], refinements,
+      CheckRefAnisoFace(ref, elem, no[7], no[4], no[5], no[6], refinements,
                         elemToRef, conflicts);
 
-      CheckRefIsoFace(elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56,
+      CheckRefIsoFace(ref, elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56,
                       mid15, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74,
+      CheckRefIsoFace(ref, elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74,
                       mid37, refinements, elemToRef, conflicts);
    }
    else if (ref_type == Refinement::XYZ) // XYZ split
@@ -1994,17 +2059,17 @@ void ParNCMesh::CheckRefinement(int elem, char ref_type,
       const int mid26 = GetMidEdgeNode(no[2], no[6]);
       const int mid37 = GetMidEdgeNode(no[3], no[7]);
 
-      CheckRefIsoFace(elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01,
+      CheckRefIsoFace(ref, elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01,
                       mid30, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45,
+      CheckRefIsoFace(ref, elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45,
                       mid04, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56,
+      CheckRefIsoFace(ref, elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56,
                       mid15, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67,
+      CheckRefIsoFace(ref, elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67,
                       mid26, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74,
+      CheckRefIsoFace(ref, elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74,
                       mid37, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67,
+      CheckRefIsoFace(ref, elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67,
                       mid74, refinements, elemToRef, conflicts);
    }
    else
@@ -2053,7 +2118,7 @@ void ParNCMesh::Refine(const Array<Refinement> &refinements)
       ElementNeighborProcessors(elem, ranks);
       for (int j = 0; j < ranks.Size(); j++)
       {
-         send_ref[ranks[j]].AddRefinement(elem, ref.GetType());
+         send_ref[ranks[j]].AddRefinement(elem, ref);
       }
    }
 
@@ -2063,8 +2128,9 @@ void ParNCMesh::Refine(const Array<Refinement> &refinements)
    // do local refinements
    for (int i = 0; i < refinements.Size(); i++)
    {
-      const Refinement &ref = refinements[i];
-      NCMesh::RefineElement(leaf_elements[ref.index], ref.GetType());
+      Refinement ref_i = refinements[i];
+      ref_i.index = leaf_elements[refinements[i].index];
+      NCMesh::RefineElement(ref_i);
    }
 
    // receive (ghost layer) refinements from all neighbors
@@ -2080,7 +2146,9 @@ void ParNCMesh::Refine(const Array<Refinement> &refinements)
       // do the ghost refinements
       for (int i = 0; i < msg.Size(); i++)
       {
-         NCMesh::RefineElement(msg.elements[i], msg.values[i]);
+         Refinement ghost_ref(msg.elements[i], msg.values[i].ref_type);
+         ghost_ref.SetScaleForType(msg.values[i].scale);
+         NCMesh::RefineElement(ghost_ref);
       }
    }
 

mesh/pncmesh.hpp

@@ -497,11 +497,27 @@ protected: // implementation
    /** Used by ParNCMesh::Refine() to inform neighbors about refinements at
     *  the processor boundary. This keeps their ghost layers synchronized.
     */
-   class NeighborRefinementMessage : public ElementValueMessage<char, false,
-      VarMessageTag::NEIGHBOR_REFINEMENT_VM>
+   struct NeighborRefinement
+   {
+      char ref_type;
+      real_t scale[3];
+   };
+
+   class NeighborRefinementMessage
+      : public ElementValueMessage<NeighborRefinement, false,
+        VarMessageTag::NEIGHBOR_REFINEMENT_VM>
    {
    public:
-      void AddRefinement(int elem, char ref_type) { Add(elem, ref_type); }
+      void AddRefinement(int elem, const Refinement &ref)
+      {
+         NeighborRefinement data{};
+         data.ref_type = ref.GetType();
+         for (int i = 0; i < 3; i++)
+         {
+            data.scale[i] = ref.s[i];
+         }
+         Add(elem, data);
+      }
       typedef std::map<int, NeighborRefinementMessage> Map;
    };
 
@@ -602,7 +618,8 @@ protected: // implementation
    /** For the face with ordered vertices vn* and neighboring element @a elem,
        check whether the other neighboring element (if it exists) is marked for
        a horizontal refinement conflicting with a vertical split. */
-   void CheckRefAnisoFace(int elem, int vn1, int vn2, int vn3, int vn4,
+   void CheckRefAnisoFace(const Refinement &ref, int elem,
+                          int vn1, int vn2, int vn3, int vn4,
                           const Array<Refinement> &refinements,
                           const std::map<int, int> &elemToRef,
                           std::set<int> &conflicts);
@@ -611,7 +628,8 @@ protected: // implementation
        neighboring element @a elem, check whether the other neighboring element
        (if it exists) is marked for a refinement conflicting with an isotropic
        refinement of the face. */
-   void CheckRefIsoFace(int elem, int vn1, int vn2, int vn3, int vn4,
+   void CheckRefIsoFace(const Refinement &ref, int elem,
+                        int vn1, int vn2, int vn3, int vn4,
                         int en1, int en2, int en3, int en4,
                         const Array<Refinement> &refinements,
                         const std::map<int, int> &elemToRef,
@@ -622,9 +640,8 @@ protected: // implementation
                               const std::map<int, int> &elemToRef,
                               std::set<int> &conflicts);
 
-   /** Check whether the refinement of the element with index @a elem and type
-       @a ref_type would cause a conflict. */
-   void CheckRefinement(int elem, char ref_type,
+   /// Check whether the input refinement would cause a conflict.
+   void CheckRefinement(int elem, const Refinement &ref,
                         const Array<Refinement> &refinements,
                         const std::map<int, int> &elemToRef,
                         std::set<int> &conflicts);

miniapps/meshing/CMakeLists.txt

@@ -151,6 +151,10 @@ if (MFEM_USE_MPI)
     MAIN phpref.cpp
     LIBRARIES mfem)
 
+  add_mfem_miniapp(pref321
+    MAIN pref321.cpp
+    LIBRARIES mfem)
+
   # Add parallel tests.
   if (MFEM_ENABLE_TESTING)
     set(PARALLEL_TESTS
@@ -160,6 +164,7 @@ if (MFEM_USE_MPI)
       fit-node-position
       pminimal-surface
       phpref
+      pref321
     )
     # Meshing miniapps that return MFEM_SKIP_RETURN_VALUE in some cases:
     set(SKIP_TESTS)

miniapps/meshing/makefile

@@ -24,7 +24,7 @@ SEQ_MINIAPPS = mobius-strip klein-bottle toroid trimmer twist mesh-explorer\
 	shaper extruder mesh-optimizer minimal-surface polar-nc reflector\
 	ref321 mesh-quality hpref
 PAR_MINIAPPS = pmesh-optimizer pminimal-surface pmesh-fitting fit-node-position\
-	phpref mesh-bounding-boxes
+	phpref pref321 mesh-bounding-boxes
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
 else
@@ -99,6 +99,8 @@ hpref-test-seq: hpref
 	@$(call mfem-test,$<,, Serial hp-refinement)
 phpref-test-par: phpref
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel hp-refinement)
+pref321-test-par: pref321
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel 3:1 refinement)
 mesh-bounding-boxes-test-par: mesh-bounding-boxes
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel bounding boxes)
 ref321-test-seq: ref321

miniapps/meshing/pref321.cpp

@@ -0,0 +1,336 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+//
+//      -----------------------------------------------------------------
+//      3:1 Refinement Miniapp: Parallel 3:1 anisotropic mesh refinements
+//      -----------------------------------------------------------------
+//
+// This miniapp performs random 3:1 refinements of a quadrilateral or hexahedral
+// mesh. A diffusion equation is solved in an H1 finite element space defined on
+// the refined mesh, and its continuity is verified across local and shared
+// faces.
+//
+// Compile with: make pref321
+//
+// Sample runs:  mpirun -np 4 pref321 -mm -dim 2 -o 2 -r 100
+//               mpirun -np 4 pref321 -mm -dim 3 -o 2 -r 100
+//               mpirun -np 4 pref321 -m ../../data/star.mesh -o 2 -r 100
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+real_t CheckH1Continuity(ParGridFunction &x);
+
+// Find the two children of parent element `elem` after its refinement in one
+// direction.
+void FindChildren(const Mesh &mesh, int elem, Array<int> &children)
+{
+   const CoarseFineTransformations &cf = mesh.ncmesh->GetRefinementTransforms();
+   MFEM_ASSERT(mesh.GetNE() == cf.embeddings.Size(), "");
+
+   // Note that row `elem` of the table constructed by cf.MakeCoarseToFineTable
+   // is an alternative to this global loop, but constructing the table is also
+   // a global operation with global storage.
+   for (int i = 0; i < mesh.GetNE(); i++)
+   {
+      const int p = cf.embeddings[i].parent;
+      if (p == elem)
+      {
+         children.Append(i);
+      }
+   }
+}
+
+// Refine 3:1 via 2 refinements with scalings 2/3 and 1/2.
+void Refine31(Mesh &mesh, int elem, char type)
+{
+   Array<Refinement> refs; // Refinement is defined in ncmesh.hpp
+   refs.Append(Refinement(elem, type, 2.0 / 3.0));
+   mesh.GeneralRefinement(refs);
+
+   // Find the elements with parent `elem`
+   Array<int> children;
+   FindChildren(mesh, elem, children);
+   MFEM_ASSERT(children.Size() == 2, "");
+
+   const int elem1 = children[0];
+
+   refs.SetSize(0);
+   refs.Append(Refinement(elem1, type)); // Default scaling of 0.5
+   mesh.GeneralRefinement(refs);
+}
+
+// Randomly select elements for 3:1 refinements in random directions.
+void TestAnisoRefRandom(int num_refs, int dim, ParMesh &mesh, int myid,
+                        int seed = 0)
+{
+   std::mt19937 gen(seed);
+   for (int i = 0; i < num_refs; i++)
+   {
+      const int elem = gen() % mesh.GetNE();
+      const int t = gen() % dim;
+      auto type = t == 0 ? Refinement::X :
+                  (t == 1 ? Refinement::Y : Refinement::Z);
+
+      // In 3D, check for conflicts in the parallel refinements.
+      if (dim == 3)
+      {
+         std::set<int> conflicts; // Indices in refs of conflicting elements
+         Array<Refinement> refs;
+         refs.Append(Refinement(elem, type));
+         const bool conflict = mesh.AnisotropicConflict(refs, conflicts);
+         if (conflict)
+         {
+            if (myid == 0)
+               cout << "Anisotropic conflict on iteration " << i
+                    << ", retrying\n";
+            i--;
+            continue;
+         }
+      }
+
+      Refine31(mesh, elem, type);
+   }
+
+   mesh.EnsureNodes();
+   mesh.SetScaledNCMesh();
+}
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init(argc, argv);
+   Hypre::Init();
+
+   const int num_procs = Mpi::WorldSize();
+   const int myid = Mpi::WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int order = 1;
+   bool visualization = true;
+   bool makeMesh = false;
+   int num_refs = 1;
+   int tdim = 2;  // Mesh dimension for Cartesian meshes.
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&makeMesh, "-mm", "--make-mesh", "-no-mm",
+                  "--no-make-mesh", "Create Cartesian mesh");
+   args.AddOption(&tdim, "-dim", "--dimension", "Dimension for Cartesian mesh");
+   args.AddOption(&num_refs, "-r", "--refs", "Number of 3:1 refinements");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Create or read the serial mesh on all ranks, then apply the same
+   //    deterministic 3:1 refinement sequence before partitioning it.
+   Mesh mesh;
+   if (makeMesh)
+   {
+      mesh = tdim == 3 ? Mesh::MakeCartesian3D(2, 2, 2, Element::HEXAHEDRON) :
+             Mesh::MakeCartesian2D(2, 2, Element::QUADRILATERAL);
+   }
+   else
+   {
+      mesh = Mesh::LoadFromFile(mesh_file, 1, 1);
+   }
+
+   const int dim = mesh.Dimension();
+
+   mesh.EnsureNCMesh();
+   mesh.SetScaledNCMesh();
+
+   // 3. Partition the refined serial mesh.
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   TestAnisoRefRandom(num_refs, dim, pmesh, myid, myid);
+
+   // 4. Define a parallel H1 finite element space and report its global size.
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: "
+           << fespace.GlobalTrueVSize() << endl;
+   }
+
+   // 5. Assemble and solve the Poisson problem, following ex1p.
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   ParLinearForm b(&fespace);
+   ConstantCoefficient one(1.0);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
+
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator());
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   Array<int> ess_tdof_list;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      ess_bdr = 0;
+      pmesh.MarkExternalBoundaries(ess_bdr);
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   HypreBoomerAMG M;
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X, b, x);
+
+   // 6. Verify the continuity of the solution in H1 over local and shared
+   //    faces and compute the global maximum jump.
+   const real_t h1err = CheckH1Continuity(x);
+   if (myid == 0)
+   {
+      cout << "Error of H1 continuity: " << h1err << endl;
+   }
+   MFEM_VERIFY(h1err < 1.0e-7, "H1 discontinuity found");
+
+   // 7. Save the refined mesh and the solution in parallel. This output can
+   //    be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 8. Send the parallel solution to GLVis.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int visport = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x << flush;
+   }
+
+   return 0;
+}
+
+real_t CheckH1Continuity(ParGridFunction &x)
+{
+   const ParFiniteElementSpace *pfes = x.ParFESpace();
+   ParMesh *pmesh = pfes->GetParMesh();
+   const int dim = pmesh->Dimension();
+
+   real_t errorMax = 0.0;
+
+   // Shared-face values require face-neighbor data.
+   x.ExchangeFaceNbrData();
+
+   // First handle faces for which both elements are local to this rank.
+   for (int f = 0; f < pmesh->GetNumFaces(); f++)
+   {
+      const auto info = pmesh->GetFaceInformation(f);
+      if (!info.IsLocal())
+      {
+         continue;
+      }
+
+      FaceElementTransformations *FT = pmesh->GetFaceElementTransformations(f);
+      const int faceOrder = dim == 3 ? pfes->GetFaceOrder(f) :
+                            pfes->GetEdgeOrder(f);
+      const IntegrationRule &ir = IntRules.Get(FT->FaceGeom, 2 * faceOrder);
+
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         const IntegrationPoint &fip = ir.IntPoint(i);
+         IntegrationPoint ip1, ip2;
+
+         FT->Loc1.Transform(fip, ip1);
+         FT->Loc2.Transform(fip, ip2);
+
+         const real_t v1 = x.GetValue(*FT->Elem1, ip1);
+         const real_t v2 = x.GetValue(*FT->Elem2, ip2);
+         errorMax = std::max(errorMax, std::abs(v1 - v2));
+      }
+   }
+
+   // Then check partition interfaces. Conforming shared faces are handled on
+   // the lower-rank side, while shared slave nonconforming faces are handled
+   // only on the slave side and therefore do not need additional filtering.
+   for (int sf = 0; sf < pmesh->GetNSharedFaces(); sf++)
+   {
+      const int f = pmesh->GetSharedFace(sf);
+      const auto info = pmesh->GetFaceInformation(f);
+      if (!info.IsShared())
+      {
+         continue;
+      }
+
+      FaceElementTransformations *FT = pmesh->GetSharedFaceTransformations(sf);
+      const int faceOrder = dim == 3 ? pfes->GetFaceOrder(f) :
+                            pfes->GetEdgeOrder(f);
+      const IntegrationRule &ir = IntRules.Get(FT->FaceGeom, 2 * faceOrder);
+
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         const IntegrationPoint &fip = ir.IntPoint(i);
+         IntegrationPoint ip1, ip2;
+
+         FT->Loc1.Transform(fip, ip1);
+         FT->Loc2.Transform(fip, ip2);
+
+         const real_t v1 = x.GetValue(*FT->Elem1, ip1);
+         const real_t v2 = x.GetValue(*FT->Elem2, ip2);
+         errorMax = std::max(errorMax, std::abs(v1 - v2));
+      }
+   }
+
+   MPI_Allreduce(MPI_IN_PLACE, &errorMax, 1, MFEM_MPI_REAL_T, MPI_MAX,
+                 pmesh->GetComm());
+
+   return errorMax;
+}

miniapps/meshing/ref321.cpp

@@ -71,22 +71,14 @@ void Refine31(Mesh & mesh, int elem, char type)
    mesh.GeneralRefinement(refs);
 }
 
-// Deterministic, somewhat random integer generator
-int MyRand(int & s)
-{
-   s++;
-   const double a = 1000 * sin(s * 1.1234 * M_PI);
-   return int(std::abs(a));
-}
-
 // Randomly select elements for 3:1 refinements in random directions.
-void TestAnisoRefRandom(int iter, int dim, Mesh & mesh)
+void TestAnisoRefRandom(int num_refs, int dim, Mesh & mesh)
 {
-   int seed = 0;
-   for (int i = 0; i < iter; i++)
+   std::mt19937 gen(1);
+   for (int i = 0; i < num_refs; i++)
    {
-      const int elem = MyRand(seed) % mesh.GetNE();
-      const int t = MyRand(seed) % dim;
+      const auto elem = gen() % mesh.GetNE();
+      const auto t = gen() % dim;
       auto type = t == 0 ? Refinement::X :
                   (t == 1 ? Refinement::Y : Refinement::Z);
       Refine31(mesh, elem, type);

miniapps/plasma/CMakeLists.txt

@@ -9,6 +9,12 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
+add_mfem_miniapp(g_eqdsk_viewer
+  MAIN g_eqdsk_viewer.cpp
+  EXTRA_SOURCES g_eqdsk_data.cpp
+  EXTRA_HEADERS g_eqdsk_data.hpp plasma.hpp ${MFEM_MINIAPPS_COMMON_HEADERS}
+  LIBRARIES mfem mfem-common)
+
 if (MFEM_USE_MPI)
   list(APPEND PLASMA_COMMON_SOURCES)
 

miniapps/plasma/dbl_null.g_eqdsk

@@ -0,0 +1,211 @@
+  SGRRATEI    01/01/2025    #1  0ms           3 20 40
+ 1.000000000E+00 2.000000000E+00 1.000000000E+00 5.000000000E-01 0.000000000E+00
+ 1.104715063E+00 5.225686728E-06-9.087992965E+00 0.000000000E+00 1.000000000E+00
+ 3.335115355E+05-9.087992965E+00 0.000000000E+00 1.104715063E+00 0.000000000E+00
+ 5.225686728E-06 0.000000000E+00 0.000000000E+00 0.000000000E+00 0.000000000E+00
+ 9.784708692E-01 9.806991943E-01 9.828024193E-01 9.847813458E-01 9.866367217E-01
+ 9.883692428E-01 9.899795542E-01 9.914682512E-01 9.928358810E-01 9.940829432E-01
+ 9.952098911E-01 9.962171323E-01 9.971050296E-01 9.978739015E-01 9.985240230E-01
+ 9.990556259E-01 9.994688994E-01 9.997639901E-01 9.999410027E-01 1.000000000E+00
+ 4.707754379E+04 4.225242157E+04 3.768811677E+04 3.338462939E+04 2.934195942E+04
+ 2.556010688E+04 2.203907174E+04 1.877885403E+04 1.577945373E+04 1.304087085E+04
+ 1.056310539E+04 8.346157347E+03 6.390026719E+03 4.694713508E+03 3.260217714E+03
+ 2.086539337E+03 1.173678377E+03 5.216348342E+02 1.304087085E+02 0.000000000E+00
+-4.353196346E-02-4.114710095E-02-3.877798694E-02-3.642358799E-02-3.408289990E-02
+-3.175494531E-02-2.943877149E-02-2.713344826E-02-2.483806598E-02-2.255173364E-02
+-2.027357702E-02-1.800273697E-02-1.573836773E-02-1.347963528E-02-1.122571577E-02
+-8.975794007E-03-6.729061934E-03-4.484717187E-03-2.241961644E-03-0.000000000E+00
+ 9.415508757E+04 8.919955665E+04 8.424402572E+04 7.928849480E+04 7.433296387E+04
+ 6.937743295E+04 6.442190202E+04 5.946637110E+04 5.451084017E+04 4.955530925E+04
+ 4.459977832E+04 3.964424740E+04 3.468871647E+04 2.973318555E+04 2.477765462E+04
+ 1.982212370E+04 1.486659277E+04 9.911061850E+03 4.955530925E+03 0.000000000E+00
+-3.910241380E-01-3.370349374E-01-3.032206099E-01-2.833941950E-01-2.709124845E-01
+-2.589045274E-01-2.408148723E-01-2.113596788E-01-1.679192587E-01-1.122605207E-01
+-5.229150156E-02-3.313845027E-03 1.199143413E-02-3.426047302E-02-1.700331905E-01
+-4.150011731E-01-7.696358166E-01-1.203902834E+00-1.649387010E+00-1.999925152E+00
+-1.925695622E-01-1.748346049E-01-1.691266514E-01-1.701675452E-01-1.715397928E-01
+-1.659119623E-01-1.456077592E-01-1.036478534E-01-3.533092381E-02 5.970389126E-02
+ 1.751878198E-01 2.962216198E-01 3.983584318E-01 4.487753542E-01 4.105383363E-01
+ 2.505857725E-01-4.884633508E-02-4.770899859E-01-9.837399355E-01-1.474339619E+00
+ 2.166315348E-02-1.102851387E-03-2.831065458E-02-5.546599960E-02-7.618766318E-02
+-8.237925771E-02-6.477862541E-02-1.405299099E-02 7.743811746E-02 2.132045638E-01
+ 3.897048312E-01 5.934227400E-01 7.987616348E-01 9.678852363E-01 1.053770529E+00
+ 1.007512540E+00 7.901446298E-01 3.878102911E-01-1.728443713E-01-8.142430666E-01
+ 2.411392375E-01 1.747358186E-01 1.103546220E-01 5.181206754E-02 5.588785514E-03
+-1.923578803E-02-1.146084602E-02 4.113304468E-02 1.498681805E-01 3.223534496E-01
+ 5.588811278E-01 8.484640691E-01 1.165346084E+00 1.467240961E+00 1.696851978E+00
+ 1.788184124E+00 1.678546763E+00 1.325771819E+00 7.280476985E-01-5.875813012E-02
+ 4.562990380E-01 3.439881454E-01 2.385637023E-01 1.432261283E-01 6.461505077E-02
+ 1.290331417E-02 1.448926330E-03 4.574285831E-02 1.613919171E-01 3.609574823E-01
+ 6.496681822E-01 1.020362387E+00 1.448488467E+00 1.888534307E+00 2.273730093E+00
+ 2.521039729E+00 2.543035410E+00 2.266964141E+00 1.659051976E+00 7.490835809E-01
+ 6.589958226E-01 4.991505383E-01 3.490864277E-01 2.113878302E-01 9.282362412E-02
+ 4.651701261E-03-3.753931076E-02-1.476326317E-02 9.331703089E-02 3.049501109E-01
+ 6.313711445E-01 1.070656937E+00 1.601160717E+00 2.175992078E+00 2.720663983E+00
+ 3.136428575E+00 3.311623934E+00 3.142190847E+00 2.560153749E+00 1.565395650E+00
+ 8.427963399E-01 6.342571609E-01 4.361842129E-01 2.504754676E-01 8.390705039E-02
+-5.130487543E-02-1.374118370E-01-1.518716892E-01-6.933566292E-02 1.347294416E-01
+ 4.785698497E-01 9.669780041E-01 1.583175086E+00 2.281269353E+00 2.981657583E+00
+ 3.572374734E+00 3.919441608E+00 3.888257701E+00 3.375670467E+00 2.348486607E+00
+ 1.003140271E+00 7.451051663E-01 4.959126068E-01 2.566355010E-01 3.385060138E-02
+-1.595013481E-01-3.037145085E-01-3.727832712E-01-3.362296400E-01-1.627893604E-01
+ 1.736969869E-01 6.862115526E-01 1.364964594E+00 2.167824224E+00 3.013335205E+00
+ 3.779769521E+00 4.313957864E+00 4.452847541E+00 4.058327204E+00 3.061671981E+00
+ 1.137365128E+00 8.293536487E-01 5.262966190E-01 2.282463102E-01-5.869372598E-02
+-3.211771932E-01-5.378523046E-01-6.794765186E-01-7.104791334E-01-5.925787521E-01
+-2.909412188E-01 2.169977855E-01 9.305878226E-01 1.814416755E+00 2.788864587E+00
+ 3.726574504E+00 4.459237386E+00 4.798518474E+00 4.572628903E+00 3.675635708E+00
+ 1.244611695E+00 8.865011676E-01 5.273739090E-01 1.660289873E-01-1.922188349E-01
+-5.339924512E-01-8.366835819E-01-1.068145986E+00-1.187890541E+00-1.150497326E+00
+-9.118739899E-01-4.386474144E-01 2.797067814E-01 1.217442845E+00 2.300606858E+00
+ 3.400695883E+00 4.338887158E+00 4.905511692E+00 4.897269966E+00 4.170164958E+00
+ 1.325631269E+00 9.177537088E-01 5.011092625E-01 7.300028320E-02-3.623736298E-01
+-7.920000920E-01-1.192445606E+00-1.529067475E+00-1.756748529E+00-1.822973413E+00
+-1.674020591E+00-1.264731297E+00-5.716334778E-01 3.919046896E-01 1.561272647E+00
+ 2.811295637E+00 3.957459998E+00 4.773149919E+00 5.026398786E+00 4.535133275E+00
+ 1.382519568E+00 9.258003322E-01 4.511913373E-01-4.572686077E-02-5.621735094E-01
+-1.085880267E+00-1.593023216E+00-2.046909304E+00-2.398174860E+00-2.587412144E+00
+-2.551140860E+00-2.231766925E+00-1.591501157E+00-6.290735117E-01 6.034811432E-01
+ 1.988428128E+00 3.340201296E+00 4.419719356E+00 4.969646189E+00 4.770667398E+00
+ 1.418403250E+00 9.145173783E-01 3.827290445E-01-1.832800729E-01-7.823963566E-01
+-1.403422928E+00-2.022546791E+00-2.601474899E+00-3.087043844E+00-3.413308233E+00
+-3.507162143E+00-3.298318507E+00-2.733827400E+00-1.796161496E+00-5.222552154E-01
+ 9.810231774E-01 2.531156764E+00 3.880848348E+00 4.750925106E+00 4.886498468E+00
+ 1.437103919E+00 8.886239873E-01 3.018699076E-01-3.314779846E-01-1.012137361E+00
+-1.730227221E+00-2.462280919E+00-3.168828947E+00-3.793395216E+00-4.263991651E+00
+-4.498294398E+00-4.413504634E+00-3.941288135E+00-3.047034463E+00-1.750973347E+00
+-1.466326055E-01 1.589788832E+00 3.206515141E+00 4.406106379E+00 4.900563280E+00
+ 1.442801674E+00 8.533119757E-01 2.153661342E-01-4.813638998E-01-1.239485158E+00
+-2.050570243E+00-2.891743239E+00-3.722729788E+00-4.484246464E+00-5.098883769E+00
+-5.475783196E+00-5.520279043E+00-5.149107296E+00-4.310641572E+00-3.007869332E+00
+-1.319645546E+00 5.863673938E-01 2.456925688E+00 3.979764627E+00 4.836980711E+00
+ 1.439716341E+00 8.138721012E-01 1.301159276E-01-6.237874768E-01-1.452273520E+00
+-2.348385346E+00-3.289975383E+00-4.236266930E+00-5.125675247E+00-5.876101086E+00
+-6.389099647E+00-6.559246706E+00-6.289497376E+00-5.512208788E+00-4.213722607E+00
+-2.458127258E+00-4.035022091E-01 1.697591881E+00 3.521261516E+00 4.723577581E+00
+ 1.431820151E+00 7.753367344E-01 5.270779132E-02-7.499966727E-01-1.638857755E+00
+-2.608282055E+00-3.636876158E+00-4.683586303E+00-5.685020856E+00-6.555215726E+00
+-7.189331085E+00-7.472729897E+00-7.296402003E+00-6.578594532E+00-5.290716634E+00
+-3.483232525E+00-1.304782694E+00 9.940036866E-01 3.080485788E+00 4.589173208E+00
+ 1.422591282E+00 7.421562942E-01-1.100563552E-02-8.522020247E-01-1.788863321E+00
+-2.816536419E+00-3.914501953E+00-5.041578424E+00-6.133043547E+00-7.099968471E+00
+-7.832519316E+00-8.208777881E+00-8.110180067E+00-7.443587519E+00-6.168205808E+00
+-4.323151304E+00-2.048874175E+00 4.063205511E-01 2.703595819E+00 4.460849974E+00
+ 1.414813950E+00 7.179237979E-01-5.628015260E-02-9.240774770E-01-1.893856117E+00
+-2.961982431E+00-4.108241194E+00-5.291406587E+00-6.445882623E+00-7.480732414E+00
+-8.282695715E+00-8.724816600E+00-8.681873871E+00-8.052739203E+00-6.787981369E+00
+-4.918586303E+00-2.578986186E+00-1.549450246E-02 2.429109334E+00 4.361436888E+00
+ 1.410427849E+00 7.051587764E-01-7.977847127E-02-9.611662324E-01-1.947889219E+00
+-3.036741970E+00-4.207778410E+00-5.419763372E+00-6.606668398E+00-7.676543111E+00
+-8.514384415E+00-8.990661063E+00-8.976734576E+00-8.367358201E+00-7.108627276E+00
+-5.227287344E+00-2.854588356E+00-2.357101966E-01 2.284656177E+00 4.307414559E+00
+ 1.410427849E+00 7.051587764E-01-7.977847127E-02-9.611662324E-01-1.947889219E+00
+-3.036741970E+00-4.207778410E+00-5.419763372E+00-6.606668398E+00-7.676543111E+00
+-8.514384415E+00-8.990661063E+00-8.976734576E+00-8.367358201E+00-7.108627276E+00
+-5.227287344E+00-2.854588356E+00-2.357101966E-01 2.284656177E+00 4.307414559E+00
+ 1.414813950E+00 7.179237979E-01-5.628015260E-02-9.240774770E-01-1.893856117E+00
+-2.961982431E+00-4.108241194E+00-5.291406587E+00-6.445882623E+00-7.480732414E+00
+-8.282695715E+00-8.724816600E+00-8.681873871E+00-8.052739203E+00-6.787981369E+00
+-4.918586303E+00-2.578986186E+00-1.549450246E-02 2.429109334E+00 4.361436888E+00
+ 1.422591282E+00 7.421562942E-01-1.100563552E-02-8.522020247E-01-1.788863321E+00
+-2.816536419E+00-3.914501953E+00-5.041578424E+00-6.133043547E+00-7.099968471E+00
+-7.832519316E+00-8.208777881E+00-8.110180067E+00-7.443587519E+00-6.168205808E+00
+-4.323151304E+00-2.048874175E+00 4.063205511E-01 2.703595819E+00 4.460849974E+00
+ 1.431820151E+00 7.753367344E-01 5.270779132E-02-7.499966727E-01-1.638857755E+00
+-2.608282055E+00-3.636876158E+00-4.683586303E+00-5.685020856E+00-6.555215726E+00
+-7.189331085E+00-7.472729897E+00-7.296402003E+00-6.578594532E+00-5.290716634E+00
+-3.483232525E+00-1.304782694E+00 9.940036866E-01 3.080485788E+00 4.589173208E+00
+ 1.439716341E+00 8.138721012E-01 1.301159276E-01-6.237874768E-01-1.452273520E+00
+-2.348385346E+00-3.289975383E+00-4.236266930E+00-5.125675247E+00-5.876101086E+00
+-6.389099647E+00-6.559246706E+00-6.289497376E+00-5.512208788E+00-4.213722607E+00
+-2.458127258E+00-4.035022091E-01 1.697591881E+00 3.521261516E+00 4.723577581E+00
+ 1.442801674E+00 8.533119757E-01 2.153661342E-01-4.813638998E-01-1.239485158E+00
+-2.050570243E+00-2.891743239E+00-3.722729788E+00-4.484246464E+00-5.098883769E+00
+-5.475783196E+00-5.520279043E+00-5.149107296E+00-4.310641572E+00-3.007869332E+00
+-1.319645546E+00 5.863673938E-01 2.456925688E+00 3.979764627E+00 4.836980711E+00
+ 1.437103919E+00 8.886239873E-01 3.018699076E-01-3.314779846E-01-1.012137361E+00
+-1.730227221E+00-2.462280919E+00-3.168828947E+00-3.793395216E+00-4.263991651E+00
+-4.498294398E+00-4.413504634E+00-3.941288135E+00-3.047034463E+00-1.750973347E+00
+-1.466326055E-01 1.589788832E+00 3.206515141E+00 4.406106379E+00 4.900563280E+00
+ 1.418403250E+00 9.145173783E-01 3.827290445E-01-1.832800729E-01-7.823963566E-01
+-1.403422928E+00-2.022546791E+00-2.601474899E+00-3.087043844E+00-3.413308233E+00
+-3.507162143E+00-3.298318507E+00-2.733827400E+00-1.796161496E+00-5.222552154E-01
+ 9.810231774E-01 2.531156764E+00 3.880848348E+00 4.750925106E+00 4.886498468E+00
+ 1.382519568E+00 9.258003322E-01 4.511913373E-01-4.572686077E-02-5.621735094E-01
+-1.085880267E+00-1.593023216E+00-2.046909304E+00-2.398174860E+00-2.587412144E+00
+-2.551140860E+00-2.231766925E+00-1.591501157E+00-6.290735117E-01 6.034811432E-01
+ 1.988428128E+00 3.340201296E+00 4.419719356E+00 4.969646189E+00 4.770667398E+00
+ 1.325631269E+00 9.177537088E-01 5.011092625E-01 7.300028320E-02-3.623736298E-01
+-7.920000920E-01-1.192445606E+00-1.529067475E+00-1.756748529E+00-1.822973413E+00
+-1.674020591E+00-1.264731297E+00-5.716334778E-01 3.919046896E-01 1.561272647E+00
+ 2.811295637E+00 3.957459998E+00 4.773149919E+00 5.026398786E+00 4.535133275E+00
+ 1.244611695E+00 8.865011676E-01 5.273739090E-01 1.660289873E-01-1.922188349E-01
+-5.339924512E-01-8.366835819E-01-1.068145986E+00-1.187890541E+00-1.150497326E+00
+-9.118739899E-01-4.386474144E-01 2.797067814E-01 1.217442845E+00 2.300606858E+00
+ 3.400695883E+00 4.338887158E+00 4.905511692E+00 4.897269966E+00 4.170164958E+00
+ 1.137365128E+00 8.293536487E-01 5.262966190E-01 2.282463102E-01-5.869372598E-02
+-3.211771932E-01-5.378523046E-01-6.794765186E-01-7.104791334E-01-5.925787521E-01
+-2.909412188E-01 2.169977855E-01 9.305878226E-01 1.814416755E+00 2.788864587E+00
+ 3.726574504E+00 4.459237386E+00 4.798518474E+00 4.572628903E+00 3.675635708E+00
+ 1.003140271E+00 7.451051663E-01 4.959126068E-01 2.566355010E-01 3.385060138E-02
+-1.595013481E-01-3.037145085E-01-3.727832712E-01-3.362296400E-01-1.627893604E-01
+ 1.736969869E-01 6.862115526E-01 1.364964594E+00 2.167824224E+00 3.013335205E+00
+ 3.779769521E+00 4.313957864E+00 4.452847541E+00 4.058327204E+00 3.061671981E+00
+ 8.427963399E-01 6.342571609E-01 4.361842129E-01 2.504754676E-01 8.390705039E-02
+-5.130487543E-02-1.374118370E-01-1.518716892E-01-6.933566292E-02 1.347294416E-01
+ 4.785698497E-01 9.669780041E-01 1.583175086E+00 2.281269353E+00 2.981657583E+00
+ 3.572374734E+00 3.919441608E+00 3.888257701E+00 3.375670467E+00 2.348486607E+00
+ 6.589958226E-01 4.991505383E-01 3.490864277E-01 2.113878302E-01 9.282362412E-02
+ 4.651701261E-03-3.753931076E-02-1.476326317E-02 9.331703089E-02 3.049501109E-01
+ 6.313711445E-01 1.070656937E+00 1.601160717E+00 2.175992078E+00 2.720663983E+00
+ 3.136428575E+00 3.311623934E+00 3.142190847E+00 2.560153749E+00 1.565395650E+00
+ 4.562990380E-01 3.439881454E-01 2.385637023E-01 1.432261283E-01 6.461505077E-02
+ 1.290331417E-02 1.448926330E-03 4.574285831E-02 1.613919171E-01 3.609574823E-01
+ 6.496681822E-01 1.020362387E+00 1.448488467E+00 1.888534307E+00 2.273730093E+00
+ 2.521039729E+00 2.543035410E+00 2.266964141E+00 1.659051976E+00 7.490835809E-01
+ 2.411392375E-01 1.747358186E-01 1.103546220E-01 5.181206754E-02 5.588785514E-03
+-1.923578803E-02-1.146084602E-02 4.113304468E-02 1.498681805E-01 3.223534496E-01
+ 5.588811278E-01 8.484640691E-01 1.165346084E+00 1.467240961E+00 1.696851978E+00
+ 1.788184124E+00 1.678546763E+00 1.325771819E+00 7.280476985E-01-5.875813012E-02
+ 2.166315348E-02-1.102851387E-03-2.831065458E-02-5.546599960E-02-7.618766318E-02
+-8.237925771E-02-6.477862541E-02-1.405299099E-02 7.743811746E-02 2.132045638E-01
+ 3.897048312E-01 5.934227400E-01 7.987616348E-01 9.678852363E-01 1.053770529E+00
+ 1.007512540E+00 7.901446298E-01 3.878102911E-01-1.728443713E-01-8.142430666E-01
+-1.925695622E-01-1.748346049E-01-1.691266514E-01-1.701675452E-01-1.715397928E-01
+-1.659119623E-01-1.456077592E-01-1.036478534E-01-3.533092381E-02 5.970389126E-02
+ 1.751878198E-01 2.962216198E-01 3.983584318E-01 4.487753542E-01 4.105383363E-01
+ 2.505857725E-01-4.884633508E-02-4.770899859E-01-9.837399355E-01-1.474339619E+00
+-3.910241380E-01-3.370349374E-01-3.032206099E-01-2.833941950E-01-2.709124845E-01
+-2.589045274E-01-2.408148723E-01-2.113596788E-01-1.679192587E-01-1.122605207E-01
+-5.229150156E-02-3.313845027E-03 1.199143413E-02-3.426047302E-02-1.700331905E-01
+-4.150011731E-01-7.696358166E-01-1.203902834E+00-1.649387010E+00-1.999925152E+00
+ 1.000000000E+00 1.050395383E+00 1.108778414E+00 1.174574361E+00 1.246881957E+00
+ 1.327322223E+00 1.420854260E+00 1.523601296E+00 1.644396040E+00 1.782867911E+00
+ 1.944699895E+00 2.137823164E+00 2.371490866E+00 2.660870810E+00 3.022189358E+00
+ 3.498374954E+00 4.155225142E+00 5.164243345E+00 7.000356534E+00 9.663564712E+00
+   60   11
+ 8.000000000E-01 8.000000000E-01 7.743225129E-01 7.532775834E-01 7.502189395E-01
+ 7.047911204E-01 7.277457261E-01 6.546541578E-01 7.069555012E-01 6.029841077E-01
+ 6.878969518E-01 5.499019727E-01 6.706147099E-01 4.955320618E-01 6.551492475E-01
+ 4.400017001E-01 6.415367820E-01 3.834409301E-01 6.298091915E-01 3.259822075E-01
+ 6.199939400E-01 2.677600906E-01 6.121140130E-01 2.089109256E-01 6.061878641E-01
+ 1.495725272E-01 6.022293713E-01 8.988385560E-02 6.002478046E-01 2.998469153E-02
+ 6.002478046E-01-2.998469153E-02 6.022293713E-01-8.988385560E-02 6.061878641E-01
+-1.495725272E-01 6.121140130E-01-2.089109256E-01 6.199939400E-01-2.677600906E-01
+ 6.298091915E-01-3.259822075E-01 6.415367820E-01-3.834409301E-01 6.551492475E-01
+-4.400017001E-01 6.706147099E-01-4.955320618E-01 6.878969518E-01-5.499019727E-01
+ 7.069555012E-01-6.029841077E-01 7.277457261E-01-6.546541578E-01 7.502189395E-01
+-7.047911204E-01 7.743225129E-01-7.532775834E-01 8.000000000E-01-8.000000000E-01
+ 8.000000000E-01-8.000000000E-01 8.770324613E-01-7.532775834E-01 9.493431816E-01
+-7.047911204E-01 1.016762822E+00-6.546541578E-01 1.079133497E+00-6.029841077E-01
+ 1.136309145E+00-5.499019727E-01 1.188155870E+00-4.955320618E-01 1.234552257E+00
+-4.400017001E-01 1.275389654E+00-3.834409301E-01 1.310572426E+00-3.259822075E-01
+ 1.340018180E+00-2.677600906E-01 1.363657961E+00-2.089109256E-01 1.381436408E+00
+-1.495725272E-01 1.393311886E+00-8.988385560E-02 1.399256586E+00-2.998469153E-02
+ 1.399256586E+00 2.998469153E-02 1.393311886E+00 8.988385560E-02 1.381436408E+00
+ 1.495725272E-01 1.363657961E+00 2.089109256E-01 1.340018180E+00 2.677600906E-01
+ 1.310572426E+00 3.259822075E-01 1.275389654E+00 3.834409301E-01 1.234552257E+00
+ 4.400017001E-01 1.188155870E+00 4.955320618E-01 1.136309145E+00 5.499019727E-01
+ 1.079133497E+00 6.029841077E-01 1.016762822E+00 6.546541578E-01 9.493431816E-01
+ 7.047911204E-01 8.770324613E-01 7.532775834E-01 8.000000000E-01 8.000000000E-01
+ 5.200000000E-01 0.000000000E+00 5.800000000E-01-6.000000000E-01 5.500000000E-01-9.000000000E-01 1.250000000E+00-9.000000000E-01 1.400000000E+00-6.000000000E-01 1.480000000E+00 0.000000000E+00 1.400000000E+00 6.000000000E-01 1.250000000E+00 9.000000000E-01 5.500000000E-01 9.000000000E-01 5.800000000E-01 6.000000000E-01 5.200000000E-01 0.000000000E+00

miniapps/plasma/ellipse.g_eqdsk

@@ -0,0 +1,211 @@
+  SGRRATEI    01/01/2025    #1  0ms           3 20 40
+ 1.000000000E+00 2.000000000E+00 1.000000000E+00 5.000000000E-01 0.000000000E+00
+ 1.109022415E+00-9.616366848E-06-3.831437870E+00 0.000000000E+00 1.000000000E+00
+ 4.810231190E+05-3.831437870E+00 0.000000000E+00 1.109022415E+00 0.000000000E+00
+-9.616366848E-06 0.000000000E+00 0.000000000E+00 0.000000000E+00 0.000000000E+00
+ 9.695197187E-01 9.726876680E-01 9.756749097E-01 9.784830989E-01 9.811137731E-01
+ 9.835683566E-01 9.858481646E-01 9.879544073E-01 9.898881924E-01 9.916505289E-01
+ 9.932423294E-01 9.946644126E-01 9.959175056E-01 9.970022456E-01 9.979191815E-01
+ 9.986687756E-01 9.992514045E-01 9.996673601E-01 9.999168504E-01 1.000000000E+00
+ 6.634939148E+04 5.954903834E+04 5.311627185E+04 4.705109202E+04 4.135349885E+04
+ 3.602349233E+04 3.106107247E+04 2.646623926E+04 2.223899271E+04 1.837933282E+04
+ 1.488725959E+04 1.176277301E+04 9.005873082E+03 6.616559816E+03 4.594833205E+03
+ 2.940693251E+03 1.654139954E+03 7.351733128E+02 1.837933282E+02 0.000000000E+00
+-6.191881812E-02-5.846888325E-02-5.505154200E-02-5.166451510E-02-4.830561217E-02
+-4.497272375E-02-4.166381400E-02-3.837691381E-02-3.511011443E-02-3.186156146E-02
+-2.862944924E-02-2.541201545E-02-2.220753614E-02-1.901432083E-02-1.583070797E-02
+-1.265506044E-02-9.485761281E-03-6.321209542E-03-3.159816165E-03-0.000000000E+00
+ 1.326987830E+05 1.257146365E+05 1.187304900E+05 1.117463436E+05 1.047621971E+05
+ 9.777805061E+04 9.079390414E+04 8.380975766E+04 7.682561119E+04 6.984146472E+04
+ 6.285731825E+04 5.587317178E+04 4.888902530E+04 4.190487883E+04 3.492073236E+04
+ 2.793658589E+04 2.095243942E+04 1.396829294E+04 6.984146472E+03 0.000000000E+00
+ 6.027880359E-01 5.444336242E-01 4.929843721E-01 4.468606903E-01 4.053015659E-01
+ 3.683453893E-01 3.367530737E-01 3.118362535E-01 2.951519289E-01 2.880365841E-01
+ 2.909818720E-01 3.029033791E-01 3.204224066E-01 3.373590566E-01 3.447032023E-01
+ 3.313561860E-01 2.858795818E-01 1.993086623E-01 6.876738196E-02-9.881690647E-02
+ 6.744595449E-01 5.996140461E-01 5.328634336E-01 4.723627302E-01 4.172570594E-01
+ 3.677070698E-01 3.248508780E-01 2.906574776E-01 2.676201889E-01 2.582462461E-01
+ 2.643256985E-01 2.860143822E-01 3.208428141E-01 3.628585676E-01 4.022050156E-01
+ 4.255007017E-01 4.173644838E-01 3.632813892E-01 2.536864244E-01 8.866344730E-02
+ 7.343619131E-01 6.436962062E-01 5.614219261E-01 4.855364655E-01 4.151880501E-01
+ 3.507641660E-01 2.939113975E-01 2.474323310E-01 2.149930564E-01 2.005771397E-01
+ 2.076460507E-01 2.380192080E-01 2.905729286E-01 3.599724996E-01 4.357778205E-01
+ 5.023656375E-01 5.401375751E-01 5.283698530E-01 4.497523801E-01 2.961440598E-01
+ 7.809349798E-01 6.751996950E-01 5.772531276E-01 4.850226141E-01 3.977360480E-01
+ 3.160900969E-01 2.423462144E-01 1.802896324E-01 1.349679043E-01 1.121217192E-01
+ 1.172407810E-01 1.542315062E-01 2.237783488E-01 3.216158548E-01 4.370886419E-01
+ 5.525253624E-01 6.440313440E-01 6.842364433E-01 6.472434068E-01 5.154640263E-01
+ 8.135690035E-01 6.934582174E-01 5.796763578E-01 4.701407046E-01 3.642092103E-01
+ 2.629430672E-01 1.692957439E-01 8.815231453E-02 2.611713847E-02-9.063055012E-03
+-9.538719707E-03 3.109598996E-02 1.158088422E-01 2.419024963E-01 3.989659604E-01
+ 5.676148854E-01 7.197683473E-01 8.211947470E-01 8.367934603E-01 7.384898454E-01
+ 8.325216321E-01 6.985747478E-01 5.687238375E-01 4.409043702E-01 3.146255305E-01
+ 1.913389993E-01 7.473585748E-02-2.911651568E-02-1.119191647E-01-1.637085739E-01
+-1.739780585E-01-1.334333429E-01-3.635127414E-02 1.166660483E-01 3.159822862E-01
+ 5.409560343E-01 7.596023466E-01 9.308263465E-01 1.009936725E+00 9.575352010E-01
+ 8.387863256E-01 6.913340506E-01 5.450887024E-01 3.979996591E-01 2.497191168E-01
+ 1.020859215E-01-4.045731398E-02-1.706141808E-01-2.782935795E-01-3.511479324E-01
+-3.757593010E-01-3.395926987E-01-2.337188599E-01-5.611233175E-02 1.849391381E-01
+ 4.680854372E-01 7.578657830E-01 1.006514653E+00 1.159586428E+00 1.165738769E+00
+ 8.339278564E-01 6.730837162E-01 5.100416932E-01 3.427308098E-01 1.709112396E-01
+-3.221170684E-03-1.744881056E-01-3.343523364E-01-4.708744696E-01-5.692032028E-01
+-6.128066860E-01-5.856071769E-01-4.750595001E-01-2.759990290E-01 5.221341658E-03
+ 3.471489908E-01 7.113877498E-01 1.043861665E+00 1.280418325E+00 1.357377573E+00
+ 8.199015377E-01 6.455959983E-01 4.653253241E-01 2.769394057E-01 8.024954100E-02
+-1.222439302E-01-3.246688374E-01-5.172678502E-01-6.862382215E-01-8.141557599E-01
+-8.812308347E-01-8.676042109E-01-7.567658125E-01-5.399480748E-01-2.210043304E-01
+ 1.791358869E-01 6.197548661E-01 1.040961106E+00 1.369135452E+00 1.528097841E+00
+ 7.988723691E-01 6.109227609E-01 4.130347848E-01 2.029036972E-01-1.967987208E-02
+-2.519798567E-01-4.874929551E-01-7.152912061E-01-9.197329238E-01-1.080803941E+00
+-1.175377832E+00-1.179650378E+00-1.072877456E+00-8.422994956E-01-4.887614330E-01
+-3.206461911E-02 4.853974608E-01 9.985106689E-01 1.424598500E+00 1.675050854E+00
+ 7.730484943E-01 5.712548386E-01 3.554945030E-01 1.232251820E-01-1.258618115E-01
+-3.888726160E-01-6.587503270E-01-9.234698604E-01-1.165611778E+00-1.362607208E+00
+-1.487983287E+00-1.513912701E+00-1.415245061E+00-1.174939292E+00-7.904115839E-01
+-2.797893042E-01 3.134946132E-01 9.197564239E-01 1.447814952E+00 1.796927636E+00
+ 7.445404658E-01 5.287952464E-01 2.951382949E-01 4.070929804E-02-2.349767287E-01
+-5.289477637E-01-8.336791693E-01-1.136141025E+00-1.417230568E+00-1.651910893E+00
+-1.810425157E+00-1.860937968E+00-1.773831419E+00-1.527614344E+00-1.115971166E+00
+-5.549039664E-01 1.117031538E-01 8.102732738E-01 1.441789269E+00 1.893892967E+00
+ 7.152541434E-01 4.856531808E-01 2.343995042E-01-4.175359515E-02-3.435345212E-01
+-6.679672634E-01-1.007147349E+00-1.347145717E+00-1.667299880E+00-1.940240855E+00
+-2.133063162E+00-2.210036310E+00-2.137134706E+00-1.888385838E+00-1.453579841E+00
+-8.462660998E-01-1.102730351E-01 6.775958354E-01 1.411245424E+00 1.967425855E+00
+ 6.868209862E-01 4.437629001E-01 1.756156706E-01-1.213017723E-01-4.480113960E-01
+-8.015956403E-01-1.173854475E+00-1.550073727E+00-1.908179626E+00-2.218653210E+00
+-2.445647332E+00-2.549748880E+00-2.492710924E+00-2.244196046E+00-1.790094098E+00
+-1.141319885E+00-3.412468720E-01 5.307237428E-01 1.362241297E+00 2.020079621E+00
+ 6.605654332E-01 4.048286553E-01 1.209502842E-01-1.952062579E-01-5.449842404E-01
+-9.255704167E-01-1.328545300E+00-1.738527527E+00-2.132201008E+00-2.478121003E+00
+-2.737773949E+00-2.868374135E+00-2.827766422E+00-2.581516128E+00-2.111771066E+00
+-1.426787661E+00-5.692100125E-01 3.795334026E-01 1.301701384E+00 2.055179962E+00
+ 6.375053763E-01 3.702941358E-01 7.233227099E-02-2.609627702E-01-6.312576177E-01
+-1.035869585E+00-1.466224516E+00-1.906392149E+00-2.331999433E+00-2.709937370E+00
+-2.999364174E+00-3.154523523E+00-3.129786863E+00-2.887037341E+00-2.405002017E+00
+-1.689415837E+00-7.820514975E-01 2.341342187E-01 1.236899458E+00 2.076483242E+00
+ 6.183787457E-01 3.413326511E-01 3.141219456E-02-3.163699380E-01-7.039792622E-01
+-1.128869268E+00-1.582363062E+00-2.048097697E+00-2.500841167E+00-2.906113655E+00
+-3.221139041E+00-3.397675320E+00-3.387166510E+00-3.148365134E+00-2.657051631E+00
+-1.916729591E+00-9.682880687E-01 1.042108382E-01 1.174925970E+00 2.087819678E+00
+ 6.036871942E-01 3.188526756E-01-4.670925182E-04-3.595932281E-01-7.607407379E-01
+-1.201485317E+00-1.673086529E+00-2.158861549E+00-2.632926762E+00-3.059751018E+00
+-3.395064527E+00-3.588695197E+00-3.589801388E+00-3.354675138E+00-2.856758537E+00
+-2.097750017E+00-1.117760284E+00-1.606164268E-03 1.122175768E+00 2.092746840E+00
+ 5.937469493E-01 3.035125582E-01-2.228838311E-02-3.892147630E-01-7.996606775E-01
+-1.251293437E+00-1.735337138E+00-2.234898256E+00-2.723655399E+00-3.165365399E+00
+-3.514741855E+00-3.720293832E+00-3.729611788E+00-3.497292139E+00-2.995154781E+00
+-2.223629545E+00-1.222250664E+00-7.630060271E-02 1.083890305E+00 2.094237915E+00
+ 5.887371279E-01 2.957381914E-01-3.336989801E-02-4.042696576E-01-8.194487014E-01
+-1.276623310E+00-1.767002017E+00-2.273586775E+00-2.769836332E+00-3.219148153E+00
+-3.575721206E+00-3.787395338E+00-3.800963568E+00-3.570157560E+00-3.065966683E+00
+-2.288166411E+00-1.275984954E+00-1.149259009E-01 1.063784750E+00 2.094426631E+00
+ 5.887371279E-01 2.957381914E-01-3.336989801E-02-4.042696576E-01-8.194487014E-01
+-1.276623310E+00-1.767002017E+00-2.273586775E+00-2.769836332E+00-3.219148153E+00
+-3.575721206E+00-3.787395338E+00-3.800963568E+00-3.570157560E+00-3.065966683E+00
+-2.288166411E+00-1.275984954E+00-1.149259009E-01 1.063784750E+00 2.094426631E+00
+ 5.937469493E-01 3.035125582E-01-2.228838311E-02-3.892147630E-01-7.996606775E-01
+-1.251293437E+00-1.735337138E+00-2.234898256E+00-2.723655399E+00-3.165365399E+00
+-3.514741855E+00-3.720293832E+00-3.729611788E+00-3.497292139E+00-2.995154781E+00
+-2.223629545E+00-1.222250664E+00-7.630060271E-02 1.083890305E+00 2.094237915E+00
+ 6.036871942E-01 3.188526756E-01-4.670925182E-04-3.595932281E-01-7.607407379E-01
+-1.201485317E+00-1.673086529E+00-2.158861549E+00-2.632926762E+00-3.059751018E+00
+-3.395064527E+00-3.588695197E+00-3.589801388E+00-3.354675138E+00-2.856758537E+00
+-2.097750017E+00-1.117760284E+00-1.606164268E-03 1.122175768E+00 2.092746840E+00
+ 6.183787457E-01 3.413326511E-01 3.141219456E-02-3.163699380E-01-7.039792622E-01
+-1.128869268E+00-1.582363062E+00-2.048097697E+00-2.500841167E+00-2.906113655E+00
+-3.221139041E+00-3.397675320E+00-3.387166510E+00-3.148365134E+00-2.657051631E+00
+-1.916729591E+00-9.682880687E-01 1.042108382E-01 1.174925970E+00 2.087819678E+00
+ 6.375053763E-01 3.702941358E-01 7.233227099E-02-2.609627702E-01-6.312576177E-01
+-1.035869585E+00-1.466224516E+00-1.906392149E+00-2.331999433E+00-2.709937370E+00
+-2.999364174E+00-3.154523523E+00-3.129786863E+00-2.887037341E+00-2.405002017E+00
+-1.689415837E+00-7.820514975E-01 2.341342187E-01 1.236899458E+00 2.076483242E+00
+ 6.605654332E-01 4.048286553E-01 1.209502842E-01-1.952062579E-01-5.449842404E-01
+-9.255704167E-01-1.328545300E+00-1.738527527E+00-2.132201008E+00-2.478121003E+00
+-2.737773949E+00-2.868374135E+00-2.827766422E+00-2.581516128E+00-2.111771066E+00
+-1.426787661E+00-5.692100125E-01 3.795334026E-01 1.301701384E+00 2.055179962E+00
+ 6.868209862E-01 4.437629001E-01 1.756156706E-01-1.213017723E-01-4.480113960E-01
+-8.015956403E-01-1.173854475E+00-1.550073727E+00-1.908179626E+00-2.218653210E+00
+-2.445647332E+00-2.549748880E+00-2.492710924E+00-2.244196046E+00-1.790094098E+00
+-1.141319885E+00-3.412468720E-01 5.307237428E-01 1.362241297E+00 2.020079621E+00
+ 7.152541434E-01 4.856531808E-01 2.343995042E-01-4.175359515E-02-3.435345212E-01
+-6.679672634E-01-1.007147349E+00-1.347145717E+00-1.667299880E+00-1.940240855E+00
+-2.133063162E+00-2.210036310E+00-2.137134706E+00-1.888385838E+00-1.453579841E+00
+-8.462660998E-01-1.102730351E-01 6.775958354E-01 1.411245424E+00 1.967425855E+00
+ 7.445404658E-01 5.287952464E-01 2.951382949E-01 4.070929804E-02-2.349767287E-01
+-5.289477637E-01-8.336791693E-01-1.136141025E+00-1.417230568E+00-1.651910893E+00
+-1.810425157E+00-1.860937968E+00-1.773831419E+00-1.527614344E+00-1.115971166E+00
+-5.549039664E-01 1.117031538E-01 8.102732738E-01 1.441789269E+00 1.893892967E+00
+ 7.730484943E-01 5.712548386E-01 3.554945030E-01 1.232251820E-01-1.258618115E-01
+-3.888726160E-01-6.587503270E-01-9.234698604E-01-1.165611778E+00-1.362607208E+00
+-1.487983287E+00-1.513912701E+00-1.415245061E+00-1.174939292E+00-7.904115839E-01
+-2.797893042E-01 3.134946132E-01 9.197564239E-01 1.447814952E+00 1.796927636E+00
+ 7.988723691E-01 6.109227609E-01 4.130347848E-01 2.029036972E-01-1.967987208E-02
+-2.519798567E-01-4.874929551E-01-7.152912061E-01-9.197329238E-01-1.080803941E+00
+-1.175377832E+00-1.179650378E+00-1.072877456E+00-8.422994956E-01-4.887614330E-01
+-3.206461911E-02 4.853974608E-01 9.985106689E-01 1.424598500E+00 1.675050854E+00
+ 8.199015377E-01 6.455959983E-01 4.653253241E-01 2.769394057E-01 8.024954100E-02
+-1.222439302E-01-3.246688374E-01-5.172678502E-01-6.862382215E-01-8.141557599E-01
+-8.812308347E-01-8.676042109E-01-7.567658125E-01-5.399480748E-01-2.210043304E-01
+ 1.791358869E-01 6.197548661E-01 1.040961106E+00 1.369135452E+00 1.528097841E+00
+ 8.339278564E-01 6.730837162E-01 5.100416932E-01 3.427308098E-01 1.709112396E-01
+-3.221170684E-03-1.744881056E-01-3.343523364E-01-4.708744696E-01-5.692032028E-01
+-6.128066860E-01-5.856071769E-01-4.750595001E-01-2.759990290E-01 5.221341658E-03
+ 3.471489908E-01 7.113877498E-01 1.043861665E+00 1.280418325E+00 1.357377573E+00
+ 8.387863256E-01 6.913340506E-01 5.450887024E-01 3.979996591E-01 2.497191168E-01
+ 1.020859215E-01-4.045731398E-02-1.706141808E-01-2.782935795E-01-3.511479324E-01
+-3.757593010E-01-3.395926987E-01-2.337188599E-01-5.611233175E-02 1.849391381E-01
+ 4.680854372E-01 7.578657830E-01 1.006514653E+00 1.159586428E+00 1.165738769E+00
+ 8.325216321E-01 6.985747478E-01 5.687238375E-01 4.409043702E-01 3.146255305E-01
+ 1.913389993E-01 7.473585748E-02-2.911651568E-02-1.119191647E-01-1.637085739E-01
+-1.739780585E-01-1.334333429E-01-3.635127414E-02 1.166660483E-01 3.159822862E-01
+ 5.409560343E-01 7.596023466E-01 9.308263465E-01 1.009936725E+00 9.575352010E-01
+ 8.135690035E-01 6.934582174E-01 5.796763578E-01 4.701407046E-01 3.642092103E-01
+ 2.629430672E-01 1.692957439E-01 8.815231453E-02 2.611713847E-02-9.063055012E-03
+-9.538719707E-03 3.109598996E-02 1.158088422E-01 2.419024963E-01 3.989659604E-01
+ 5.676148854E-01 7.197683473E-01 8.211947470E-01 8.367934603E-01 7.384898454E-01
+ 7.809349798E-01 6.751996950E-01 5.772531276E-01 4.850226141E-01 3.977360480E-01
+ 3.160900969E-01 2.423462144E-01 1.802896324E-01 1.349679043E-01 1.121217192E-01
+ 1.172407810E-01 1.542315062E-01 2.237783488E-01 3.216158548E-01 4.370886419E-01
+ 5.525253624E-01 6.440313440E-01 6.842364433E-01 6.472434068E-01 5.154640263E-01
+ 7.343619131E-01 6.436962062E-01 5.614219261E-01 4.855364655E-01 4.151880501E-01
+ 3.507641660E-01 2.939113975E-01 2.474323310E-01 2.149930564E-01 2.005771397E-01
+ 2.076460507E-01 2.380192080E-01 2.905729286E-01 3.599724996E-01 4.357778205E-01
+ 5.023656375E-01 5.401375751E-01 5.283698530E-01 4.497523801E-01 2.961440598E-01
+ 6.744595449E-01 5.996140461E-01 5.328634336E-01 4.723627302E-01 4.172570594E-01
+ 3.677070698E-01 3.248508780E-01 2.906574776E-01 2.676201889E-01 2.582462461E-01
+ 2.643256985E-01 2.860143822E-01 3.208428141E-01 3.628585676E-01 4.022050156E-01
+ 4.255007017E-01 4.173644838E-01 3.632813892E-01 2.536864244E-01 8.866344730E-02
+ 6.027880359E-01 5.444336242E-01 4.929843721E-01 4.468606903E-01 4.053015659E-01
+ 3.683453893E-01 3.367530737E-01 3.118362535E-01 2.951519289E-01 2.880365841E-01
+ 2.909818720E-01 3.029033791E-01 3.204224066E-01 3.373590566E-01 3.447032023E-01
+ 3.313561860E-01 2.858795818E-01 1.993086623E-01 6.876738196E-02-9.881690647E-02
+ 1.000000000E+00 1.050035307E+00 1.107285429E+00 1.171072127E+00 1.241305586E+00
+ 1.318263580E+00 1.405184527E+00 1.505114754E+00 1.616506900E+00 1.730427615E+00
+ 1.891151823E+00 2.065212169E+00 2.268460038E+00 2.512075631E+00 2.811055433E+00
+ 3.186196537E+00 3.671181078E+00 4.326097962E+00 5.264420087E+00 6.784828742E+00
+   60   11
+ 1.400000000E+00 0.000000000E+00 1.397733920E+00 8.503478852E-02 1.390961356E+00
+ 1.691060989E-01 1.379759043E+00 2.512613696E-01 1.364253909E+00 3.305697486E-01
+ 1.344621633E+00 4.061326403E-01 1.321084655E+00 4.770938868E-01 1.293909660E+00
+ 5.426494690E-01 1.263404552E+00 6.020566159E-01 1.229914964E+00 6.546422207E-01
+ 1.193820348E+00 6.998104676E-01 1.155529670E+00 7.370495823E-01 1.115476779E+00
+ 7.659376307E-01 1.074115490E+00 7.861472997E-01 1.031914442E+00 7.974496056E-01
+ 9.893517914E-01 7.997164889E-01 9.469097891E-01 7.929222648E-01 9.050693201E-01
+ 7.771439146E-01 8.643044536E-01 7.525602133E-01 8.250770718E-01 7.194497041E-01
+ 7.878316368E-01 6.781875423E-01 7.529901540E-01 6.292412446E-01 7.209473915E-01
+ 5.731653922E-01 6.920664066E-01 5.105953469E-01 6.666744324E-01 4.422400523E-01
+ 6.450591700E-01 3.688740012E-01 6.274655290E-01 2.913284602E-01 6.140928525E-01
+ 2.104820514E-01 6.050926581E-01 1.272507968E-01 6.005669217E-01 4.257773987E-02
+ 6.005669217E-01-4.257773987E-02 6.050926581E-01-1.272507968E-01 6.140928525E-01
+-2.104820514E-01 6.274655290E-01-2.913284602E-01 6.450591700E-01-3.688740012E-01
+ 6.666744324E-01-4.422400523E-01 6.920664066E-01-5.105953469E-01 7.209473915E-01
+-5.731653922E-01 7.529901540E-01-6.292412446E-01 7.878316368E-01-6.781875423E-01
+ 8.250770718E-01-7.194497041E-01 8.643044536E-01-7.525602133E-01 9.050693201E-01
+-7.771439146E-01 9.469097891E-01-7.929222648E-01 9.893517914E-01-7.997164889E-01
+ 1.031914442E+00-7.974496056E-01 1.074115490E+00-7.861472997E-01 1.115476779E+00
+-7.659376307E-01 1.155529670E+00-7.370495823E-01 1.193820348E+00-6.998104676E-01
+ 1.229914964E+00-6.546422207E-01 1.263404552E+00-6.020566159E-01 1.293909660E+00
+-5.426494690E-01 1.321084655E+00-4.770938868E-01 1.344621633E+00-4.061326403E-01
+ 1.364253909E+00-3.305697486E-01 1.379759043E+00-2.512613696E-01 1.390961356E+00
+-1.691060989E-01 1.397733920E+00-8.503478852E-02 1.400000000E+00-1.959434879E-16
+ 5.200000000E-01 0.000000000E+00 5.800000000E-01-6.000000000E-01 5.500000000E-01-9.000000000E-01 1.250000000E+00-9.000000000E-01 1.400000000E+00-6.000000000E-01 1.480000000E+00 0.000000000E+00 1.400000000E+00 6.000000000E-01 1.250000000E+00 9.000000000E-01 5.500000000E-01 9.000000000E-01 5.800000000E-01 6.000000000E-01 5.200000000E-01 0.000000000E+00

miniapps/plasma/g_eqdsk_data.cpp

@@ -0,0 +1,826 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include <limits>
+#include "plasma.hpp"
+#include "g_eqdsk_data.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+namespace plasma
+{
+
+G_EQDSK_Data::G_EQDSK_Data(istream &is, int logging)
+   : logging_(logging), init_flag_(0)
+{
+   /// The following file format is taken from the C-Mod Wiki at
+   /// https://cmodwiki.psfc.mit.edu/index.php/G_EQDSK
+   real_t XDUM = 0.0;
+
+   const int buflen = 1024;
+   char buf[buflen];
+   is.getline(buf, buflen);
+   istringstream iss(buf);
+   string word;
+   iss >> std::ws;
+   while (!iss.eof())
+   {
+      iss >> word;
+      CASE_.push_back(word);
+      iss >> std::ws;
+   }
+
+   NW_ = to_int(CASE_[CASE_.size()-2]);
+   NH_ = to_int(CASE_[CASE_.size()-1]);
+
+   is >> RDIM_ >> ZDIM_ >> RCENTR_ >> RLEFT_ >> ZMID_;
+   is >> RMAXIS_ >> ZMAXIS_ >> SIMAG_ >> SIBRY_ >> BCENTR_;
+   is >> CURRENT_ >> SIMAG_ >> XDUM >> RMAXIS_ >> XDUM;
+   is >> ZMAXIS_ >> XDUM >> SIBRY_ >> XDUM >> XDUM;
+
+   FPOL_.resize(NW_);
+   PRES_.resize(NW_);
+   FFPRIM_.resize(NW_);
+   PPRIME_.resize(NW_);
+   PSIRZ_.resize(NW_ * NH_);
+   QPSI_.resize(NW_);
+
+   for (int i=0; i<NW_; i++) { is >> FPOL_[i]; }
+   for (int i=0; i<NW_; i++) { is >> PRES_[i]; }
+   for (int i=0; i<NW_; i++) { is >> FFPRIM_[i]; }
+   for (int i=0; i<NW_; i++) { is >> PPRIME_[i]; }
+   for (int j=0; j<NH_; j++)
+   {
+      for (int i=0; i<NW_; i++)
+      {
+         is >> PSIRZ_[NH_ * i + j];
+      }
+   }
+   for (int i=0; i<NW_; i++) { is >> QPSI_[i]; }
+
+   is >> NBBBS_ >> LIMITR_;
+
+   RBBBS_.resize(NBBBS_);
+   ZBBBS_.resize(NBBBS_);
+   RLIM_.resize(LIMITR_);
+   ZLIM_.resize(LIMITR_);
+
+   for (int i=0; i<NBBBS_; i++) { is >> RBBBS_[i] >> ZBBBS_[i]; }
+   for (int i=0; i<LIMITR_; i++) { is >> RLIM_[i] >> ZLIM_[i]; }
+
+   if (logging_ > 0) { checkPsiBoundary(); }
+
+   dr_ = RDIM_ / (NW_ - 1);
+   dz_ = ZDIM_ / (NH_ - 1);
+
+   dpsi_ = (SIBRY_ - SIMAG_) / (NW_ - 1);
+}
+
+void G_EQDSK_Data::PrintInfo(ostream & out) const
+{
+   out << endl << "G EQDSK File Info:" << endl;
+   out << "Size of grid: " << NW_ << " x " << NH_ << endl;
+   out << "Number of boundary points: " << NBBBS_ << endl;
+   out << "Number of limiter points:  " << LIMITR_ << endl;
+   out << endl;
+   out << "Range of R: " << RLEFT_ << " -> " << RLEFT_ + RDIM_ << endl;
+   out << "Range of Z: " << ZMID_ - 0.5 * ZDIM_
+       << " -> " << ZMID_ + 0.5 * ZDIM_ << endl;
+   out << "Location of magnetic axis: "
+       << "(" << RMAXIS_ << "," << ZMAXIS_ << ")" << endl;
+   out << "Poloidal flux at magnetic axis:   " << SIMAG_ << endl;
+   out << "Poloidal flux at plasma boundary: " << SIBRY_ << endl;
+   out << "R in meter of vacuum toroidal magnetic field BCENTR: "
+       << RCENTR_ << endl;
+   out << "Vacuum toroidal magnetic field in Tesla at RCENTR: "
+       << BCENTR_ << endl;
+   out << "Plasma current in Ampere: " << CURRENT_ << endl << endl;
+}
+
+void G_EQDSK_Data::DumpGnuPlotData(const string &file) const
+{
+   real_t  fmin =  std::numeric_limits<real_t>::max();
+   real_t  fmax = -std::numeric_limits<real_t>::max();
+   real_t  pmin =  std::numeric_limits<real_t>::max();
+   real_t  pmax = -std::numeric_limits<real_t>::max();
+   real_t ffmin =  std::numeric_limits<real_t>::max();
+   real_t ffmax = -std::numeric_limits<real_t>::max();
+   real_t ppmin =  std::numeric_limits<real_t>::max();
+   real_t ppmax = -std::numeric_limits<real_t>::max();
+   real_t  qmin =  std::numeric_limits<real_t>::max();
+   real_t  qmax = -std::numeric_limits<real_t>::max();
+
+   ostringstream oss_dat, oss_inp;
+   oss_inp << file << ".inp";
+   oss_dat << file << ".dat";
+   ofstream ofs_inp(oss_inp.str().c_str());
+   ofstream ofs_dat(oss_dat.str().c_str());
+
+   for (int i=0; i<NW_; i++)
+   {
+      ofs_dat << real_t(i) / (NW_ - 1)
+              << '\t' << FPOL_[i]
+              << '\t' << PRES_[i]
+              << '\t' << FFPRIM_[i]
+              << '\t' << PPRIME_[i]
+              << '\t' << QPSI_[i]
+              << '\n';
+      fmin = min(FPOL_[i], fmin);
+      fmax = max(FPOL_[i], fmax);
+      pmin = min(PRES_[i], pmin);
+      pmax = max(PRES_[i], pmax);
+      ffmin = min(FFPRIM_[i], ffmin);
+      ffmax = max(FFPRIM_[i], ffmax);
+      ppmin = min(PPRIME_[i], ppmin);
+      ppmax = max(PPRIME_[i], ppmax);
+      qmin = min(QPSI_[i], qmin);
+      qmax = max(QPSI_[i], qmax);
+   }
+
+   ofs_dat << "\n\n";
+   for (int i=0; i<NW_; i++)
+   {
+      for (int j=0; j<NH_; j++)
+      {
+         ofs_dat << RLEFT_ + RDIM_ * i / (NW_ - 1)
+                 << '\t' << ZMID_ - 0.5 * ZDIM_ + ZDIM_ * j / (NH_ - 1)
+                 << '\t' << PSIRZ_[NH_ * i + j]
+                 << '\n';
+      }
+      ofs_dat << '\n';
+   }
+   ofs_dat << "\n\n";
+   for (int i=0; i<NBBBS_; i++)
+   {
+      ofs_dat << RBBBS_[i] << '\t' << ZBBBS_[i] << '\n';
+   }
+   ofs_dat << "\n\n";
+   for (int i=0; i<LIMITR_; i++)
+   {
+      ofs_dat << RLIM_[i] << '\t' << ZLIM_[i] << '\n';
+   }
+   ofs_dat.close();
+
+   ofs_inp << "set xrange [0:1];\n";
+   ofs_inp << "set yrange [" << fmin << ":" << fmax << "];\n";
+   ofs_inp << "plot '" << oss_dat.str()
+           << "' index 0 using 1:2 w l t 'FPOL';\n";
+   ofs_inp << "pause -1;\n";
+   ofs_inp << "set yrange [" << pmin << ":" << pmax << "];\n";
+   ofs_inp << "plot '" << oss_dat.str()
+           << "' index 0 using 1:3 w l t 'PRES';\n";
+   ofs_inp << "pause -1;\n";
+   ofs_inp << "set yrange [" << ffmin << ":" << ffmax << "];\n";
+   ofs_inp << "plot '" << oss_dat.str()
+           << "' index 0 using 1:4 w l t 'FFPRIME';\n";
+   ofs_inp << "pause -1;\n";
+   ofs_inp << "set yrange [" << ppmin << ":" << ppmax << "];\n";
+   ofs_inp << "plot '" << oss_dat.str()
+           << "' index 0 using 1:5 w l t 'PPRIME';\n";
+   ofs_inp << "pause -1;\n";
+   ofs_inp << "set yrange [" << qmin << ":" << qmax << "];\n";
+   ofs_inp << "plot '" << oss_dat.str()
+           << "' index 0 using 1:6 w l t 'QPSI';\n";
+   ofs_inp << "pause -1;\n";
+
+   ofs_inp << "unset xrange\n";
+   ofs_inp << "unset yrange\n";
+   ofs_inp << "set view map;\n";
+   ofs_inp << "unset surface;\n";
+   ofs_inp << "set contour base;\n";
+   ofs_inp << "set cntrparam levels 20;\n";
+   ofs_inp << "set size ratio -1;\n";
+   ofs_inp << "set nokey;\n";
+   ofs_inp << "splot '" << oss_dat.str()
+           << "' index 1 with lines pal t 'PSIRZ';\n";
+   ofs_inp << "set key;\n";
+   ofs_inp << "pause -1;\n";
+   ofs_inp << "set size ratio -1;\n";
+   ofs_inp << "plot '" << oss_dat.str()
+           << "' index 2 using 1:2 w l t 'BOUNDARY',";
+   ofs_inp << " '" << oss_dat.str()
+           << "' index 3 using 1:2 w l t 'LIMITER';\n";
+
+   ofs_inp.close();
+}
+
+void G_EQDSK_Data::checkPsiBoundary()
+{
+   real_t psi_avg = 0.0;
+   real_t psi_dif = 0.0;
+   real_t psi_min = std::numeric_limits<real_t>::max();
+   real_t psi_max = std::numeric_limits<real_t>::min();
+
+   Vector rz(2);
+   real_t psi = 0.0;
+   for (int i=0; i<NBBBS_; i++)
+   {
+      rz[0] = RBBBS_[i];
+      rz[1] = ZBBBS_[i];
+      psi = this->InterpPsiRZ(rz);
+
+      psi_min = std::min(psi, psi_min);
+      psi_max = std::max(psi, psi_max);
+
+      psi_avg += psi;
+      psi_dif += abs(psi - SIBRY_);
+   }
+   psi_avg /= NBBBS_;
+   psi_dif /= NBBBS_;
+
+   if (logging_ > 1)
+   {
+      mfem::out << psi_min << " <= (Psi on plasma boundary) <= "
+                << psi_max << endl;
+      mfem::out << "Average of Psi on plasma boundary: " << psi_avg << endl;
+      mfem::out << "Average of |Psi - SIBRY| on plasma boundary: "
+                << psi_dif << endl;
+   }
+
+   MFEM_VERIFY(psi_dif < 1e-2 * abs(SIMAG_), "Psi differs from its imposed "
+               "boundary value more than expected.");
+}
+
+real_t G_EQDSK_Data::InterpFPolRZ(const Vector &rz)
+{
+   real_t psi = InterpPsiRZ(rz);
+
+   if (!checkFlag(FPOL))
+   {
+      initInterpPsi(FPOL_, FPOL_t_);
+      setFlag(FPOL);
+   }
+
+   return interpPsi(psi, FPOL_, FPOL_t_);
+}
+
+real_t G_EQDSK_Data::InterpPresRZ(const Vector &rz)
+{
+   real_t psi = InterpPsiRZ(rz);
+
+   if (!checkFlag(PRES))
+   {
+      initInterpPsi(PRES_, PRES_t_);
+      setFlag(PRES);
+   }
+
+   return interpPsi(psi, PRES_, PRES_t_);
+}
+
+real_t G_EQDSK_Data::InterpFFPrimeRZ(const Vector &rz)
+{
+   real_t psi = InterpPsiRZ(rz);
+
+   if (!checkFlag(FFPRIM))
+   {
+      initInterpPsi(FFPRIM_, FFPRIM_t_);
+      setFlag(FFPRIM);
+   }
+   return interpPsi(psi, FFPRIM_, FFPRIM_t_);
+}
+
+real_t G_EQDSK_Data::InterpPPrimeRZ(const Vector &rz)
+{
+   real_t psi = InterpPsiRZ(rz);
+
+   if (!checkFlag(PPRIME))
+   {
+      initInterpPsi(PPRIME_, PPRIME_t_);
+      setFlag(PPRIME);
+   }
+   return interpPsi(psi, PPRIME_, PPRIME_t_);
+}
+
+real_t G_EQDSK_Data::InterpPsiRZ(const Vector &rz)
+{
+   if (!checkFlag(PSIRZ))
+   {
+      initInterpRZ(PSIRZ_, PSIRZ_c_, PSIRZ_d_, PSIRZ_e_);
+      setFlag(PSIRZ);
+   }
+   return interpRZ(rz, PSIRZ_, PSIRZ_c_, PSIRZ_d_, PSIRZ_e_);
+}
+
+real_t G_EQDSK_Data::InterpQRZ(const Vector &rz)
+{
+   real_t psi = InterpPsiRZ(rz);
+
+   if (!checkFlag(QPSI))
+   {
+      initInterpPsi(QPSI_, QPSI_t_);
+      setFlag(QPSI);
+   }
+
+   return interpPsi(psi, QPSI_, QPSI_t_);
+}
+
+void G_EQDSK_Data::InterpNxGradPsiRZ(const Vector &rz, Vector &nxdp)
+{
+   if (!checkFlag(PSIRZ))
+   {
+      initInterpRZ(PSIRZ_, PSIRZ_c_, PSIRZ_d_, PSIRZ_e_);
+      setFlag(PSIRZ);
+   }
+   interpNxGradRZ(rz, PSIRZ_, PSIRZ_c_, PSIRZ_d_, PSIRZ_e_, nxdp);
+}
+
+void G_EQDSK_Data::InterpBPolRZ(const Vector &rz, Vector &bpol)
+{
+   InterpNxGradPsiRZ(rz, bpol);
+   if (rz[0] > 1e-6 * RDIM_) { bpol /= rz[0]; }
+}
+
+real_t G_EQDSK_Data::InterpBTorRZ(const Vector &rz)
+{
+   if (rz[0] > 1e-6 * RDIM_)
+   {
+      return InterpFPolRZ(rz) / rz[0];
+   }
+   else
+   {
+      return 0.0;
+   }
+}
+
+real_t G_EQDSK_Data::InterpJTorRZ(const Vector &rz)
+{
+   if (rz[0] > 1e-6 * RDIM_)
+   {
+      return InterpPPrimeRZ(rz) * rz[0] +
+             InterpFFPrimeRZ(rz) / rz[0] / mu0_;
+   }
+   else
+   {
+      return 0.0;
+   }
+}
+
+void G_EQDSK_Data::initInterpRZ(const std::vector<real_t> &v,
+                                ShiftedDenseMatrix &c,
+                                ShiftedDenseMatrix &d,
+                                ShiftedDenseMatrix &e)
+{
+   ExtendedDenseMatrix ve(&v[0], NW_, NH_);
+
+   c.SetSize(NW_ + 3, NH_ + 2); c.SetShifts(2, 1); c = 0.0;
+   d.SetSize(NW_ + 2, NH_ + 3); d.SetShifts(1, 2); d = 0.0;
+   e.SetSize(NW_ + 1, NH_ + 1); e.SetShifts(1, 1); e = 0.0;
+
+   // x-directed divided differences
+   for (int i=-1; i<NW_; i++)
+   {
+      c(i,-1) = (ve(i+1,-1) - ve(i,-1)) / dr_;
+   }
+   for (int j=0; j<NH_; j++)
+   {
+      for (int i=-2; i<=NW_; i++)
+      {
+         c(i,j) = (ve(i+1,j) - ve(i,j)) / dr_;
+      }
+   }
+   for (int i=-1; i<NW_; i++)
+   {
+      c(i,NH_) = (ve(i+1,NH_) - ve(i,NH_)) / dr_;
+   }
+
+   // y-directed divided differences
+   for (int j=-1; j<NH_; j++)
+   {
+      d(-1,j) = (ve(-1,j+1) - ve(-1,j)) / dz_;
+   }
+   for (int i=0; i<NW_; i++)
+   {
+      for (int j=-2; j<=NH_; j++)
+      {
+         d(i,j) = (ve(i,j+1) - ve(i,j)) / dz_;
+      }
+   }
+   for (int j=-1; j<NH_; j++)
+   {
+      d(NW_,j) = (ve(NW_,j+1) - ve(NW_,j)) / dz_;
+   }
+
+   // Second order divided differences
+   for (int i=-1; i<NW_; i++)
+   {
+      for (int j=-1; j<NH_; j++)
+      {
+         e(i,j) = (c(i,j+1) - c(i,j)) / dz_;
+      }
+   }
+}
+
+real_t G_EQDSK_Data::interpRZ(const Vector &rz,
+                              const std::vector<real_t> &v,
+                              const ShiftedDenseMatrix &c,
+                              const ShiftedDenseMatrix &d,
+                              const ShiftedDenseMatrix &e)
+{
+   real_t r = rz[0];
+   real_t z = rz[1];
+
+   real_t rs = (r - RLEFT_) / RDIM_;
+   real_t zs = (z - ZMID_ + 0.5 * ZDIM_) / ZDIM_;
+
+   int i = std::max(0, std::min((int)floor(real_t(NW_-1) * rs), NW_-2));
+   int j = std::max(0, std::min((int)floor(real_t(NH_-1) * zs), NH_-2));
+
+   // Compute corners of local patch
+   real_t r0 = RLEFT_ + RDIM_ * i / (NW_ - 1);
+   real_t r1 = r0 + RDIM_ / (NW_ - 1);
+   real_t z0 = ZMID_ - 0.5 * ZDIM_ + ZDIM_ * j / (NH_ - 1);
+   real_t z1 = z0 + ZDIM_ / (NH_ - 1);
+
+   // Prepare position dependent factors
+   real_t wra = (r1 - r) / dr_;
+   real_t wrb = (r - r0) / dr_;
+   real_t wrc = (1.0 + 2.0 * wra);
+   real_t wrd = (1.0 + 2.0 * wrb);
+   real_t wra2 = wra * wra;
+   real_t wrb2 = wrb * wrb;
+
+   real_t wza = (z1 - z) / dz_;
+   real_t wzb = (z - z0) / dz_;
+   real_t wzc = (1.0 + 2.0 * wza);
+   real_t wzd = (1.0 + 2.0 * wzb);
+   real_t wza2 = wza * wza;
+   real_t wzb2 = wzb * wzb;
+
+   // Extract variable values at corners of local patch
+   real_t p00 = v[NH_ * i + j];
+   real_t p10 = v[NH_ * (i + 1) + j];
+   real_t p01 = v[NH_ * i + j + 1];
+   real_t p11 = v[NH_ * (i + 1) + j + 1];
+
+   real_t var = p00 * wra2 * wrd * wza2 * wzd
+                + p10 * wrb2 * wrc * wza2 * wzd
+                + p01 * wra2 * wrd * wzb2 * wzc
+                + p11 * wrb2 * wrc * wzb2 * wzc;
+
+   // Compute dvar/dx at corners of local patch
+   real_t wx00a = fabs(c(i-1,j) - c(i-2,j));
+   real_t wx00b = fabs(c(i+1,j) - c(i,j));
+
+   real_t wx10a = fabs(c(i,j) - c(i-1,j));
+   real_t wx10b = fabs(c(i+2,j) - c(i+1,j));
+
+   real_t wx01a = fabs(c(i-1,j+1) - c(i-2,j+1));
+   real_t wx01b = fabs(c(i+1,j+1) - c(i,j+1));
+
+   real_t wx11a = fabs(c(i,j+1) - c(i-1,j+1));
+   real_t wx11b = fabs(c(i+2,j+1) - c(i+1,j+1));
+
+   if (wx00a == 0.0 && wx00b == 0.0) { wx00a = 1.0; wx00b = 1.0; }
+   if (wx10a == 0.0 && wx10b == 0.0) { wx10a = 1.0; wx10b = 1.0; }
+   if (wx01a == 0.0 && wx01b == 0.0) { wx01a = 1.0; wx01b = 1.0; }
+   if (wx11a == 0.0 && wx11b == 0.0) { wx11a = 1.0; wx11b = 1.0; }
+
+   real_t px00 = (wx00b * c(i-1,j) + wx00a * c(i,j)) / (wx00b + wx00a);
+   real_t px10 = (wx10b * c(i,j) + wx10a * c(i+1,j)) / (wx10b + wx10a);
+   real_t px01 = (wx01b * c(i-1,j+1) + wx01a * c(i,j+1)) / (wx01b + wx01a);
+   real_t px11 = (wx11b * c(i,j+1) + wx11a * c(i+1,j+1)) / (wx11b + wx11a);
+
+   real_t varx = px00 * wra2 * wrb * wza2 * wzd
+                 - px10 * wrb2 * wra * wza2 * wzd
+                 + px01 * wrb * wra2 * wzb2 * wzc
+                 - px11 * wra * wrb2 * wzb2 * wzc;
+   var += varx * dr_;
+
+   // Compute dvar/dy at corners of local patch
+   real_t wy00a = fabs(d(i,j-1) - d(i,j-2));
+   real_t wy00b = fabs(d(i,j+1) - d(i,j));
+
+   real_t wy10a = fabs(d(i+1,j-1) - d(i+1,j-2));
+   real_t wy10b = fabs(d(i+1,j+1) - d(i+1,j));
+
+   real_t wy01a = fabs(d(i,j) - d(i,j-1));
+   real_t wy01b = fabs(d(i,j+2) - d(i,j+1));
+
+   real_t wy11a = fabs(d(i+1,j) - d(i+1,j-1));
+   real_t wy11b = fabs(d(i+1,j+2) - d(i+1,j+1));
+
+   if (wy00a == 0.0 && wy00b == 0.0) { wy00a = 1.0; wy00b = 1.0; }
+   if (wy10a == 0.0 && wy10b == 0.0) { wy10a = 1.0; wy10b = 1.0; }
+   if (wy01a == 0.0 && wy01b == 0.0) { wy01a = 1.0; wy01b = 1.0; }
+   if (wy11a == 0.0 && wy11b == 0.0) { wy11a = 1.0; wy11b = 1.0; }
+
+   real_t py00 = (wy00b * d(i,j-1) + wy00a * d(i,j)) / (wy00b + wy00a);
+   real_t py10 = (wy10b * d(i+1,j-1) + wy10a * d(i+1,j)) / (wy10b + wy10a);
+   real_t py01 = (wy01b * d(i,j) + wy01a * d(i,j+1)) / (wy01b + wy01a);
+   real_t py11 = (wy11b * d(i+1,j) + wy11a * d(i+1,j)) / (wy11b + wy11a);
+
+   real_t vary = py00 * wra2 * wrd * wza2 * wzb
+                 + py10 * wrb2 * wrc * wza2 * wzb
+                 - py01 * wra2 * wrd * wza * wzb2
+                 - py11 * wrb2 * wrc * wza * wzb2;
+   var += vary * dz_;
+
+   // Compute d^2var/dxdy at corners of local patch
+   real_t pxy00 = (wx00b * (wy00b * e(i-1,j-1) + wy00a * e(i-1,j)) +
+                   wx00a * (wy00b * e(i,j-1) + wy00a * e(i,j))) /
+                  ((wx00b + wx00a) * (wy00b + wy00a));
+   real_t pxy10 = (wx10b * (wy10b * e(i,j-1) + wy10a * e(i,j)) +
+                   wx10a * (wy10b * e(i+1,j-1) + wy10a * e(i+1,j))) /
+                  ((wx10b + wx10a) * (wy10b + wy10a));
+   real_t pxy01 = (wx01b * (wy01b * e(i-1,j) + wy01a * e(i-1,j+1)) +
+                   wx01a * (wy01b * e(i,j) + wy01a * e(i,j+1))) /
+                  ((wx01b + wx01a) * (wy01b + wy01a));
+   real_t pxy11 = (wx11b * (wy11b * e(i,j) + wy11a * e(i,j+1)) +
+                   wx11a * (wy11b * e(i+1,j) + wy11a * e(i+1,j+1))) /
+                  ((wx11b + wx11a) * (wy11b + wy11a));
+
+   real_t varxy = pxy00 * wra2 * wrb * wza2 * wzb
+                  - pxy10 * wra * wrb2 * wza2 * wzb
+                  - pxy01 * wra2 * wrb * wza * wzb2
+                  + pxy11 * wra * wrb2 * wza * wzb2;
+
+   var += dr_ * dz_ * varxy;
+
+   return var;
+}
+
+void G_EQDSK_Data::interpNxGradRZ(const Vector &rz,
+                                  const std::vector<real_t> &v,
+                                  const ShiftedDenseMatrix &c,
+                                  const ShiftedDenseMatrix &d,
+                                  const ShiftedDenseMatrix &e,
+                                  Vector &b)
+{
+   b.SetSize(2);
+   b = 0.0;
+
+   real_t r = rz[0];
+   real_t z = rz[1];
+
+   real_t rs = (r - RLEFT_) / RDIM_;
+   real_t zs = (z - ZMID_ + 0.5 * ZDIM_) / ZDIM_;
+
+   int i = std::max(0, std::min((int)floor(real_t(NW_-1) * rs), NW_-2));
+   int j = std::max(0, std::min((int)floor(real_t(NH_-1) * zs), NH_-2));
+
+   // Compute corners of local patch
+   real_t r0 = RLEFT_ + RDIM_ * i / (NW_ - 1);
+   real_t r1 = r0 + RDIM_ / (NW_ - 1);
+   real_t z0 = ZMID_ - 0.5 * ZDIM_ + ZDIM_ * j / (NH_ - 1);
+   real_t z1 = z0 + ZDIM_ / (NH_ - 1);
+
+   // Prepare position dependent factors
+   real_t wra = (r1 - r) / dr_, dwra = -1.0 / dr_;
+   real_t wrb = (r - r0) / dr_, dwrb = 1.0 / dr_;
+   real_t wrc = (1.0 + 2.0 * wra), dwrc = 2.0 * dwra;
+   real_t wrd = (1.0 + 2.0 * wrb), dwrd = 2.0 * dwrb;
+   real_t wra2 = wra * wra, dwra2 = 2.0 * wra * dwra;
+   real_t wrb2 = wrb * wrb, dwrb2 = 2.0 * wrb * dwrb;
+
+   real_t wza = (z1 - z) / dz_, dwza = -1.0 / dz_;
+   real_t wzb = (z - z0) / dz_, dwzb = 1.0 / dz_;
+   real_t wzc = (1.0 + 2.0 * wza), dwzc = 2.0 * dwza;
+   real_t wzd = (1.0 + 2.0 * wzb), dwzd = 2.0 * dwzb;
+   real_t wza2 = wza * wza, dwza2 = 2.0 * wza * dwza;
+   real_t wzb2 = wzb * wzb, dwzb2 = 2.0 * wzb * dwzb;
+
+   // Extract var values at corners of local patch
+   real_t p00 = v[NH_ * i + j];
+   real_t p10 = v[NH_ * (i + 1) + j];
+   real_t p01 = v[NH_ * i + j + 1];
+   real_t p11 = v[NH_ * (i + 1) + j + 1];
+
+   b[0] -=
+      (p00 * wra2 * wrd + p10 * wrb2 * wrc ) * (dwza2 * wzd + wza2 * dwzd)
+      + (p01 * wra2 * wrd + p11 * wrb2 * wrc) * (dwzb2 * wzc + wzb2 * dwzc);
+   b[1] +=
+      (p00 * wza2 * wzd + p01 * wzb2 * wzc) * (dwra2 * wrd + wra2 * dwrd)
+      + (p10 * wza2 * wzd + p11 * wzb2 * wzc) * (dwrb2 * wrc + wrb2 * dwrc);
+
+   // Compute dvar/dx at corners of local patch
+   real_t wx00a = fabs(c(i-1,j) - c(i-2,j));
+   real_t wx00b = fabs(c(i+1,j) - c(i,j));
+
+   real_t wx10a = fabs(c(i,j) - c(i-1,j));
+   real_t wx10b = fabs(c(i+2,j) - c(i+1,j));
+
+   real_t wx01a = fabs(c(i-1,j+1) - c(i-2,j+1));
+   real_t wx01b = fabs(c(i+1,j+1) - c(i,j+1));
+
+   real_t wx11a = fabs(c(i,j+1) - c(i-1,j+1));
+   real_t wx11b = fabs(c(i+2,j+1) - c(i+1,j+1));
+
+   if (wx00a == 0.0 && wx00b == 0.0) { wx00a = 1.0; wx00b = 1.0; }
+   if (wx10a == 0.0 && wx10b == 0.0) { wx10a = 1.0; wx10b = 1.0; }
+   if (wx01a == 0.0 && wx01b == 0.0) { wx01a = 1.0; wx01b = 1.0; }
+   if (wx11a == 0.0 && wx11b == 0.0) { wx11a = 1.0; wx11b = 1.0; }
+
+   real_t px00 = (wx00b * c(i-1,j) + wx00a * c(i,j)) / (wx00b + wx00a);
+   real_t px10 = (wx10b * c(i,j) + wx10a * c(i+1,j)) / (wx10b + wx10a);
+   real_t px01 = (wx01b * c(i-1,j+1) + wx01a * c(i,j+1)) / (wx01b + wx01a);
+   real_t px11 = (wx11b * c(i,j+1) + wx11a * c(i+1,j+1)) / (wx11b + wx11a);
+
+   b[0] -= dr_ *
+           ((px00 * wra2 * wrb - px10 * wrb2 * wra) *
+            (dwza2 * wzd + wza2 * dwzd) +
+            (px01 * wrb * wra2 - px11 * wra * wrb2) *
+            (dwzb2 * wzc + wzb2 * dwzc));
+
+   b[1] += dr_ *
+           ((px00 * wza2 * wzd + px01 * wzb2 * wzc) *
+            (dwra2 * wrb + wra2 * dwrb ) -
+            (px10 * wza2 * wzd + px11 * wzb2 * wzc) *
+            (dwra * wrb2 + wra * dwrb2));
+
+   // Compute dvar/dy at corners of local patch
+   real_t wy00a = fabs(d(i,j-1) - d(i,j-2));
+   real_t wy00b = fabs(d(i,j+1) - d(i,j));
+
+   real_t wy10a = fabs(d(i+1,j-1) - d(i+1,j-2));
+   real_t wy10b = fabs(d(i+1,j+1) - d(i+1,j));
+
+   real_t wy01a = fabs(d(i,j) - d(i,j-1));
+   real_t wy01b = fabs(d(i,j+2) - d(i,j+1));
+
+   real_t wy11a = fabs(d(i+1,j) - d(i+1,j-1));
+   real_t wy11b = fabs(d(i+1,j+2) - d(i+1,j+1));
+
+   if (wy00a == 0.0 && wy00b == 0.0) { wy00a = 1.0; wy00b = 1.0; }
+   if (wy10a == 0.0 && wy10b == 0.0) { wy10a = 1.0; wy10b = 1.0; }
+   if (wy01a == 0.0 && wy01b == 0.0) { wy01a = 1.0; wy01b = 1.0; }
+   if (wy11a == 0.0 && wy11b == 0.0) { wy11a = 1.0; wy11b = 1.0; }
+
+   real_t py00 = (wy00b * d(i,j-1) + wy00a * d(i,j)) / (wy00b + wy00a);
+   real_t py10 = (wy10b * d(i+1,j-1) + wy10a * d(i+1,j)) / (wy10b + wy10a);
+   real_t py01 = (wy01b * d(i,j) + wy01a * d(i,j+1)) / (wy01b + wy01a);
+   real_t py11 = (wy11b * d(i+1,j) + wy11a * d(i+1,j)) / (wy11b + wy11a);
+
+   b[0] -= dz_ *
+           ((py00 * wra2 * wrd + py10 * wrb2 * wrc) *
+            (dwza2 * wzb + wza2 * dwzb) -
+            (py01 * wra2 * wrd + py11 * wrb2 * wrc) *
+            (dwza * wzb2 + wza * dwzb2));
+   b[1] += dz_ *
+           ((py00 * wza2 * wzb - py01 * wza * wzb2) *
+            (dwra2 * wrd + wra2 * dwrd) +
+            (py10 * wza2 * wzb - py11 * wza * wzb2) *
+            (dwrb2 * wrc + wrb2 * dwrc));
+
+   // Compute d^2var/dxdy at corners of local patch
+   real_t pxy00 = (wx00b * (wy00b * e(i-1,j-1) + wy00a * e(i-1,j)) +
+                   wx00a * (wy00b * e(i,j-1) + wy00a * e(i,j))) /
+                  ((wx00b + wx00a) * (wy00b + wy00a));
+   real_t pxy10 = (wx10b * (wy10b * e(i,j-1) + wy10a * e(i,j)) +
+                   wx10a * (wy10b * e(i+1,j-1) + wy10a * e(i+1,j))) /
+                  ((wx10b + wx10a) * (wy10b + wy10a));
+   real_t pxy01 = (wx01b * (wy01b * e(i-1,j) + wy01a * e(i-1,j+1)) +
+                   wx01a * (wy01b * e(i,j) + wy01a * e(i,j+1))) /
+                  ((wx01b + wx01a) * (wy01b + wy01a));
+   real_t pxy11 = (wx11b * (wy11b * e(i,j) + wy11a * e(i,j+1)) +
+                   wx11a * (wy11b * e(i+1,j) + wy11a * e(i+1,j+1))) /
+                  ((wx11b + wx11a) * (wy11b + wy11a));
+
+   b[0] -= dr_ * dz_ * ((pxy00 * wra2 * wrb - pxy10 * wra * wrb2)
+                        * (dwza2 * wzb + wza2 * dwzb) +
+                        (pxy11 * wra * wrb2 - pxy01 * wra2 * wrb)
+                        * (dwza * wzb2 + wza * dwzb2));
+   b[1] += dr_ * dz_ * ((pxy00 * wza2 * wzb - pxy01 * wza * wzb2)
+                        * (dwra2 * wrb + wra2 * dwrb) +
+                        (pxy11 * wza * wzb2 - pxy10 * wza2 * wzb)
+                        * (dwra * wrb2 + wra * dwrb2));
+}
+
+void G_EQDSK_Data::initInterpPsi(const std::vector<real_t> &v,
+                                 std::vector<real_t> &t)
+{
+   // Initialize the divided differences
+   ShiftedVector m(NW_-1, 2); m = 0.0;
+
+   m(-2) = -2.0 * v[2] + 5.0 * v[1] - 3.0 * v[0];
+   m(-1) = -1.0 * v[2] + 3.0 * v[1] - 2.0 * v[0];
+   for (int i=0; i<NW_-1; i++)
+   {
+      m(i) = v[i+1] - v[i];
+   }
+   m(NW_-1) = 2.0 * v[NW_-1] - 3.0 * v[NW_-2] + v[NW_-3];
+   m(NW_)   = 3.0 * v[NW_-1] - 5.0 * v[NW_-2] + 2.0 * v[NW_-3];
+
+   // Initialize the Slopes
+   t.resize(NW_);
+
+   for (int i=0; i<NW_; i++)
+   {
+      if (m(i+1) == m(i) && m(i-1) == m(i-2))
+      {
+         if (m(i) == m(i-1))
+         {
+            t[i] = m(i) * dpsi_;
+         }
+         else
+         {
+            t[i] = 0.5 * (m(i-1) + m(i)) * dpsi_;
+         }
+      }
+      else
+      {
+         t[i] = (fabs(m(i+1) - m(i)) * m(i-1) +
+                 fabs(m(i-1) - m(i-2)) * m(i)) * dpsi_ /
+                (fabs(m(i+1) - m(i)) + fabs(m(i-1) - m(i-2)));
+      }
+   }
+}
+
+real_t G_EQDSK_Data::interpPsi(real_t psi, const vector<real_t> &v,
+                               const vector<real_t> &t)
+{
+   real_t psimin = std::min(SIMAG_, SIBRY_);
+   real_t psimax = std::max(SIMAG_, SIBRY_);
+
+   // Psi constrained to be between psimin and psimax
+   real_t psic = std::max(psimin, std::min(psi, psimax));
+
+   // Psi scaled to the range 0 -> 1
+   real_t psis = (psic - SIMAG_) / (SIBRY_ - SIMAG_);
+
+   // Located the bin containing psis counting from 0
+   int i0 = std::max(0, std::min((int)floor(real_t(NW_-1) * psis), NW_-2));
+   int i1 = i0 + 1;
+
+   // Compute ends of local patch
+   real_t psi0 = SIMAG_ + (SIBRY_ - SIMAG_) * i0 / (NW_ - 1);
+   real_t psi1 = psi0 + (SIBRY_ - SIMAG_) / (NW_ - 1);
+
+   // Prepare position dependent factors
+   real_t wra = (psi1 - psic) / dpsi_;
+   real_t wrb = (psic - psi0) / dpsi_;
+   real_t wrc = (1.0 + 2.0 * wra);
+   real_t wrd = (1.0 + 2.0 * wrb);
+   real_t wra2 = wra * wra;
+   real_t wrb2 = wrb * wrb;
+
+   // Extract variable values at ends of local patch
+   const real_t &p0 = v[i0];
+   const real_t &p1 = v[i1];
+
+   real_t var = p0 * wra2 * wrd + p1 * wrb2 * wrc;
+
+   // Extract dvar/dx at ends of local patch
+   const real_t &px0 = t[i0];
+   const real_t &px1 = t[i1];
+
+   real_t varx = px0 * wra2 * wrb - px1 * wrb2 * wra;
+
+   var += varx * dpsi_;
+
+   return var;
+}
+
+void G_EQDSK_Data::ExtendedDenseMatrix::init()
+{
+   // Populate four corners
+   SW_ = 3.0 * ((*this)(0,0) - (*this)(1,1)) + (*this)(2,2);
+   SE_ = 3.0 * ((*this)(m_-1,0) - (*this)(m_-2,1)) + (*this)(m_-3,2);
+   NW_ = 3.0 * ((*this)(0,n_-1) - (*this)(1,n_-2)) + (*this)(2,n_-3);
+   NE_ = 3.0 * ((*this)(m_-1,n_-1) - (*this)(m_-2,n_-2))
+         + (*this)(m_-3,n_-3);
+
+   // Populate lowest rows
+   for (int j=0; j<n_; j++)
+   {
+      S_(1,j) = 3.0 * ((*this)(0,j) - (*this)(1,j)) + (*this)(2,j);
+      S_(0,j) = 3.0 * (2.0 * (*this)(0,j) + (*this)(2,j)) - 8.0 * (*this)(1,j);
+   }
+
+   // Populate highest rows
+   for (int j=0; j<n_; j++)
+   {
+      N_(1,j) = 3.0 * (2.0 * (*this)(m_-1,j) + (*this)(m_-3,j))
+                - 8.0 * (*this)(m_-2,j);
+      N_(0,j) = 3.0 * ((*this)(m_-1,j) - (*this)(m_-2,j)) + (*this)(m_-3,j);
+   }
+
+   // Populate lowest columns
+   for (int i=0; i<m_; i++)
+   {
+      W_(i,0) = 3.0 * (2.0 * (*this)(i,0) + (*this)(i,2)) - 8.0 * (*this)(i,1);
+      W_(i,1) = 3.0 * ((*this)(i,0) - (*this)(i,1)) + (*this)(i,2);
+   }
+
+   // Populate highest columns
+   for (int i=0; i<m_; i++)
+   {
+      E_(i,0) = 3.0 * ((*this)(i,n_-1) - (*this)(i,n_-2)) + (*this)(i,n_-3);
+      E_(i,1) = 3.0 * (2.0 * (*this)(i,n_-1) + (*this)(i,n_-3))
+                - 8.0 * (*this)(i,n_-2);
+   }
+}
+
+} // namespace plasma
+
+} // namespace mfem

miniapps/plasma/g_eqdsk_data.hpp

@@ -0,0 +1,587 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_G_EQDSK_DATA_HPP
+#define MFEM_G_EQDSK_DATA_HPP
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+#include "mfem.hpp"
+#include "../../general/text.hpp"
+
+namespace mfem
+{
+
+namespace plasma
+{
+
+/// Class for reading and interpolating data stored in ASCII files
+/// following the G_EQDSK format as described in the C-Mod Wiki at
+/// https://cmodwiki.psfc.mit.edu/index.php/G_EQDSK
+///
+/// G_EQDSK files contain four types of data:
+///
+///    1) A poloidal flux function, Psi, stored as a uniform 2D grid
+///       of data values along with information describing the grid
+///       and values of the flux at the magnetic axis (SIMAG) and the
+///       plasma boundary (SIBRY).
+///
+///    2) Five 1D fields which are functions of Psi. These fields are
+///       defined on a uniform grid of points ranging from Psi = 0 to
+///       Psi = SIMAG.
+///
+///    3) Curve data describing the location of the plasma boundary
+///       and location of the limiter.
+///
+///    4) A handful of individual data values specifiying things like
+///       the total plasma current and the location of the magnetic
+///       axis.
+///
+/// The interpolation scheme is described in "A Method of Bivariate
+/// Interpolation and Smooth Surface Fitting Based on Local
+/// Procedures" by Hiroshi Akima and published in the Communications
+/// of the ACM, Numerical Mathematics, Volume 17, Number 1, January
+/// 1974.
+class G_EQDSK_Data
+{
+public:
+   G_EQDSK_Data(std::istream &is, int logging = 0);
+
+   // Number of points in radial direction
+   int GetNumPtsR() const { return NW_; }
+
+   // Number of points in z direction
+   int GetNumPtsZ() const { return NH_; }
+
+   // Width of domain in radial dimension (in meters)
+   real_t GetRExtent() const { return RDIM_; }
+
+   // Height of domain in z dimension (in meters)
+   real_t GetZExtent() const { return ZDIM_; }
+
+   // Radial coordinate at innermost edge of domain (in meters)
+   real_t GetRMin() const { return RLEFT_; }
+
+   // Z coordinate of the middle of the domain (in meters)
+   real_t GetZMid() const { return ZMID_; }
+
+   // R coordinate of the magnetic axis (in meters)
+   real_t GetRMagAxis() const { return RMAXIS_; }
+
+   // Z coordinate of the magnetic axis (in meters)
+   real_t GetZMagAxis() const { return ZMAXIS_; }
+
+   // Value of poloidal flux at the magnetic axis (in Weber / rad)
+   real_t GetPsiMagAxis() const {return SIMAG_; }
+
+   // Value of poloidal flux at the plasma boundary (in Weber / rad)
+   real_t GetPsiBdry() const {return SIBRY_; }
+
+   // Value of plasma current (in Ampere)
+   real_t GetPlasmaCurrent() const {return CURRENT_; }
+
+   // Values of poloidal flux (in Weber / rad) on the full grid in a
+   // flattened array with z-direction cycling the fastest
+   std::vector<real_t> & GetPsi() { return PSIRZ_ ;}
+
+   // Print a text block to the output stream containing basic
+   // information about the domain and the fields defined in the eqdsk
+   // file.
+   void PrintInfo(std::ostream &out = std::cout) const;
+
+   // Create a GnuPlot input file and associated data file for
+   // visualizing the fields stored in the eqdsk file.
+   void DumpGnuPlotData(const std::string &file) const;
+
+   // In the following interpolation functions the Vector argument rz
+   // is a two component vector containing first the radial coordinate
+   // and nex the z coordinate both expressed in meters.
+
+   // Interpolate the toroidal field function, F(Psi(rz) / SIMAG)
+   // (in Tesla meters), at the point rz
+   real_t InterpFPolRZ(const Vector &rz);
+
+   // Interpolate the pressure, P(Psi(rz) / SIMAG) (in N / m^2), at the
+   // point rz
+   real_t InterpPresRZ(const Vector &rz);
+
+   // Interpolate the function, F(Psi(rz) / SIMAG) * F'(Psi / SIMAG)
+   // (in (m T)^2 / (Weber / rad)), at the point rz
+   real_t InterpFFPrimeRZ(const Vector &rz);
+
+   // Interpolate the function, P'(Psi(rz) / SIMAG)
+   // (in (N / m^2) / (Weber / rad)), at the point rz
+   real_t InterpPPrimeRZ(const Vector &rz);
+
+   // Interpolate the poloidal flux function, Psi(rz) (in Weber / rad), at
+   // the point rz
+   real_t InterpPsiRZ(const Vector &rz);
+
+   // Interpolate the safety factor, q(Psi(rz) / SIMAG), at the point rz
+   real_t InterpQRZ(const Vector &rz);
+
+   // Interpolate the toroidal magnetic fleid (in Tesla) at the
+   // point rz
+   //    B_T = F(Psi(rz) / SIMAG) / r
+   real_t InterpBTorRZ(const Vector &rz);
+
+   // Interpolate the toroidal current density (in Ampere / m^2) at
+   // the point rz
+   //    J_T = r P'((Psi(rz) / SIMAG) + FF'(Psi(rz) / SIMAG) / (r mu0)
+   real_t InterpJTorRZ(const Vector &rz);
+
+   // Interpolate the rotated gradient of Psi (in Tesla) at the
+   // point rz
+   //    nxdp = (n x Grad Psi(rz))
+   // where n is the unit vector in the toroidal direction
+   void InterpNxGradPsiRZ(const Vector &rz, Vector &nxdp);
+
+   // Interpolate the poloidal magnetic field (in Tesla) at the
+   // point rz
+   //    B_P = (n x Grad Psi(rz)) / r
+   // where n is the unit vector in the toroidal direction
+   void InterpBPolRZ(const Vector &rz, Vector &b);
+
+   int GetNumBoundaryPts() const { return NBBBS_; }
+   const std::vector<real_t> & GetBoundaryRVals() const { return RBBBS_; }
+   const std::vector<real_t> & GetBoundaryZVals() const { return ZBBBS_; }
+
+   int GetNumLimiterPts() const { return LIMITR_; }
+   const std::vector<real_t> & GetLimiterRVals() const { return RLIM_; }
+   const std::vector<real_t> & GetLimiterZVals() const { return ZLIM_; }
+
+private:
+   class ShiftedVector;
+   class ShiftedDenseMatrix;
+   class ExtendedDenseMatrix;
+
+   enum FieldType {FPOL, PRES, FFPRIM, PPRIME, PSIRZ, QPSI/*, BTOR*/};
+
+   int logging_;
+   int init_flag_;
+   inline bool checkFlag(int flag) { return (init_flag_ >> flag) & 1; }
+   inline void   setFlag(int flag) { init_flag_ |= (1 << flag); }
+   inline void clearFlag(int flag) { init_flag_ &= ~(1 << flag); }
+
+   void checkPsiBoundary();
+
+   void initInterpPsi(const std::vector<real_t> &v,
+                      std::vector<real_t> &t);
+   void initInterpRZ(const std::vector<real_t> &v,
+                     ShiftedDenseMatrix &c,
+                     ShiftedDenseMatrix &d,
+                     ShiftedDenseMatrix &e);
+
+   real_t interpRZ(const Vector &rz,
+                   const std::vector<real_t> &v,
+                   const ShiftedDenseMatrix &c,
+                   const ShiftedDenseMatrix &d,
+                   const ShiftedDenseMatrix &e);
+   void interpNxGradRZ(const Vector &rz,
+                       const std::vector<real_t> &v,
+                       const ShiftedDenseMatrix &c,
+                       const ShiftedDenseMatrix &d,
+                       const ShiftedDenseMatrix &e,
+                       Vector &b);
+   real_t interpPsi(real_t psi, const std::vector<real_t> &v,
+                    const std::vector<real_t> &t);
+
+   /// The following variable names are taken from the C-Mod Wiki at
+   /// https://cmodwiki.psfc.mit.edu/index.php/G_EQDSK
+
+   std::vector<std::string> CASE_; // Identification character string
+
+   int NW_; // Number of horizontal R grid points
+   int NH_; // Number of vertical Z grid points
+
+   real_t RDIM_;    // Horizontal dimension in meter of computational box
+   real_t ZDIM_;    // Vertical dimension in meter of computational box
+   real_t RLEFT_;   // Minimum R in meter of rectangular computational box
+   real_t ZMID_;    // Z of center of computational box in meter
+   real_t RMAXIS_;  // R of magnetic axis in meter
+   real_t ZMAXIS_;  // Z of magnetic axis in meter
+   real_t SIMAG_;   // poloidal flux at magnetic axis in Weber /rad
+   real_t SIBRY_;   // poloidal flux at the plasma boundary in Weber /rad
+   real_t RCENTR_;  // R in meter of vacuum toroidal magnetic field BCENTR
+   real_t BCENTR_;  // Vacuum toroidal magnetic field in Tesla at RCENTR
+   real_t CURRENT_; // Plasma current in Ampere
+
+   // Poloidal current function in m-T, F = RBT on flux grid
+   std::vector<real_t> FPOL_;
+
+   // Plasma pressure in nt / m^2 on uniform flux grid
+   std::vector<real_t> PRES_;
+
+   // FF’(ψ) in (mT)^2 / (Weber /rad) on uniform flux grid
+   std::vector<real_t> FFPRIM_;
+
+   // P’(ψ) in (nt /m^2) / (Weber /rad) on uniform flux grid
+   std::vector<real_t> PPRIME_;
+
+   // Poloidal flux in Weber / rad on the rectangular grid points
+   std::vector<real_t> PSIRZ_;
+
+   // q values on uniform flux grid from axis to boundary
+   std::vector<real_t> QPSI_;
+
+   int                 NBBBS_;  // Number of boundary points
+   std::vector<real_t> RBBBS_;  // R of boundary points in meter
+   std::vector<real_t> ZBBBS_;  // Z of boundary points in meter
+
+   int                 LIMITR_; // Number of limiter points
+   std::vector<real_t> RLIM_;   // R of surrounding limiter contour in meter
+   std::vector<real_t> ZLIM_;   // Z of surrounding limiter contour in meter
+
+   class ShiftedVector : public Vector
+   {
+   private:
+      int si_;
+   public:
+      ShiftedVector()
+         : si_(0) {}
+
+      ShiftedVector(int s, int si)
+         : Vector(s+2*si), si_(si) {}
+
+      void SetShift(int si) { si_ = si; }
+
+      ShiftedVector &operator=(real_t c)
+      { Vector::operator=(c); return *this; }
+
+      inline real_t &operator()(int i)
+      { return Vector::operator()(i + si_); }
+
+      inline const real_t &operator()(int i) const
+      { return Vector::operator()(i + si_); }
+   };
+
+   class ShiftedDenseMatrix : public DenseMatrix
+   {
+   private:
+      int si_, sj_;
+   public:
+      ShiftedDenseMatrix()
+         : si_(0), sj_(0) {}
+
+      ShiftedDenseMatrix(int m, int n, int si, int sj)
+         : DenseMatrix(m+2*si, n+2*sj), si_(si), sj_(sj) {}
+
+      void SetShifts(int si, int sj) { si_ = si; sj_ = sj; }
+
+      ShiftedDenseMatrix &operator=(real_t c)
+      { DenseMatrix::operator=(c); return *this; }
+
+      inline real_t &operator()(int i, int j)
+      { return DenseMatrix::operator()(i + si_, j + sj_); }
+
+      inline const real_t &operator()(int i, int j) const
+      { return DenseMatrix::operator()(i + si_, j + sj_); }
+   };
+
+   class ExtendedDenseMatrix
+   {
+   private:
+      int m_, n_;
+      const real_t *C_;
+      DenseMatrix N_;
+      DenseMatrix S_;
+      DenseMatrix E_;
+      DenseMatrix W_;
+      real_t SW_, SE_, NW_, NE_, DUMMY_;
+
+      void init();
+
+   public:
+      ExtendedDenseMatrix(const real_t *C, int m, int n)
+         : m_(m), n_(n), C_(C),
+           N_(2, n), S_(2, n),
+           E_(m, 2), W_(m, 2),
+           SW_(0.0), SE_(0.0), NW_(0.0), NE_(0.0), DUMMY_(0.0)
+      { N_ = 0.0; S_ = 0.0; E_ = 0.0; W_ = 0.0; init(); }
+
+      const real_t &operator()(int i, int j) const
+      {
+         if (i >= 0 && i < m_ && j >= 0 && j < n_)
+         {
+            return C_[n_ * i + j];
+         }
+         else if (i >= 0 && i < m_)
+         {
+            if (j < 0)
+            {
+               return W_(i, j + 2);
+            }
+            else
+            {
+               return E_(i, j - n_);
+            }
+         }
+         else if (j >= 0 && j < n_)
+         {
+            if (i < 0)
+            {
+               return S_(i + 2, j);
+            }
+            else
+            {
+               return N_(i - m_, j);
+            }
+         }
+         else if (i == -1 && j == -1)
+         {
+            return SW_;
+         }
+         else if (i == -1 && j == n_)
+         {
+            return SE_;
+         }
+         else if (i == m_ && j == -1)
+         {
+            return NW_;
+         }
+         else if (i == m_ && j == n_)
+         {
+            return NE_;
+         }
+         return DUMMY_;
+      }
+   };
+
+   // Divided differences for Akima's interpolation method
+   real_t dr_, dz_, dpsi_;
+
+   std::vector<real_t> FPOL_t_;
+   std::vector<real_t> PRES_t_;
+   std::vector<real_t> FFPRIM_t_;
+   std::vector<real_t> PPRIME_t_;
+   ShiftedDenseMatrix  PSIRZ_c_;
+   ShiftedDenseMatrix  PSIRZ_d_;
+   ShiftedDenseMatrix  PSIRZ_e_;
+   std::vector<real_t> QPSI_t_;
+};
+
+class G_EQDSK_Psi_Coefficient : public Coefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+
+public:
+
+   G_EQDSK_Psi_Coefficient(G_EQDSK_Data &g_eqdsk) : eqdsk(g_eqdsk) {}
+
+   real_t Eval(ElementTransformation & T,
+               const IntegrationPoint & ip)
+   {
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      return eqdsk.InterpPsiRZ(transip);
+   }
+};
+
+class G_EQDSK_FPol_Coefficient : public Coefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+
+public:
+
+   G_EQDSK_FPol_Coefficient(G_EQDSK_Data &g_eqdsk) : eqdsk(g_eqdsk) {}
+
+   real_t Eval(ElementTransformation & T,
+               const IntegrationPoint & ip)
+   {
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      return eqdsk.InterpFPolRZ(transip);
+   }
+};
+
+class G_EQDSK_Pres_Coefficient : public Coefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+
+public:
+
+   G_EQDSK_Pres_Coefficient(G_EQDSK_Data &g_eqdsk) : eqdsk(g_eqdsk) {}
+
+   real_t Eval(ElementTransformation & T,
+               const IntegrationPoint & ip)
+   {
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      return eqdsk.InterpPresRZ(transip);
+   }
+};
+
+class G_EQDSK_Q_Coefficient : public Coefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+
+public:
+
+   G_EQDSK_Q_Coefficient(G_EQDSK_Data &g_eqdsk) : eqdsk(g_eqdsk) {}
+
+   real_t Eval(ElementTransformation & T,
+               const IntegrationPoint & ip)
+   {
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      return eqdsk.InterpQRZ(transip);
+   }
+};
+
+class G_EQDSK_BTor_Coefficient : public Coefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+
+public:
+
+   G_EQDSK_BTor_Coefficient(G_EQDSK_Data &g_eqdsk) : eqdsk(g_eqdsk) {}
+
+   real_t Eval(ElementTransformation & T,
+               const IntegrationPoint & ip)
+   {
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      return eqdsk.InterpBTorRZ(transip);
+   }
+};
+
+class G_EQDSK_JTor_Coefficient : public Coefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+
+public:
+
+   G_EQDSK_JTor_Coefficient(G_EQDSK_Data &g_eqdsk) : eqdsk(g_eqdsk) {}
+
+   real_t Eval(ElementTransformation & T,
+               const IntegrationPoint & ip)
+   {
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      return eqdsk.InterpJTorRZ(transip);
+   }
+};
+
+class G_EQDSK_NxGradPsi_Coefficient : public VectorCoefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+
+public:
+
+   G_EQDSK_NxGradPsi_Coefficient(G_EQDSK_Data &g_eqdsk)
+      : VectorCoefficient(2), eqdsk(g_eqdsk) {}
+
+   void Eval(Vector &b, ElementTransformation & T,
+             const IntegrationPoint & ip)
+   {
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      eqdsk.InterpNxGradPsiRZ(transip, b);
+   }
+
+};
+
+class G_EQDSK_BPol_Coefficient : public VectorCoefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+
+public:
+
+   G_EQDSK_BPol_Coefficient(G_EQDSK_Data &g_eqdsk)
+      : VectorCoefficient(2), eqdsk(g_eqdsk) {}
+
+   void Eval(Vector &b, ElementTransformation & T,
+             const IntegrationPoint & ip)
+   {
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      eqdsk.InterpBPolRZ(transip, b);
+   }
+
+};
+
+class G_EQDSK_BField_VecCoefficient : public VectorCoefficient
+{
+private:
+   G_EQDSK_Data &eqdsk;
+   bool unit_;
+
+public:
+
+   G_EQDSK_BField_VecCoefficient(G_EQDSK_Data &g_eqdsk, bool unit)
+      : VectorCoefficient(3), eqdsk(g_eqdsk), unit_(unit) {}
+
+   void Eval(Vector &V, ElementTransformation & T,
+             const IntegrationPoint & ip)
+   {
+      V.SetSize(3);
+      Vector b;
+      b.SetSize(2);
+      real_t x[3];
+      Vector transip(x, 3);
+
+      T.Transform(ip, transip);
+
+      eqdsk.InterpBPolRZ(transip, b);
+      real_t btor = eqdsk.InterpBTorRZ(transip);
+
+      V[0] = b[0];
+      V[1] = b[1];
+      V[2] = btor;
+
+      if ( unit_ )
+      {
+         real_t bmag = sqrt(V * V);
+         V /= bmag;
+      }
+   }
+};
+
+} // namespace plasma
+
+} // namespace mfem
+
+#endif // MFEM_G_EQDSK_DATA_HPP

miniapps/plasma/g_eqdsk_viewer.cpp

@@ -0,0 +1,361 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "mfem.hpp"
+#include "../common/fem_extras.hpp"
+#include "g_eqdsk_data.hpp"
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::common;
+using namespace mfem::plasma;
+
+void ShiftMesh(real_t x0, real_t y0, Mesh &mesh);
+
+int main(int argc, char *argv[])
+{
+   const char *eqdsk_file = "";
+   const char *mesh_file = "";
+
+   int order = 1;
+   bool visualization = true;
+   bool visit = false;
+   bool paraview = false;
+   bool binary = false;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&eqdsk_file, "-eqdsk", "--eqdsk-file",
+                  "G EQDSK input file.");
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
+                  "--no-paraview-datafiles",
+                  "Save data files for ParaView (paraview.org) visualization.");
+   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
+                  "--ascii-datafiles",
+                  "Use binary (Sidre) or ascii format for VisIt data files.");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   named_ifgzstream ieqdsk(eqdsk_file);
+   if (!ieqdsk)
+   {
+      return 1;
+   }
+
+   G_EQDSK_Data eqdsk(ieqdsk);
+   eqdsk.PrintInfo();
+   eqdsk.DumpGnuPlotData("gnuplot_eqdsk");
+
+   G_EQDSK_Psi_Coefficient psiCoef(eqdsk);
+   G_EQDSK_FPol_Coefficient fPolCoef(eqdsk);
+   G_EQDSK_Pres_Coefficient presCoef(eqdsk);
+   G_EQDSK_Q_Coefficient qCoef(eqdsk);
+   G_EQDSK_NxGradPsi_Coefficient nxGradPsiCoef(eqdsk);
+   G_EQDSK_BPol_Coefficient BPolCoef(eqdsk);
+   G_EQDSK_BTor_Coefficient BTorCoef(eqdsk);
+   G_EQDSK_JTor_Coefficient JTorCoef(eqdsk);
+
+   Mesh mesh;
+   if (strcmp(mesh_file, "") == 0)
+   {
+      mesh = Mesh::MakeCartesian2D(eqdsk.GetNumPtsR(),
+                                   eqdsk.GetNumPtsZ(),
+                                   Element::QUADRILATERAL,
+                                   false,
+                                   eqdsk.GetRExtent(),
+                                   eqdsk.GetZExtent());
+
+      real_t zmin = eqdsk.GetZMid() - eqdsk.GetZExtent()/2.0;
+      ShiftMesh(eqdsk.GetRMin(), zmin, mesh);
+   }
+   else
+   {
+      mesh = Mesh(mesh_file);
+   }
+
+   H1_FECollection fec_h1(order, 2);
+   FiniteElementSpace fes_h1(&mesh, &fec_h1);
+   FiniteElementSpace fes_h1v(&mesh, &fec_h1, 2);
+
+   GridFunction psi(&fes_h1);
+   psi.ProjectCoefficient(psiCoef);
+
+   GridFunction nxGradPsi(&fes_h1v);
+   nxGradPsi.ProjectCoefficient(nxGradPsiCoef);
+
+   GridFunction fPol(&fes_h1);
+   fPol.ProjectCoefficient(fPolCoef);
+
+   GridFunction pres(&fes_h1);
+   pres.ProjectCoefficient(presCoef);
+
+   GridFunction q(&fes_h1);
+   q.ProjectCoefficient(qCoef);
+
+   GridFunction BPol(&fes_h1v);
+   BPol.ProjectCoefficient(BPolCoef);
+
+   GridFunction BTor(&fes_h1);
+   BTor.ProjectCoefficient(BTorCoef);
+
+   GridFunction JTor(&fes_h1);
+   JTor.ProjectCoefficient(JTorCoef);
+
+   int xPos = 0, yPos = 0, w = 400, h = 300, b = 30, m = 65;
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+
+      char skeys[] = "mmaaAcjR";
+      char vkeys[] = "vvvmmaaAcjR";
+
+      socketstream sock_fpol;
+      VisualizeField(sock_fpol, vishost, visport, fPol, "Current Flux",
+                     xPos, yPos, w, h, skeys);
+      xPos += w;
+
+      socketstream sock_pres;
+      VisualizeField(sock_pres, vishost, visport, pres, "Pressure",
+                     xPos, yPos, w, h, "mmaaAcjR");
+      xPos += w;
+
+      socketstream sock_psi;
+      VisualizeField(sock_psi, vishost, visport, psi, "Poloidal Flux",
+                     xPos, yPos, w, h, "mmaaAcjR");
+      xPos += w;
+
+      socketstream sock_q;
+      VisualizeField(sock_q, vishost, visport, q, "Safety Factor (q)",
+                     xPos, yPos, w, h, "mmaaAcjR");
+      xPos = 0; yPos += h + b + m;
+
+      socketstream sock_bpol;
+      VisualizeField(sock_bpol, vishost, visport, BPol, "Poloidal B",
+                     xPos, yPos, w, h, vkeys, true);
+      xPos += w;
+
+      socketstream sock_btor;
+      VisualizeField(sock_btor, vishost, visport, BTor, "Toroidal B",
+                     xPos, yPos, w, h, "mmaaAcjR");
+      xPos += w;
+
+      socketstream sock_jtor;
+      VisualizeField(sock_jtor, vishost, visport, JTor, "Toroidal J",
+                     xPos, yPos, w, h, "mmaaAcjR");
+      xPos = 0; yPos += h + b;
+   }
+
+   Array<DataCollection*> dc(2); dc = NULL;
+
+   if (visit)
+   {
+#ifdef MFEM_USE_SIDRE
+      if (binary)
+      {
+         dc[0] = new SidreDataCollection("G_EQDSK_Viewer", &mesh);
+      }
+      else
+#else
+      {
+         dc[0] = new VisItDataCollection("G_EQDSK_Viewer", &mesh);
+         dc[0]->SetPrecision(precision);
+      }
+#endif
+      }
+   if (paraview)
+   {
+      ParaViewDataCollection *pd =
+         new ParaViewDataCollection("G_EQDSK_Viewer", &mesh);
+      pd->SetPrefixPath("ParaView");
+      pd->SetHighOrderOutput(true);
+      if (binary) { pd->SetDataFormat(VTKFormat::BINARY); }
+      dc[1] = pd;
+   }
+
+   for (int i=0; i<2; i++)
+   {
+      if (dc[i] == NULL) { continue; }
+
+      dc[i]->SetCycle(0);
+      dc[i]->SetTime(0.0);
+
+      dc[i]->RegisterField("Psi", &psi);
+      dc[i]->RegisterField("FPol", &fPol);
+      dc[i]->RegisterField("Pres", &pres);
+      dc[i]->RegisterField("Q", &q);
+      dc[i]->RegisterField("nxGradPsi", &nxGradPsi);
+      dc[i]->RegisterField("BPol", &BPol);
+      dc[i]->RegisterField("BTor", &BTor);
+      dc[i]->RegisterField("JTor", &JTor);
+
+      dc[i]->Save();
+   }
+   delete dc[0];
+   delete dc[1];
+
+   {
+      int nbdr = eqdsk.GetNumBoundaryPts();
+      const vector<real_t> &r = eqdsk.GetBoundaryRVals();
+      const vector<real_t> &z = eqdsk.GetBoundaryZVals();
+
+      Mesh bdr(1, nbdr, nbdr-1, 2, 2);
+
+      for (int i=0; i<nbdr; i++)
+      {
+         bdr.AddVertex(r[i], z[i]);
+      }
+
+      for (int i=1; i<nbdr; i++)
+      {
+         bdr.AddSegment(i-1, i);
+      }
+
+      bdr.AddBdrPoint(0);
+      bdr.AddBdrPoint(nbdr-1);
+
+      bdr.FinalizeMesh();
+
+      if (visualization)
+      {
+         socketstream sock;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         VisualizeMesh(sock, vishost, visport, bdr, "Plasma Boundary",
+                       xPos, yPos, w, h, "aaA");
+         xPos += w;
+      }
+      if (visit)
+      {
+#ifdef MFEM_USE_SIDRE
+         if (binary)
+         {
+            SidreDataCollection sd("G_EQDSK_Viewer_Boundary", &bdr);
+            sd.Save();
+         }
+         else
+#else
+         {
+            VisItDataCollection vd("G_EQDSK_Viewer_Boundary", &bdr);
+            vd.SetPrecision(precision);
+            vd.Save();
+         }
+#endif
+         }
+      if (paraview)
+      {
+         ParaViewDataCollection pd("G_EQDSK_Viewer_Boundary", &bdr);
+         pd.SetPrefixPath("ParaView");
+         pd.SetHighOrderOutput(true);
+         if (binary) { pd.SetDataFormat(VTKFormat::BINARY); }
+         pd.Save();
+      }
+   }
+
+   {
+      int nlim = eqdsk.GetNumLimiterPts();
+      const vector<real_t> &r = eqdsk.GetLimiterRVals();
+      const vector<real_t> &z = eqdsk.GetLimiterZVals();
+
+      Mesh lim(1, nlim, nlim-1, 2, 2);
+
+      for (int i=0; i<nlim; i++)
+      {
+         lim.AddVertex(r[i], z[i]);
+      }
+
+      for (int i=1; i<nlim; i++)
+      {
+         lim.AddSegment(i-1, i);
+      }
+
+      lim.AddBdrPoint(0);
+      lim.AddBdrPoint(nlim-1);
+
+      lim.FinalizeMesh();
+
+      if (visualization)
+      {
+         socketstream sock;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         VisualizeMesh(sock, vishost, visport, lim, "Limiter",
+                       xPos, yPos, w, h, "aaA");
+         xPos += w;
+      }
+      if (visit)
+      {
+#ifdef MFEM_USE_SIDRE
+         if (binary)
+         {
+            SidreDataCollection sd("G_EQDSK_Viewer_Limiter", &lim);
+            sd.Save();
+         }
+         else
+#else
+         {
+            VisItDataCollection vd("G_EQDSK_Viewer_Limiter", &lim);
+            vd.SetPrecision(precision);
+            vd.Save();
+         }
+#endif
+         }
+      if (paraview)
+      {
+         ParaViewDataCollection pd("G_EQDSK_Viewer_Limiter", &lim);
+         pd.SetPrefixPath("ParaView");
+         pd.SetHighOrderOutput(true);
+         if (binary) { pd.SetDataFormat(VTKFormat::BINARY); }
+         pd.Save();
+      }
+   }
+}
+
+void ShiftMesh(real_t x0, real_t y0, Mesh &mesh)
+{
+   class ShiftCoef : public VectorCoefficient
+   {
+   private:
+      real_t xs_, ys_;
+
+   public:
+      ShiftCoef(real_t xs, real_t ys) : VectorCoefficient(2), xs_(xs), ys_(ys) {}
+
+      void Eval(Vector &v, ElementTransformation &T, const IntegrationPoint &ip)
+      {
+         T.Transform(ip, v);
+         v[0] += xs_;
+         v[1] += ys_;
+      }
+   };
+
+   ShiftCoef shift(x0, y0);
+   mesh.Transform(shift);
+}

miniapps/plasma/makefile

@@ -18,7 +18,7 @@ CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_MINIAPPS = 
+SEQ_MINIAPPS = g_eqdsk_viewer
 PAR_MINIAPPS = 
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
@@ -39,11 +39,14 @@ SUBDIRS_TPRINT = $(addsuffix /test-print,$(PLASMA_SUBDIRS))
 .PHONY: all lib-common clean clean-build clean-exec
 .PRECIOUS: %.o
 
+COMMON_O = g_eqdsk_data.o
+
 COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common
 
 # If MFEM_SHARED is set, add the ../common rpath
 COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\
-   $(MFEM_XLINKER)-rpath,$(abspath $(MFEM_BUILD_DIR)/miniapps/common))
+   $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\
+   $(MFEM_BUILD_DIR)/miniapps/common))
 
 # Remove built-in rules
 %: %.cpp
@@ -53,15 +56,22 @@ all: $(MINIAPPS) $(SUBDIRS_ALL)
 
 .PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) \
    $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
+
+# Rules for building the miniapps
+%: $(SRC)%.cpp $(COMMON_O) $(MFEM_LIB_FILE) $(CONFIG_MK) | lib-common
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) $< -o $@ $(COMMON_O) $(COMMON_LIB) \
+	  $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$(COMMON_O) $(addsuffix _solver.o,$(MINIAPPS)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
+
 $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) $(SUBDIRS_CLEAN):
 	$(MAKE) -C $(@D) $(@F)
 $(SUBDIRS_TPRINT):
 	@$(MAKE) -C $(@D) $(@F)
 
-# Rules for building the miniapps
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK) | lib-common
-	$(MFEM_CXX) $(MFEM_LINK_FLAGS) $< -o $@ $(COMMON_LIB) $(MFEM_LIBS)
-
 # Rule for building lib-common
 lib-common:
 	$(MAKE) -C $(MFEM_BUILD_DIR)/miniapps/common
@@ -88,3 +98,4 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
+	@rm -rf G_EQDSK_Viewer_* gnuplot_eqdsk.*

miniapps/plasma/plasma.hpp

@@ -59,4 +59,3 @@ typedef std::complex<real_t> complex_t;
 } // namespace mfem
 
 #endif // MFEM_PLASMA_HPP
-

miniapps/plasma/sgl_null.g_eqdsk

@@ -0,0 +1,211 @@
+  SGRRATEI    01/01/2025    #1  0ms           3 20 40
+ 1.000000000E+00 2.000000000E+00 1.000000000E+00 5.000000000E-01 0.000000000E+00
+ 1.105630563E+00 3.500092352E-02 2.347299201E+01 0.000000000E+00 1.000000000E+00
+ 4.093014437E+05 2.347299201E+01 0.000000000E+00 1.105630563E+00 0.000000000E+00
+ 3.500092352E-02 0.000000000E+00 0.000000000E+00 0.000000000E+00 0.000000000E+00
+ 9.738254114E-01 9.765403940E-01 9.791016917E-01 9.815105078E-01 9.837679622E-01
+ 9.858750948E-01 9.878328675E-01 9.896421668E-01 9.913038055E-01 9.928185251E-01
+ 9.941869971E-01 9.954098248E-01 9.964875442E-01 9.974206257E-01 9.982094750E-01
+ 9.988544337E-01 9.993557806E-01 9.997137315E-01 9.999284406E-01 1.000000000E+00
+ 5.710133214E+04 5.124884104E+04 4.571270080E+04 4.049291144E+04 3.558947294E+04
+ 3.100238532E+04 2.673164857E+04 2.277726268E+04 1.913922767E+04 1.581754353E+04
+ 1.281221026E+04 1.012322786E+04 7.750596330E+03 5.694315671E+03 3.954385882E+03
+ 2.530806965E+03 1.423578918E+03 6.327017412E+02 1.581754353E+02 0.000000000E+00
+-5.305270078E-02-5.012071900E-02-4.721240459E-02-4.432615176E-02-4.146040916E-02
+-3.861367525E-02-3.578449395E-02-3.297145057E-02-3.017316804E-02-2.738830324E-02
+-2.461554357E-02-2.185360373E-02-1.910122258E-02-1.635716013E-02-1.362019471E-02
+-1.088912014E-02-8.162743045E-03-5.439880231E-03-2.719356078E-03-0.000000000E+00
+ 1.142026643E+05 1.081919977E+05 1.021813312E+05 9.617066466E+04 9.015999812E+04
+ 8.414933158E+04 7.813866504E+04 7.212799850E+04 6.611733196E+04 6.010666541E+04
+ 5.409599887E+04 4.808533233E+04 4.207466579E+04 3.606399925E+04 3.005333271E+04
+ 2.404266617E+04 1.803199962E+04 1.202133308E+04 6.010666541E+03 0.000000000E+00
+ 1.177939031E+00 7.114757766E-01 3.462248040E-01 5.354204611E-02-1.920202929E-01
+-4.129262653E-01-6.282055905E-01-8.521376374E-01-1.092012071E+00-1.345131509E+00
+-1.595544748E+00-1.811409078E+00-1.944310502E+00-1.932184779E+00-1.707470864E+00
+-1.211535856E+00-4.150082925E-01 6.586307082E-01 1.911775275E+00 3.162156344E+00
+ 1.061937089E+00 6.633173301E-01 3.492668082E-01 9.315032283E-02-1.304531101E-01
+-3.458479130E-01-5.756143723E-01-8.390126101E-01-1.149238113E+00-1.509585272E+00
+-1.908915919E+00-2.317294334E+00-2.683185289E+00-2.934087898E+00-2.982678795E+00
+-2.740175983E+00-2.137415484E+00-1.151880382E+00 1.642429581E-01 1.663372295E+00
+ 8.051939480E-01 5.069112372E-01 2.743284476E-01 8.370657885E-02-8.974129464E-02
+-2.719121972E-01-4.891295045E-01-7.663397670E-01-1.123909079E+00-1.572938455E+00
+-2.109368849E+00-2.707664515E+00-3.315512301E+00-3.851636238E+00-4.209280562E+00
+-4.267839746E+00-3.914144066E+00-3.072748961E+00-1.741194852E+00-2.206373816E-02
+ 4.386618733E-01 2.671699153E-01 1.423176570E-01 4.390607362E-02-5.171653318E-02
+-1.717821368E-01-3.464493081E-01-6.068989469E-01-9.817630445E-01-1.491645200E+00
+-2.141844861E+00-2.913964020E+00-3.757840297E+00-4.586112160E+00-5.274458926E+00
+-5.670818478E+00-5.616224816E+00-4.977901073E+00-3.691687034E+00-1.806058709E+00
+-4.298567177E-03-2.885908269E-02-2.408384075E-02-6.214900590E-03 2.635241343E-03
+-2.582736598E-02-1.255823084E-01-3.344252645E-01-6.901634068E-01-1.224454514E+00
+-1.954213674E+00-2.871118150E+00-3.930618618E+00-5.042932053E+00-6.069530696E+00
+-6.829272879E+00-7.118023483E+00-6.743823605E+00-5.575998604E+00-3.601125638E+00
+-4.904906367E-01-3.541583261E-01-2.023319275E-01-4.707332712E-02 9.136750994E-02
+ 1.839291606E-01 1.929332900E-01 7.375989635E-02-2.212853526E-01-7.362958772E-01
+-1.501990597E+00-2.523186013E+00-3.764838139E+00-5.139247280E+00-6.498368268E+00
+-7.636196547E+00-8.306311506E+00-8.258154862E+00-7.291916149E+00-5.325855281E+00
+-9.891517976E-01-6.836808359E-01-3.717549013E-01-6.118268240E-02 2.299116539E-01
+ 4.720201133E-01 6.239925947E-01 6.343426860E-01 4.449799758E-01-1.810539076E-03
+-7.526144758E-01-1.828479362E+00-3.208038614E+00-4.810814863E+00-6.485007525E+00
+-8.005508573E+00-9.088448866E+00-9.427424816E+00-8.752906042E+00-6.909813980E+00
+-1.473771796E+00-9.959343881E-01-5.149915677E-01-3.452799782E-02 4.296819687E-01
+ 8.480325882E-01 1.176224437E+00 1.356019748E+00 1.318626308E+00 9.917412651E-01
+ 3.110382488E-01-7.637621813E-01-2.229234570E+00-4.017630769E+00-5.979886299E+00
+-7.878690259E+00-9.399094337E+00-1.018339893E+01-9.893563051E+00-8.297535837E+00
+-1.923358234E+00-1.274118331E+00-6.190449342E-01 4.241617768E-02 6.970427222E-01
+ 1.315504578E+00 1.850755536E+00 2.238061887E+00 2.397821839E+00 2.242420073E+00
+ 1.688171532E+00 6.727868194E-01-8.223908342E-01-2.747908855E+00-4.964257479E+00
+-7.229487499E+00-9.204966355E+00-1.048758220E+01-1.067347645E+01-9.451344740E+00
+-2.323104633E+00-1.506765900E+00-6.759012096E-01 1.740804302E-01 1.032700577E+00
+ 1.871293443E+00 2.640527557E+00 3.269546701E+00 3.668019576E+00 3.732554236E+00
+ 3.358812466E+00 2.460076739E+00 9.918161564E-01-1.020108269E+00-3.452453075E+00
+-6.066340582E+00-8.507319402E+00-1.033357420E+01-1.107925638E+01-1.035282773E+01
+-2.664471122E+00-1.687890207E+00-6.827027259E-01 3.596274997E-01 1.431550261E+00
+ 2.505452940E+00 3.530213458E+00 4.429318046E+00 5.101959709E+00 5.428780921E+00
+ 5.283882595E+00 4.554194090E+00 3.166091448E+00 1.117147246E+00-1.491830392E+00
+-4.432367149E+00-7.341965785E+00-9.747121227E+00-1.112459668E+01-1.100287931E+01
+-2.944738080E+00-1.816674081E+00-6.415005103E-01 5.931963199E-01 1.882965615E+00
+ 3.201616345E+00 4.496734579E+00 5.686676598E+00 6.660570233E+00 7.283188581E+00
+ 7.406605934E+00 6.890087547E+00 5.628476275E+00 3.587263544E+00 8.396022394E-01
+-2.402899635E+00-5.776849067E+00-8.783877585E+00-1.084839190E+01-1.142033969E+01
+-3.166134203E+00-1.896779380E+00-5.586445387E-01 8.645161131E-01 2.371494866E+00
+ 3.937845874E+00 5.510336076E+00 7.002752880E+00 8.294707488E+00 9.235489503E+00
+ 9.655166287E+00 9.384741989E+00 8.286038706E+00 6.289838350E+00 3.437717133E+00
+-8.075321028E-02-3.907350460E+00-7.525051779E+00-1.031106577E+01-1.163935097E+01
+-3.334669430E+00-1.935377409E+00-4.438947453E-01 1.159817788E+00 2.877890628E+00
+ 4.687875792E+00 6.536138946E+00 8.332465883E+00 9.947618768E+00 1.121607806E+01
+ 1.194643788E+01 1.194163044E+01 1.103005146E+01 9.106506552E+00 6.178640076E+00
+ 2.410438564E+00-1.849670608E+00-6.071264098E+00-9.589395893E+00-1.170564693E+01
+-3.458817908E+00-1.942017179E+00-3.093519611E-01 1.462951722E+00 3.380383697E+00
+ 5.422650509E+00 7.536054524E+00 9.626930107E+00 1.155795906E+01 1.314976842E+01
+ 1.419053815E+01 1.445614268E+01 1.374230350E+01 1.190806493E+01 8.926518370E+00
+ 4.934346872E+00 2.672349659E-01-4.535068265E+00-8.770225658E+00-1.167206778E+01
+-3.548196371E+00-1.927451469E+00-1.683142698E-01 1.756611464E+00 3.856096505E+00
+ 6.112041329E+00 8.470924349E+00 1.083614428E+01 1.306315655E+01 1.495995902E+01
+ 1.629589474E+01 1.682162166E+01 1.630211309E+01 1.456238365E+01 1.154214088E+01
+ 7.350954429E+00 2.310587678E+00-3.032682981E+00-7.943532064E+00-1.159364507E+01
+-3.612369480E+00-1.902532645E+00-3.416086848E-02 2.023562298E+00 4.282487674E+00
+ 6.726617299E+00 9.302735925E+00 1.191177809E+01 1.440289942E+01 1.657294315E+01
+ 1.817448446E+01 1.893559777E+01 1.859356268E+01 1.694255836E+01 1.389180895E+01
+ 9.525810665E+00 4.152947024E+00-1.675532760E+00-7.195361659E+00-1.152263127E+01
+-3.659888135E+00-1.877272235E+00 8.064666376E-02 2.247781331E+00 4.638723390E+00
+ 7.239346930E+00 9.996763752E+00 1.280986971E+01 1.552251082E+01 1.792207710E+01
+ 1.974689233E+01 2.070579945E+01 2.051246236E+01 1.893474235E+01 1.585584220E+01
+ 1.133888571E+01 5.680950215E+00-5.622075887E-01-6.601153114E+00-1.150384995E+01
+-3.697632389E+00-1.860132684E+00 1.653751070E-01 2.415430773E+00 4.906883382E+00
+ 7.627118007E+00 1.052349470E+01 1.349325702E+01 1.637598974E+01 1.895153030E+01
+ 2.094685366E+01 2.205553690E+01 2.197257081E+01 2.044512383E+01 1.733613236E+01
+ 1.269240556E+01 6.803013202E+00 2.285826261E-01-6.219933983E+00-1.157071687E+01
+-3.730491265E+00-1.857589350E+00 2.115959967E-01 2.515606446E+00 5.072927797E+00
+ 7.871980905E+00 1.086021634E+01 1.393358738E+01 1.692852212E+01 1.961937366E+01
+ 2.172498135E+01 2.292810207E+01 2.291065737E+01 2.140557651E+01 1.826222886E+01
+ 1.351712669E+01 7.455453684E+00 6.435576415E-01-6.089813048E+00-1.174223127E+01
+-3.761371558E+00-1.873968915E+00 2.135078960E-01 2.540827286E+00 5.127374233E+00
+ 7.962045089E+00 1.099217427E+01 1.411278363E+01 1.715830588E+01 1.989981072E+01
+ 2.205143928E+01 2.328989944E+01 2.329007266E+01 2.177760782E+01 1.859554648E+01
+ 1.377662221E+01 7.606612427E+00 6.585296979E-01-6.225093469E+00-1.202116407E+01
+-3.791490666E+00-1.911540109E+00 1.680110330E-01 2.487256301E+00 5.065659571E+00
+ 7.891986670E+00 1.091323759E+01 1.402388427E+01 1.705758364E+01 1.978441808E+01
+ 2.191739820E+01 2.313211229E+01 2.310260165E+01 2.155435103E+01 1.833142069E+01
+ 1.346929542E+01 7.258694626E+00 2.799266470E-01-6.615212606E+00-1.239358030E+01
+-3.820878312E+00-1.970807023E+00 7.456425210E-02 2.354666889E+00 4.888186740E+00
+ 7.663154519E+00 1.062604784E+01 1.367122044E+01 1.663283261E+01 1.928233088E+01
+ 2.133519587E+01 2.247081246E+01 2.236849430E+01 2.076048600E+01 1.749888920E+01
+ 1.262799924E+01 6.447217361E+00-4.559222017E-01-7.225580581E+00-1.282973439E+01
+-3.848991491E+00-2.050936655E+00-6.512725250E-02 2.146188553E+00 4.600080065E+00
+ 7.283291332E+00 1.014166289E+01 1.306993852E+01 1.590411969E+01 1.841938220E+01
+ 2.033721296E+01 2.134552973E+01 2.113469622E+01 1.945011906E+01 1.615823925E+01
+ 1.131731051E+01 5.238120693E+00-1.485673552E+00-8.000251170E+00-1.328627499E+01
+-3.875338057E+00-2.150242329E+00-2.475329332E-01 1.867881232E+00 4.210692584E+00
+ 6.765911862E+00 9.478741185E+00 1.224492114E+01 1.490368573E+01 1.723627585E+01
+ 1.897356375E+01 1.981640216E+01 1.947142584E+01 1.770279405E+01 1.439651648E+01
+ 9.628673275E+00 3.722766155E+00-2.723810795E+00-8.866225270E+00-1.370959898E+01
+-3.900007644E+00-2.266644041E+00-4.677614828E-01 1.528196641E+00 3.732923227E+00
+ 6.129401538E+00 8.662340621E+00 1.122919771E+01 1.367387342E+01 1.578593608E+01
+ 1.730877932E+01 1.796012480E+01 1.746735709E+01 1.561793473E+01 1.232133201E+01
+ 7.673774441E+00 2.011195548E+00-4.069078223E+00-9.739068744E+00-1.404010861E+01
+-3.924023591E+00-2.398035334E+00-7.200067369E-01 1.137387716E+00 3.182410201E+00
+ 5.395913565E+00 7.722428677E+00 1.006196670E+01 1.226455371E+01 1.413022949E+01
+ 1.541772644E+01 1.586499240E+01 1.522375896E+01 1.330812440E+01 1.005342068E+01
+ 5.576636546E+00 2.241444128E-01-5.412036085E+00-1.052943124E+01-1.421705809E+01
+-3.949454495E+00-2.542502924E+00-9.979222946E-01 7.069235958E-01 2.576669018E+00
+ 4.590150437E+00 6.692214307E+00 8.786372669E+00 1.073023468E+01 1.233629223E+01
+ 1.338105163E+01 1.362538857E+01 1.284800990E+01 1.089170363E+01 7.718485117E+00
+ 3.464998876E+00-1.515610191E+00-6.643185284E+00-1.114998745E+01-1.418363094E+01
+-3.979256188E+00-2.698367977E+00-1.294887519E+00 2.489584160E-01 1.934239644E+00
+ 3.738116641E+00 5.606417678E+00 7.447191213E+00 9.127051556E+00 1.047271535E+01
+ 1.128046570E+01 1.133610480E+01 1.044694098E+01 8.485207794E+00 5.438904548E+00
+ 1.461602678E+00-3.091944768E+00-7.661071980E+00-1.152229191E+01-1.389187001E+01
+-4.016850914E+00-2.864044197E+00-1.604146536E+00-2.241112683E-01 1.273898011E+00
+ 2.865923368E+00 4.499590437E+00 6.088573689E+00 7.509837123E+00 8.605843241E+00
+ 9.194186653E+00 9.086875972E+00 8.120465300E+00 6.196168770E+00 3.325888584E+00
+-3.240034153E-01-4.403506209E+00-8.379786406E+00-1.158304470E+01-1.330709073E+01
+-4.065486440E+00-3.037734815E+00-1.918822347E+00-6.999816548E-01 6.139739607E-01
+ 1.998714410E+00 3.404587974E+00 4.751991854E+00 5.929458212E+00 6.796423679E+00
+ 7.192839193E+00 6.957494890E+00 5.955921640E+00 4.116771010E+00 1.472617199E+00
+-1.802565272E+00-5.370675409E+00-8.735316260E+00-1.128930792E+01-1.241144279E+01
+-4.127449814E+00-3.217015136E+00-2.231827841E+00-1.166667513E+00-2.819799411E-02
+ 1.159766364E+00 2.351277206E+00 3.474501101E+00 4.430577848E+00 5.096815722E+00
+ 5.336068777E+00 5.013819256E+00 4.023501957E+00 2.318783269E+00-5.116971402E-02
+-2.910644185E+00-5.940889782E+00-8.690299713E+00-1.062228866E+01-1.120634490E+01
+-4.203235039E+00-3.398366877E+00-2.535712222E+00-1.612632387E+00-6.366850978E-01
+ 3.697974838E-01 1.365540530E+00 2.287412293E+00 3.049966071E+00 3.548915381E+00
+ 3.670070593E+00 3.304913826E+00 2.373047603E+00 8.500896067E-01-1.202779494E+00
+-3.614186892E+00-6.091958209E+00-8.236839977E+00-9.589407101E+00-9.713596789E+00
+-4.290778163E+00-3.576742698E+00-2.822493099E+00-2.026849804E+00-1.197225471E+00
+-3.535002237E-01 4.686108471E-01 1.215428449E+00 1.815438588E+00 2.182902917E+00
+ 2.226163321E+00 1.861484244E+00 1.032396256E+00-2.669763141E-01-1.968150648E+00
+-3.909918455E+00-5.833148754E+00-7.397184068E+00-8.224491749E+00-7.975065939E+00
+-4.384875340E+00-3.745242651E+00-3.083530003E+00-2.398830761E+00-1.697378506E+00
+-9.957099940E-01-3.232539991E-01 2.762663994E-01 7.454546190E-01 1.016853765E+00
+ 1.020459654E+00 6.956632448E-01 7.347095482E-03-1.035422001E+00-2.361341617E+00
+-3.824490342E+00-5.203991097E+00-6.222222620E+00-6.586078401E+00-6.050948439E+00
+-4.476888583E+00-3.894978023E+00-3.309493489E+00-2.718656353E+00-2.126584605E+00
+-1.545741319E+00-9.987741727E-01-5.192520663E-01-1.506429331E-01 5.719217068E-02
+ 5.460577895E-02-1.978882146E-01-7.168054582E-01-1.481575987E+00-2.422003763E+00
+-3.411478000E+00-4.270900179E+00-4.787924263E+00-4.753925197E+00-4.016704838E+00
+-4.554820792E+00-4.015182505E+00-3.490475381E+00-2.977052429E+00-2.476167439E+00
+-1.995716824E+00-1.551366984E+00-1.166687615E+00-8.718068197E-01-7.000927896E-01
+-6.825136301E-01-8.396957198E-01-1.172330775E+00-1.651449890E+00-2.211061872E+00
+-2.746475866E+00-3.121883414E+00-3.189965874E+00-2.823984049E+00-1.958867057E+00
+-4.603807992E+00-4.093607026E+00-3.616270651E+00-3.165536031E+00-2.739312360E+00
+-2.340737292E+00-1.978638221E+00-1.667124626E+00-1.423984835E+00-1.267600894E+00
+-1.212274719E+00-1.262214897E+00-1.404973257E+00-1.605789134E+00-1.804951596E+00
+-1.920672159E+00-1.859729673E+00-1.536948455E+00-9.021790676E-01 3.000631406E-02
+-4.607035449E+00-4.117205045E+00-3.676841544E+00-3.276651652E+00-2.911050569E+00
+-2.578570662E+00-2.281713129E+00-2.026061651E+00-1.818469520E+00-1.664216751E+00
+-1.563240181E+00-1.505881440E+00-1.469052924E+00-1.414208492E+00-1.288862796E+00
+-1.033391503E+00-5.941846495E-01 5.731167636E-02 9.025721545E-01 1.856887103E+00
+-4.547039198E+00-4.073082299E+00-3.662951516E+00-3.304300534E+00-2.988270833E+00
+-2.709309505E+00-2.464479642E+00-2.252158241E+00-2.070047391E+00-1.912547284E+00
+-1.767768432E+00-1.614788936E+00-1.422144468E+00-1.148868356E+00-7.495037445E-01
+-1.841629701E-01 5.663134332E-01 1.481862283E+00 2.485604631E+00 3.433969848E+00
+ 1.000000000E+00 1.052063172E+00 1.110825146E+00 1.175313684E+00 1.246728314E+00
+ 1.329465325E+00 1.421360710E+00 1.524560763E+00 1.640637648E+00 1.780226739E+00
+ 1.940075364E+00 2.127842689E+00 2.354934271E+00 2.628982190E+00 2.978396113E+00
+ 3.421740456E+00 4.029874510E+00 4.918435235E+00 6.433511058E+00 8.575101979E+00
+   60   11
+ 1.400000000E+00 0.000000000E+00 1.394544521E+00 1.316756722E-01 1.378326897E+00
+ 2.597595754E-01 1.351789500E+00 3.807579144E-01 1.315656204E+00 4.913701702E-01
+ 1.270912629E+00 5.885791285E-01 1.218779263E+00 6.697331826E-01 1.160678170E+00
+ 7.326186613E-01 1.098194195E+00 7.755202128E-01 1.033031738E+00 7.972675944E-01
+ 9.669682618E-01 7.972675944E-01 9.018058051E-01 7.755202128E-01 8.393218301E-01
+ 7.326186613E-01 7.812207368E-01 6.697331826E-01 7.290873713E-01 5.885791285E-01
+ 6.843437962E-01 4.913701702E-01 6.482104995E-01 3.807579144E-01 6.216731033E-01
+ 2.597595754E-01 6.054554786E-01 1.316756722E-01 6.000000000E-01 9.797174393E-17
+ 6.000000000E-01 1.084348500E-16 6.002466000E-01-5.253082994E-02 6.009855315E-01
+-1.048766012E-01 6.022141911E-01-1.568529072E-01 6.039282505E-01-2.082766426E-01
+ 6.061216714E-01-2.589666491E-01 6.087867266E-01-3.087443528E-01 6.119140275E-01
+-3.574343941E-01 6.154925571E-01-4.048652448E-01 6.195097087E-01-4.508698128E-01
+ 6.239513305E-01-4.952860305E-01 6.288017752E-01-5.379574260E-01 6.340439555E-01
+-5.787336739E-01 6.396594039E-01-6.174711254E-01 6.456283379E-01-6.540333138E-01
+ 6.519297299E-01-6.882914358E-01 6.585413809E-01-7.201248050E-01 6.654399990E-01
+-7.494212768E-01 6.726012815E-01-7.760776440E-01 6.800000000E-01-8.000000000E-01
+ 6.800000000E-01-8.000000000E-01 7.465884669E-01-7.760776440E-01 8.110400088E-01
+-7.494212768E-01 8.731275722E-01-7.201248050E-01 9.326324313E-01-6.882914358E-01
+ 9.893449591E-01-6.540333138E-01 1.043065365E+00-6.174711254E-01 1.093604401E+00
+-5.787336739E-01 1.140784023E+00-5.379574260E-01 1.184438026E+00-4.952860305E-01
+ 1.224412622E+00-4.508698128E-01 1.260566986E+00-4.048652448E-01 1.292773752E+00
+-3.574343941E-01 1.320919461E+00-3.087443528E-01 1.344904957E+00-2.589666491E-01
+ 1.364645745E+00-2.082766426E-01 1.380072280E+00-1.568529072E-01 1.391130217E+00
+-1.048766012E-01 1.397780600E+00-5.253082994E-02 1.400000000E+00-2.168697000E-16
+ 5.200000000E-01 0.000000000E+00 5.800000000E-01-6.000000000E-01 5.500000000E-01-9.000000000E-01 1.250000000E+00-9.000000000E-01 1.400000000E+00-6.000000000E-01 1.480000000E+00 0.000000000E+00 1.400000000E+00 6.000000000E-01 1.250000000E+00 9.000000000E-01 5.500000000E-01 9.000000000E-01 5.800000000E-01 6.000000000E-01 5.200000000E-01 0.000000000E+00

tests/benchmarks/CMakeLists.txt

@@ -56,3 +56,4 @@ add_benchmark(elasticity)
 add_benchmark(tmop)
 add_benchmark(vector)
 add_benchmark(virtuals)
+add_benchmark(nlvc)

tests/benchmarks/bench_nlvc.cpp

@@ -0,0 +1,244 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "bench.hpp" // IWYU pragma: keep
+
+#ifdef MFEM_USE_BENCHMARK
+
+#include <cassert>
+#include <cstdlib>
+#include <functional>
+
+#include "fem/qinterp/grad.hpp"
+
+using namespace mfem;
+
+// Custom benchmark arguments generator ///////////////////////////////////////
+static void CustomArguments(bm::Benchmark *b) noexcept
+{
+   constexpr int MAX_NDOFS = 8 * 1024 * (mfem_use_gpu ? 1024 : 8);
+
+   const auto orders = { 6, 5, 4, 3, 2, 1 };
+
+   constexpr auto ndofs = [](int n) constexpr noexcept -> int
+   {
+      return (n + 1) * (n + 1) * (n + 1);
+   };
+
+   constexpr auto inc = [](int n) constexpr noexcept -> int
+   {
+      return n < 160 ?  4 : n < 240 ?  8 : n < 320 ? 16 : 32;
+   };
+
+   for (auto p : orders)
+   {
+      for (int n = (mfem_use_gpu ? 16 : 8); ndofs(n) <= MAX_NDOFS; n += inc(n))
+      {
+         b->Args({p, n});
+      }
+   }
+}
+
+/// Basic Kernels Specializations /////////////////////////////////////////////
+static void AddBasicKernelSpecializations()
+{
+   using Grad = QuadratureInterpolator::GradKernels;
+   // 2D
+   Grad::Specialization<2, QVectorLayout::byNODES, false, 2,2,7>::Add();
+   Grad::Specialization<2, QVectorLayout::byNODES, false, 2,2,8>::Add();
+   Grad::Specialization<2, QVectorLayout::byNODES, false, 2,2,10>::Add();
+   // 3D
+   Grad::Specialization<3, QVectorLayout::byNODES, false, 3,2,7>::Add();
+   Grad::Specialization<3, QVectorLayout::byNODES, false, 3,2,9>::Add();
+   Grad::Specialization<3, QVectorLayout::byNODES, false, 3,2,10>::Add();
+}
+
+/// VectorConvectionNLFBenchmark //////////////////////////////////////////////
+template <int DIM>
+struct VectorConvectionNLFBenchmark
+{
+   const int p, c, q, n, nx, ny, nz;
+   const std::function<Mesh()> MakeCartesianMesh = [&]()
+   {
+      if constexpr (DIM == 2)
+      {
+         return Mesh::MakeCartesian2D(nx, ny, Element::QUADRILATERAL);
+      }
+      else
+      {
+         return Mesh::MakeCartesian3D(nx, ny, nz, Element::HEXAHEDRON);
+      }
+   };
+   Mesh mesh;
+   H1_FECollection fec;
+   FiniteElementSpace fes;
+   const Geometry::Type geom_type;
+   IntegrationRules irs;
+   const IntegrationRule *ir;
+   ConstantCoefficient const_coeff { M_2_SQRTPI };
+   NonlinearFormIntegrator *nlfi;
+   NonlinearForm nlf;
+   Operator *grad;
+   GridFunction x, dx, y_pa;
+   Vector xe, dxe, ye;
+   const int dofs;
+   const int q1d;
+   double mdofs{};
+
+   VectorConvectionNLFBenchmark(int p, int side):
+      p(p), c(side), q(2 * p + 3), n((assert(c >= p), c / p)),
+      nx(n + (p * (n + 1) * p * n * p * n < c * c * c ? 1 : 0)),
+      ny(n + (p * (n + 1) * p * (n + 1) * p * n < c * c * c ? 1 : 0)), nz(n),
+      mesh(MakeCartesianMesh()),
+      fec(p, DIM),
+      fes(&mesh, &fec, DIM),
+      geom_type(mesh.GetTypicalElementGeometry()),
+      irs(0, Quadrature1D::GaussLegendre),
+      ir(&irs.Get(geom_type, q)),
+      nlfi(new VectorConvectionNLFIntegrator(const_coeff)),
+      nlf(&fes),
+      x(&fes),
+      dx(&fes),
+      y_pa(&fes),
+      dofs(fes.GetTrueVSize()),
+      q1d(IntRules.Get(Geometry::SEGMENT, ir->GetOrder()).GetNPoints())
+   {
+      MFEM_VERIFY(q1d*q1d*(DIM == 3 ? q1d : 1) == ir->GetNPoints(), "");
+
+      nlf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      nlf.AddDomainIntegrator(nlfi);
+      nlf.Setup();
+
+      dx.Randomize(0x9e3779b9), x.Randomize(0x100001b3);
+
+      grad = &nlf.GetGradient(x);
+
+      const Table &el2dof = fes.GetElementToDofTable();
+      const int e_size = el2dof.Size_of_connections()*fes.GetVDim();
+      const auto R = fes.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
+      MFEM_VERIFY(e_size == R->Height(), "Input/Output E-vector size mismatch!");
+      xe.SetSize(R->Height()), dxe.SetSize(R->Height()), ye.SetSize(R->Height());
+      xe.UseDevice(true), dxe.UseDevice(true), ye.UseDevice(true);
+      xe.Randomize(0x100001b3), dxe.Randomize(0x9e3779b9), ye = 0.0;
+
+      mdofs = 0.0;
+   }
+
+   void Setup()
+   {
+      nlfi->AssembleGradPA(xe, fes);
+      MFEM_DEVICE_SYNC;
+      mdofs += this->MDofs();
+   }
+
+   void AddMult()
+   {
+      nlf.AddMult(x, y_pa);
+      MFEM_DEVICE_SYNC;
+      mdofs += this->MDofs();
+   }
+
+   void AddMultPA()
+   {
+      nlfi->AddMultPA(xe, ye);
+      MFEM_DEVICE_SYNC;
+      mdofs += this->MDofs();
+   }
+
+   void AddMultGrad()
+   {
+      grad->Mult(dx, y_pa);
+      MFEM_DEVICE_SYNC;
+      mdofs += this->MDofs();
+   }
+
+   void AddMultGradPA()
+   {
+      nlfi->AddMultGradPA(dxe, ye);
+      MFEM_DEVICE_SYNC;
+      mdofs += this->MDofs();
+   }
+
+   void AssembleGradDiagonal()
+   {
+      grad->AssembleDiagonal(ye);
+      MFEM_DEVICE_SYNC;
+      mdofs += this->MDofs();
+   }
+
+   [[nodiscard]] double SumMdofs() const noexcept { return mdofs; }
+
+   [[nodiscard]] double MDofs() const noexcept { return 1e-6 * dofs; }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+#define RegisterVectorConvectionNLFBenchmark(Benchmark, DIM)         \
+   static void Benchmark##DIM##d(bm::State &state)                   \
+   {                                                                 \
+      const auto order = static_cast<int>(state.range(0));           \
+      const auto side = static_cast<int>(state.range(1));            \
+      VectorConvectionNLFBenchmark<DIM> ker(order, side);            \
+      while (state.KeepRunning()) { ker.Benchmark(); }               \
+      bm::Counter::Flags flags = bm::Counter::kIsRate;               \
+      state.counters["MDof/s"] = bm::Counter(ker.SumMdofs(), flags); \
+      state.counters["Dofs"] = bm::Counter(ker.dofs);                \
+      state.counters["p"] = bm::Counter(order);                      \
+   }                                                                 \
+   BENCHMARK(Benchmark##DIM##d)                                      \
+      ->Apply(CustomArguments)                                       \
+      ->Unit(bm::kMillisecond)
+
+RegisterVectorConvectionNLFBenchmark(Setup,3);
+RegisterVectorConvectionNLFBenchmark(AddMult,3);
+RegisterVectorConvectionNLFBenchmark(AddMultPA,3);
+RegisterVectorConvectionNLFBenchmark(AddMultGrad,3);
+RegisterVectorConvectionNLFBenchmark(AddMultGradPA,3);
+RegisterVectorConvectionNLFBenchmark(AssembleGradDiagonal,3);
+
+RegisterVectorConvectionNLFBenchmark(Setup,2);
+RegisterVectorConvectionNLFBenchmark(AddMult,2);
+RegisterVectorConvectionNLFBenchmark(AddMultPA,2);
+RegisterVectorConvectionNLFBenchmark(AddMultGrad,2);
+RegisterVectorConvectionNLFBenchmark(AddMultGradPA,2);
+RegisterVectorConvectionNLFBenchmark(AssembleGradDiagonal,2);
+
+/// main //////////////////////////////////////////////////////////////////////
+int main(int argc, char *argv[])
+{
+   AddBasicKernelSpecializations();
+
+   bm::ConsoleReporter CR;
+   bm::Initialize(&argc, argv);
+
+   // Device setup, cpu by default
+   std::string device_context = "cpu";
+   const auto global_context = bmi::GetGlobalContext();
+   if (global_context != nullptr)
+   {
+      const auto device = global_context->find("device");
+      if (device != global_context->end())
+      {
+         mfem::out << device->first << " : "
+                   << device->second << std::endl;
+         device_context = device->second;
+      }
+   }
+   Device device(device_context.c_str());
+   device.Print();
+
+   if (bm::ReportUnrecognizedArguments(argc, argv)) { return EXIT_FAILURE; }
+
+   bm::RunSpecifiedBenchmarks(&CR);
+
+   return EXIT_SUCCESS;
+}
+
+#endif // MFEM_USE_BENCHMARK

tests/benchmarks/makefile

@@ -21,7 +21,7 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 SEQ_TESTS = bench_assembly_levels bench_ceed bench_dg_amr bench_elasticity \
-            bench_tmop bench_vector bench_virtuals
+            bench_nlvc bench_tmop bench_vector bench_virtuals
 PAR_TESTS = 
 ifeq ($(MFEM_USE_MPI),NO)
    TESTS = $(SEQ_TESTS)

tests/unit/CMakeLists.txt

@@ -82,6 +82,7 @@ set(UNIT_TESTS_SRCS
   mesh/test_face_orientations.cpp
   mesh/test_fms.cpp
   mesh/test_geometric_factors.cpp
+  mesh/test_ho_rw.cpp
   mesh/test_mesh.cpp
   mesh/test_ncmesh.cpp
   mesh/test_nurbs.cpp
@@ -139,6 +140,7 @@ set(UNIT_TESTS_SRCS
   fem/test_lor_batched.cpp
   fem/test_lor_dg.cpp
   fem/test_lor.cpp
+  fem/test_mixedsesqform.cpp
   fem/test_nonlinearform.cpp
   fem/test_operatorjacobismoother.cpp
   fem/test_oscillation.cpp
@@ -147,6 +149,8 @@ set(UNIT_TESTS_SRCS
   fem/test_pa_grad.cpp
   fem/test_pa_idinterp.cpp
   fem/test_pa_kernels.cpp
+  fem/test_pa_nlvc.cpp
+  fem/test_pa_vecdiv.cpp
   fem/test_pa_simplices.cpp
   fem/test_particleset.cpp
   fem/test_pgridfunc_save_serial.cpp
@@ -165,6 +169,20 @@ set(UNIT_TESTS_SRCS
   fem/test_transfer.cpp
   fem/test_var_order.cpp
   fem/test_white_noise.cpp
+  fem/specializations/test_diffusion_integ.cpp
+  fem/specializations/test_mass_integ.cpp
+  fem/specializations/test_convection_integ.cpp
+  fem/specializations/test_vecmass_integ.cpp
+  fem/specializations/test_curlcurl_integ.cpp
+  fem/specializations/test_vecdiffusion_integ.cpp
+  fem/specializations/test_dgtrace_integ.cpp
+  fem/specializations/test_dgdiffusion_integ.cpp
+  fem/specializations/test_dgmassinv.cpp
+  fem/specializations/test_qinterp_det.cpp
+  fem/specializations/test_qinterp_eval.cpp
+  fem/specializations/test_qinterp_grad.cpp
+  fem/specializations/test_qinterp_tensoreval.cpp
+  fem/specializations/test_qinterp_eval_hdiv.cpp
   enzyme/compatibility.cpp
   # The following are tested separately (keep the comment as a reminder).
   # This list can be updated using (in bash):

tests/unit/data/hex-nc-cross.mesh

@@ -0,0 +1,379 @@
+MFEM NC mesh v1.0
+
+# NCMesh supported geometry types:
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+
+dimension
+3
+
+rank
+0
+
+# rank attr geom ref_type nodes/children
+elements
+75
+0 1 5 0 0 1 5 4 16 17 21 20
+0 1 5 0 16 17 21 20 32 33 37 36
+-1 1 5 7 59 60 61 62 63 64 65 66
+0 1 5 0 1 2 6 5 17 18 22 21
+-1 1 5 7 27 28 29 30 31 32 33 34
+0 1 5 0 21 22 26 25 37 38 42 41
+-1 1 5 7 43 44 45 46 47 48 49 50
+0 1 5 0 4 5 9 8 20 21 25 24
+0 1 5 0 8 9 13 12 24 25 29 28
+0 1 5 0 24 25 29 28 40 41 45 44
+0 1 5 0 9 10 14 13 25 26 30 29
+-1 1 5 7 67 68 69 70 71 72 73 74
+0 1 5 0 41 42 46 45 57 58 62 61
+0 1 5 0 40 41 45 44 56 57 61 60
+0 1 5 0 36 37 41 40 52 53 57 56
+-1 1 5 7 35 36 37 38 39 40 41 42
+0 1 5 0 32 33 37 36 48 49 53 52
+0 1 5 0 33 34 38 37 49 50 54 53
+0 1 5 0 34 35 39 38 50 51 55 54
+0 1 5 0 38 39 43 42 54 55 59 58
+0 1 5 0 42 43 47 46 58 59 63 62
+0 1 5 0 26 27 31 30 42 43 47 46
+0 1 5 0 10 11 15 14 26 27 31 30
+0 1 5 0 6 7 11 10 22 23 27 26
+-1 1 5 7 51 52 53 54 55 56 57 58
+0 1 5 0 18 19 23 22 34 35 39 38
+0 1 5 0 2 3 7 6 18 19 23 22
+0 1 5 0 5 94 208 99 74 209 214 212
+0 1 5 0 94 6 97 208 209 96 210 214
+0 1 5 0 208 97 10 98 214 210 103 211
+0 1 5 0 99 208 98 9 212 214 211 104
+0 1 5 0 74 209 214 212 21 86 213 102
+0 1 5 0 209 96 210 214 86 22 100 213
+0 1 5 0 214 210 103 211 213 100 26 101
+0 1 5 0 212 214 211 104 102 213 101 25
+0 1 5 0 37 89 269 107 156 270 275 273
+0 1 5 0 89 38 105 269 270 159 271 275
+0 1 5 0 269 105 42 106 275 271 144 272
+0 1 5 0 107 269 106 41 273 275 272 143
+0 1 5 0 156 270 275 273 53 157 274 153
+0 1 5 0 270 159 271 275 157 54 158 274
+0 1 5 0 275 271 144 272 274 158 58 139
+0 1 5 0 273 275 272 143 153 274 139 57
+0 1 5 0 20 70 330 111 83 331 336 334
+0 1 5 0 70 21 102 330 331 82 332 336
+0 1 5 0 330 102 25 110 336 332 109 333
+0 1 5 0 111 330 110 24 334 336 333 114
+0 1 5 0 83 331 336 334 36 78 335 113
+0 1 5 0 331 82 332 336 78 37 107 335
+0 1 5 0 336 332 109 333 335 107 41 112
+0 1 5 0 334 336 333 114 113 335 112 40
+0 1 5 0 22 198 387 100 91 388 393 391
+0 1 5 0 198 23 199 387 388 201 389 393
+0 1 5 0 387 199 27 186 393 389 189 390
+0 1 5 0 100 387 186 26 391 393 390 108
+0 1 5 0 91 388 393 391 38 170 392 105
+0 1 5 0 388 201 389 393 170 39 176 392
+0 1 5 0 393 389 189 390 392 176 43 177
+0 1 5 0 391 393 390 108 105 392 177 42
+0 1 5 0 17 84 444 69 81 445 450 448
+0 1 5 0 84 18 85 444 445 90 446 450
+0 1 5 0 444 85 22 86 450 446 91 447
+0 1 5 0 69 444 86 21 448 450 447 82
+0 1 5 0 81 445 450 448 33 87 449 77
+0 1 5 0 445 90 446 450 87 34 88 449
+0 1 5 0 450 446 91 447 449 88 38 89
+0 1 5 0 448 450 447 82 77 449 89 37
+0 1 5 0 25 101 497 121 109 498 503 501
+0 1 5 0 101 26 133 497 498 108 499 503
+0 1 5 0 497 133 30 134 503 499 138 500
+0 1 5 0 121 497 134 29 501 503 500 129
+0 1 5 0 109 498 503 501 41 106 502 126
+0 1 5 0 498 108 499 503 106 42 136 502
+0 1 5 0 503 499 138 500 502 136 46 137
+0 1 5 0 501 503 500 129 126 502 137 45
+
+# attr geom nodes
+boundary
+72
+1 3 4 5 1 0
+1 3 0 1 17 16
+1 3 4 0 16 20
+1 3 16 17 33 32
+1 3 20 16 32 36
+1 3 5 6 2 1
+1 3 1 2 18 17
+1 3 8 9 5 4
+1 3 8 4 20 24
+1 3 12 13 9 8
+1 3 13 12 28 29
+1 3 12 8 24 28
+1 3 29 28 44 45
+1 3 28 24 40 44
+1 3 13 14 10 9
+1 3 14 13 29 30
+1 3 46 45 61 62
+1 3 57 58 62 61
+1 3 45 44 60 61
+1 3 44 40 56 60
+1 3 56 57 61 60
+1 3 40 36 52 56
+1 3 52 53 57 56
+1 3 32 33 49 48
+1 3 36 32 48 52
+1 3 48 49 53 52
+1 3 33 34 50 49
+1 3 49 50 54 53
+1 3 34 35 51 50
+1 3 35 39 55 51
+1 3 50 51 55 54
+1 3 39 43 59 55
+1 3 54 55 59 58
+1 3 43 47 63 59
+1 3 47 46 62 63
+1 3 58 59 63 62
+1 3 27 31 47 43
+1 3 31 30 46 47
+1 3 14 15 11 10
+1 3 11 15 31 27
+1 3 15 14 30 31
+1 3 10 11 7 6
+1 3 7 11 27 23
+1 3 18 19 35 34
+1 3 19 23 39 35
+1 3 6 7 3 2
+1 3 2 3 19 18
+1 3 3 7 23 19
+2 3 99 208 94 5
+2 3 208 97 6 94
+2 3 98 10 97 208
+2 3 9 98 208 99
+2 3 53 157 274 153
+2 3 157 54 158 274
+2 3 274 158 58 139
+2 3 153 274 139 57
+2 3 111 20 83 334
+2 3 24 111 334 114
+2 3 334 83 36 113
+2 3 114 334 113 40
+2 3 23 199 389 201
+2 3 199 27 189 389
+2 3 201 389 176 39
+2 3 389 189 43 176
+2 3 17 84 445 81
+2 3 84 18 90 445
+2 3 81 445 87 33
+2 3 445 90 34 87
+2 3 30 134 500 138
+2 3 134 29 129 500
+2 3 138 500 137 46
+2 3 500 129 45 137
+
+# vert_id p1 p2
+vertex_parents
+102
+69 17 21
+70 20 21
+74 5 21
+77 33 37
+78 36 37
+81 17 33
+82 21 37
+83 20 36
+84 17 18
+85 18 22
+86 21 22
+87 33 34
+88 34 38
+89 37 38
+90 18 34
+91 22 38
+94 5 6
+96 6 22
+97 6 10
+98 9 10
+99 5 9
+100 22 26
+101 25 26
+102 21 25
+103 10 26
+104 9 25
+105 38 42
+106 41 42
+107 37 41
+108 26 42
+109 25 41
+110 24 25
+111 20 24
+112 40 41
+113 36 40
+114 24 40
+121 25 29
+126 41 45
+129 29 45
+133 26 30
+134 29 30
+136 42 46
+137 45 46
+138 30 46
+139 57 58
+143 41 57
+144 42 58
+153 53 57
+156 37 53
+157 53 54
+158 54 58
+159 38 54
+170 38 39
+176 39 43
+177 42 43
+186 26 27
+189 27 43
+198 22 23
+199 23 27
+201 23 39
+208 97 99
+209 74 96
+210 96 103
+211 103 104
+212 74 104
+213 100 102
+214 209 211
+269 105 107
+270 156 159
+271 144 159
+272 143 144
+273 143 156
+274 153 158
+275 270 272
+330 102 111
+331 82 83
+332 82 109
+333 109 114
+334 83 114
+335 107 113
+336 331 333
+387 100 199
+388 91 201
+389 189 201
+390 108 189
+391 91 108
+392 105 176
+393 388 390
+444 69 85
+445 81 90
+446 90 91
+447 82 91
+448 81 82
+449 77 88
+450 445 447
+497 121 133
+498 108 109
+499 108 138
+500 129 138
+501 109 129
+502 126 136
+503 498 500
+
+# root element orientation
+root_state
+27
+0
+1
+1
+15
+15
+6
+6
+22
+15
+8
+12
+10
+10
+18
+18
+13
+7
+22
+22
+15
+16
+16
+16
+7
+8
+6
+21
+
+# top-level node coordinates
+coordinates
+64
+3
+0 0 0
+0.33333333 0 0
+0.66666667 0 0
+1 0 0
+0 0.33333333 0
+0.33333333 0.33333333 0
+0.66666667 0.33333333 0
+1 0.33333333 0
+0 0.66666667 0
+0.33333333 0.66666667 0
+0.66666667 0.66666667 0
+1 0.66666667 0
+0 1 0
+0.33333333 1 0
+0.66666667 1 0
+1 1 0
+0 0 0.33333333
+0.33333333 0 0.33333333
+0.66666667 0 0.33333333
+1 0 0.33333333
+0 0.33333333 0.33333333
+0.33333333 0.33333333 0.33333333
+0.66666667 0.33333333 0.33333333
+1 0.33333333 0.33333333
+0 0.66666667 0.33333333
+0.33333333 0.66666667 0.33333333
+0.66666667 0.66666667 0.33333333
+1 0.66666667 0.33333333
+0 1 0.33333333
+0.33333333 1 0.33333333
+0.66666667 1 0.33333333
+1 1 0.33333333
+0 0 0.66666667
+0.33333333 0 0.66666667
+0.66666667 0 0.66666667
+1 0 0.66666667
+0 0.33333333 0.66666667
+0.33333333 0.33333333 0.66666667
+0.66666667 0.33333333 0.66666667
+1 0.33333333 0.66666667
+0 0.66666667 0.66666667
+0.33333333 0.66666667 0.66666667
+0.66666667 0.66666667 0.66666667
+1 0.66666667 0.66666667
+0 1 0.66666667
+0.33333333 1 0.66666667
+0.66666667 1 0.66666667
+1 1 0.66666667
+0 0 1
+0.33333333 0 1
+0.66666667 0 1
+1 0 1
+0 0.33333333 1
+0.33333333 0.33333333 1
+0.66666667 0.33333333 1
+1 0.33333333 1
+0 0.66666667 1
+0.33333333 0.66666667 1
+0.66666667 0.66666667 1
+1 0.66666667 1
+0 1 1
+0.33333333 1 1
+0.66666667 1 1
+1 1 1
+
+mfem_mesh_end

tests/unit/data/tet-nc-cross.mesh

@@ -0,0 +1,555 @@
+MFEM NC mesh v1.0
+
+# NCMesh supported geometry types:
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+
+dimension
+3
+
+rank
+0
+
+# rank attr geom ref_type nodes/children
+elements
+258
+0 1 4 0 21 0 5 1
+0 1 4 0 21 0 1 17
+0 1 4 0 21 0 17 16
+0 1 4 0 21 0 4 5
+0 1 4 0 21 0 20 4
+0 1 4 0 21 0 16 20
+0 1 4 0 22 1 6 2
+0 1 4 0 22 1 2 18
+0 1 4 0 22 1 18 17
+0 1 4 0 22 1 5 6
+0 1 4 0 22 1 21 5
+0 1 4 0 22 1 17 21
+0 1 4 0 23 2 7 3
+0 1 4 0 23 2 3 19
+0 1 4 0 23 2 19 18
+0 1 4 0 23 2 6 7
+0 1 4 0 23 2 22 6
+0 1 4 0 23 2 18 22
+0 1 4 0 25 4 9 5
+0 1 4 0 25 4 5 21
+0 1 4 0 25 4 21 20
+0 1 4 0 25 4 8 9
+0 1 4 0 25 4 24 8
+0 1 4 0 25 4 20 24
+-1 1 4 7 170 171 172 173 174 175 176 177
+0 1 4 0 26 5 6 22
+0 1 4 0 26 5 22 21
+-1 1 4 7 162 163 164 165 166 167 168 169
+0 1 4 0 26 5 25 9
+0 1 4 0 26 5 21 25
+0 1 4 0 27 6 11 7
+0 1 4 0 27 6 7 23
+0 1 4 0 27 6 23 22
+0 1 4 0 27 6 10 11
+0 1 4 0 27 6 26 10
+0 1 4 0 27 6 22 26
+0 1 4 0 29 8 13 9
+0 1 4 0 29 8 9 25
+0 1 4 0 29 8 25 24
+0 1 4 0 29 8 12 13
+0 1 4 0 29 8 28 12
+0 1 4 0 29 8 24 28
+0 1 4 0 30 9 14 10
+0 1 4 0 30 9 10 26
+0 1 4 0 30 9 26 25
+0 1 4 0 30 9 13 14
+0 1 4 0 30 9 29 13
+0 1 4 0 30 9 25 29
+0 1 4 0 31 10 15 11
+0 1 4 0 31 10 11 27
+0 1 4 0 31 10 27 26
+0 1 4 0 31 10 14 15
+0 1 4 0 31 10 30 14
+0 1 4 0 31 10 26 30
+0 1 4 0 37 16 21 17
+0 1 4 0 37 16 17 33
+0 1 4 0 37 16 33 32
+0 1 4 0 37 16 20 21
+0 1 4 0 37 16 36 20
+0 1 4 0 37 16 32 36
+0 1 4 0 38 17 22 18
+-1 1 4 7 226 227 228 229 230 231 232 233
+-1 1 4 7 234 235 236 237 238 239 240 241
+0 1 4 0 38 17 21 22
+0 1 4 0 38 17 37 21
+0 1 4 0 38 17 33 37
+0 1 4 0 39 18 23 19
+0 1 4 0 39 18 19 35
+0 1 4 0 39 18 35 34
+0 1 4 0 39 18 22 23
+0 1 4 0 39 18 38 22
+0 1 4 0 39 18 34 38
+0 1 4 0 41 20 25 21
+0 1 4 0 41 20 21 37
+0 1 4 0 41 20 37 36
+0 1 4 0 41 20 24 25
+-1 1 4 7 202 203 204 205 206 207 208 209
+-1 1 4 7 194 195 196 197 198 199 200 201
+0 1 4 0 42 21 26 22
+0 1 4 0 42 21 22 38
+0 1 4 0 42 21 38 37
+0 1 4 0 42 21 25 26
+0 1 4 0 42 21 41 25
+0 1 4 0 42 21 37 41
+-1 1 4 7 210 211 212 213 214 215 216 217
+-1 1 4 7 218 219 220 221 222 223 224 225
+0 1 4 0 43 22 39 38
+0 1 4 0 43 22 26 27
+0 1 4 0 43 22 42 26
+0 1 4 0 43 22 38 42
+0 1 4 0 45 24 29 25
+0 1 4 0 45 24 25 41
+0 1 4 0 45 24 41 40
+0 1 4 0 45 24 28 29
+0 1 4 0 45 24 44 28
+0 1 4 0 45 24 40 44
+0 1 4 0 46 25 30 26
+0 1 4 0 46 25 26 42
+0 1 4 0 46 25 42 41
+-1 1 4 7 250 251 252 253 254 255 256 257
+-1 1 4 7 242 243 244 245 246 247 248 249
+0 1 4 0 46 25 41 45
+0 1 4 0 47 26 31 27
+0 1 4 0 47 26 27 43
+0 1 4 0 47 26 43 42
+0 1 4 0 47 26 30 31
+0 1 4 0 47 26 46 30
+0 1 4 0 47 26 42 46
+0 1 4 0 53 32 37 33
+0 1 4 0 53 32 33 49
+0 1 4 0 53 32 49 48
+0 1 4 0 53 32 36 37
+0 1 4 0 53 32 52 36
+0 1 4 0 53 32 48 52
+0 1 4 0 54 33 38 34
+0 1 4 0 54 33 34 50
+0 1 4 0 54 33 50 49
+0 1 4 0 54 33 37 38
+0 1 4 0 54 33 53 37
+0 1 4 0 54 33 49 53
+0 1 4 0 55 34 39 35
+0 1 4 0 55 34 35 51
+0 1 4 0 55 34 51 50
+0 1 4 0 55 34 38 39
+0 1 4 0 55 34 54 38
+0 1 4 0 55 34 50 54
+0 1 4 0 57 36 41 37
+0 1 4 0 57 36 37 53
+0 1 4 0 57 36 53 52
+0 1 4 0 57 36 40 41
+0 1 4 0 57 36 56 40
+0 1 4 0 57 36 52 56
+0 1 4 0 58 37 42 38
+0 1 4 0 58 37 38 54
+-1 1 4 7 178 179 180 181 182 183 184 185
+0 1 4 0 58 37 41 42
+0 1 4 0 58 37 57 41
+-1 1 4 7 186 187 188 189 190 191 192 193
+0 1 4 0 59 38 43 39
+0 1 4 0 59 38 39 55
+0 1 4 0 59 38 55 54
+0 1 4 0 59 38 42 43
+0 1 4 0 59 38 58 42
+0 1 4 0 59 38 54 58
+0 1 4 0 61 40 45 41
+0 1 4 0 61 40 41 57
+0 1 4 0 61 40 57 56
+0 1 4 0 61 40 44 45
+0 1 4 0 61 40 60 44
+0 1 4 0 61 40 56 60
+0 1 4 0 62 41 46 42
+0 1 4 0 62 41 42 58
+0 1 4 0 62 41 58 57
+0 1 4 0 62 41 45 46
+0 1 4 0 62 41 61 45
+0 1 4 0 62 41 57 61
+0 1 4 0 63 42 47 43
+0 1 4 0 63 42 43 59
+0 1 4 0 63 42 59 58
+0 1 4 0 63 42 46 47
+0 1 4 0 63 42 62 46
+0 1 4 0 63 42 58 62
+0 1 4 0 26 125 132 126
+0 1 4 0 125 5 115 128
+0 1 4 0 132 115 9 133
+0 1 4 0 126 128 133 10
+0 1 4 0 125 133 132 126
+0 1 4 0 125 133 126 128
+0 1 4 0 125 133 128 115
+0 1 4 0 125 133 115 132
+0 1 4 0 26 125 126 127
+0 1 4 0 125 5 128 95
+0 1 4 0 126 128 10 129
+0 1 4 0 127 95 129 6
+0 1 4 0 125 129 126 127
+0 1 4 0 125 129 127 95
+0 1 4 0 125 129 95 128
+0 1 4 0 125 129 128 126
+0 1 4 0 58 305 308 309
+0 1 4 0 305 37 283 262
+0 1 4 0 308 283 54 284
+0 1 4 0 309 262 284 53
+0 1 4 0 305 284 308 309
+0 1 4 0 305 284 309 262
+0 1 4 0 305 284 262 283
+0 1 4 0 305 284 283 308
+0 1 4 0 58 305 309 311
+0 1 4 0 305 37 262 297
+0 1 4 0 309 262 53 298
+0 1 4 0 311 297 298 57
+0 1 4 0 305 298 309 311
+0 1 4 0 305 298 311 297
+0 1 4 0 305 298 297 262
+0 1 4 0 305 298 262 309
+0 1 4 0 41 213 217 219
+0 1 4 0 213 20 191 220
+0 1 4 0 217 191 36 222
+0 1 4 0 219 220 222 40
+0 1 4 0 213 222 217 219
+0 1 4 0 213 222 219 220
+0 1 4 0 213 222 220 191
+0 1 4 0 213 222 191 217
+0 1 4 0 41 213 219 218
+0 1 4 0 213 20 220 124
+0 1 4 0 219 220 40 221
+0 1 4 0 218 124 221 24
+0 1 4 0 213 221 219 218
+0 1 4 0 213 221 218 124
+0 1 4 0 213 221 124 220
+0 1 4 0 213 221 220 219
+0 1 4 0 43 230 231 232
+0 1 4 0 230 22 141 110
+0 1 4 0 231 141 27 140
+0 1 4 0 232 110 140 23
+0 1 4 0 230 140 231 232
+0 1 4 0 230 140 232 110
+0 1 4 0 230 140 110 141
+0 1 4 0 230 140 141 231
+0 1 4 0 43 230 232 233
+0 1 4 0 230 22 110 211
+0 1 4 0 232 110 23 204
+0 1 4 0 233 211 204 39
+0 1 4 0 230 204 232 233
+0 1 4 0 230 204 233 211
+0 1 4 0 230 204 211 110
+0 1 4 0 230 204 110 232
+0 1 4 0 38 193 195 196
+0 1 4 0 193 17 93 197
+0 1 4 0 195 93 18 198
+0 1 4 0 196 197 198 34
+0 1 4 0 193 198 195 196
+0 1 4 0 193 198 196 197
+0 1 4 0 193 198 197 93
+0 1 4 0 193 198 93 195
+0 1 4 0 38 193 196 199
+0 1 4 0 193 17 197 184
+0 1 4 0 196 197 34 200
+0 1 4 0 199 184 200 33
+0 1 4 0 193 200 196 199
+0 1 4 0 193 200 199 184
+0 1 4 0 193 200 184 197
+0 1 4 0 193 200 197 196
+0 1 4 0 46 247 253 252
+0 1 4 0 247 25 239 150
+0 1 4 0 253 239 45 238
+0 1 4 0 252 150 238 29
+0 1 4 0 247 238 253 252
+0 1 4 0 247 238 252 150
+0 1 4 0 247 238 150 239
+0 1 4 0 247 238 239 253
+0 1 4 0 46 247 252 248
+0 1 4 0 247 25 150 165
+0 1 4 0 252 150 29 168
+0 1 4 0 248 165 168 30
+0 1 4 0 247 168 252 248
+0 1 4 0 247 168 248 165
+0 1 4 0 247 168 165 150
+0 1 4 0 247 168 150 252
+
+# attr geom nodes
+boundary
+144
+1 2 0 5 1
+1 2 0 1 17
+1 2 0 17 16
+1 2 0 4 5
+1 2 0 20 4
+1 2 0 16 20
+1 2 1 6 2
+1 2 1 2 18
+1 2 1 18 17
+1 2 1 5 6
+1 2 2 7 3
+1 2 23 3 7
+1 2 2 3 19
+1 2 23 19 3
+1 2 2 19 18
+1 2 2 6 7
+1 2 4 9 5
+1 2 4 8 9
+1 2 4 24 8
+1 2 4 20 24
+1 2 6 11 7
+1 2 27 7 11
+1 2 27 23 7
+1 2 6 10 11
+1 2 8 13 9
+1 2 8 12 13
+1 2 29 13 12
+1 2 8 28 12
+1 2 29 12 28
+1 2 8 24 28
+1 2 9 14 10
+1 2 9 13 14
+1 2 30 14 13
+1 2 30 13 29
+1 2 10 15 11
+1 2 31 11 15
+1 2 31 27 11
+1 2 10 14 15
+1 2 31 15 14
+1 2 31 14 30
+1 2 16 17 33
+1 2 16 33 32
+1 2 16 36 20
+1 2 16 32 36
+1 2 39 19 23
+1 2 18 19 35
+1 2 39 35 19
+1 2 18 35 34
+1 2 45 29 28
+1 2 24 44 28
+1 2 45 28 44
+1 2 24 40 44
+1 2 47 27 31
+1 2 47 43 27
+1 2 47 31 30
+1 2 47 30 46
+1 2 32 33 49
+1 2 32 49 48
+1 2 53 48 49
+1 2 32 52 36
+1 2 32 48 52
+1 2 53 52 48
+1 2 33 34 50
+1 2 33 50 49
+1 2 54 49 50
+1 2 54 53 49
+1 2 55 35 39
+1 2 34 35 51
+1 2 55 51 35
+1 2 34 51 50
+1 2 55 50 51
+1 2 55 54 50
+1 2 57 52 53
+1 2 36 56 40
+1 2 36 52 56
+1 2 57 56 52
+1 2 59 39 43
+1 2 59 55 39
+1 2 59 54 55
+1 2 59 58 54
+1 2 61 56 57
+1 2 61 45 44
+1 2 40 60 44
+1 2 61 44 60
+1 2 40 56 60
+1 2 61 60 56
+1 2 62 57 58
+1 2 62 46 45
+1 2 62 45 61
+1 2 62 61 57
+1 2 63 43 47
+1 2 63 59 43
+1 2 63 58 59
+1 2 63 47 46
+1 2 63 46 62
+1 2 63 62 58
+2 2 5 115 128
+2 2 115 9 133
+2 2 128 133 10
+2 2 133 128 115
+2 2 5 128 95
+2 2 128 10 129
+2 2 95 129 6
+2 2 129 95 128
+2 2 58 309 308
+2 2 308 284 54
+2 2 309 53 284
+2 2 284 308 309
+2 2 58 311 309
+2 2 309 298 53
+2 2 311 57 298
+2 2 298 309 311
+2 2 20 191 220
+2 2 191 36 222
+2 2 220 222 40
+2 2 222 220 191
+2 2 20 220 124
+2 2 220 40 221
+2 2 124 221 24
+2 2 221 124 220
+2 2 43 232 231
+2 2 231 140 27
+2 2 232 23 140
+2 2 140 231 232
+2 2 43 233 232
+2 2 232 204 23
+2 2 233 39 204
+2 2 204 232 233
+2 2 17 93 197
+2 2 93 18 198
+2 2 197 198 34
+2 2 198 197 93
+2 2 17 197 184
+2 2 197 34 200
+2 2 184 200 33
+2 2 200 184 197
+2 2 46 252 253
+2 2 253 238 45
+2 2 252 29 238
+2 2 238 253 252
+2 2 46 248 252
+2 2 252 168 29
+2 2 248 30 168
+2 2 168 252 248
+
+# vert_id p1 p2
+vertex_parents
+54
+93 17 18
+95 5 6
+110 22 23
+115 5 9
+124 20 24
+125 5 26
+126 10 26
+127 6 26
+128 5 10
+129 6 10
+132 9 26
+133 9 10
+140 23 27
+141 22 27
+150 25 29
+165 25 30
+168 29 30
+184 17 33
+191 20 36
+193 17 38
+195 18 38
+196 34 38
+197 17 34
+198 18 34
+199 33 38
+200 33 34
+204 23 39
+211 22 39
+213 20 41
+217 36 41
+218 24 41
+219 40 41
+220 20 40
+221 24 40
+222 36 40
+230 22 43
+231 27 43
+232 23 43
+233 39 43
+238 29 45
+239 25 45
+247 25 46
+248 30 46
+252 29 46
+253 45 46
+262 37 53
+283 37 54
+284 53 54
+297 37 57
+298 53 57
+305 37 58
+308 54 58
+309 53 58
+311 57 58
+
+# top-level node coordinates
+coordinates
+64
+3
+0 0 0
+0.33333333 0 0
+0.66666667 0 0
+1 0 0
+0 0.33333333 0
+0.33333333 0.33333333 0
+0.66666667 0.33333333 0
+1 0.33333333 0
+0 0.66666667 0
+0.33333333 0.66666667 0
+0.66666667 0.66666667 0
+1 0.66666667 0
+0 1 0
+0.33333333 1 0
+0.66666667 1 0
+1 1 0
+0 0 0.33333333
+0.33333333 0 0.33333333
+0.66666667 0 0.33333333
+1 0 0.33333333
+0 0.33333333 0.33333333
+0.33333333 0.33333333 0.33333333
+0.66666667 0.33333333 0.33333333
+1 0.33333333 0.33333333
+0 0.66666667 0.33333333
+0.33333333 0.66666667 0.33333333
+0.66666667 0.66666667 0.33333333
+1 0.66666667 0.33333333
+0 1 0.33333333
+0.33333333 1 0.33333333
+0.66666667 1 0.33333333
+1 1 0.33333333
+0 0 0.66666667
+0.33333333 0 0.66666667
+0.66666667 0 0.66666667
+1 0 0.66666667
+0 0.33333333 0.66666667
+0.33333333 0.33333333 0.66666667
+0.66666667 0.33333333 0.66666667
+1 0.33333333 0.66666667
+0 0.66666667 0.66666667
+0.33333333 0.66666667 0.66666667
+0.66666667 0.66666667 0.66666667
+1 0.66666667 0.66666667
+0 1 0.66666667
+0.33333333 1 0.66666667
+0.66666667 1 0.66666667
+1 1 0.66666667
+0 0 1
+0.33333333 0 1
+0.66666667 0 1
+1 0 1
+0 0.33333333 1
+0.33333333 0.33333333 1
+0.66666667 0.33333333 1
+1 0.33333333 1
+0 0.66666667 1
+0.33333333 0.66666667 1
+0.66666667 0.66666667 1
+1 0.66666667 1
+0 1 1
+0.33333333 1 1
+0.66666667 1 1
+1 1 1
+
+mfem_mesh_end

tests/unit/fem/specializations/test_convection_integ.cpp

@@ -0,0 +1,14 @@
+/// Tests which make sure adding user-defined kernel specializations work.
+/// These tests are compile/link-only tests
+
+#include "mfem.hpp"
+#include "unit_tests.hpp"
+
+#include "fem/integ/bilininteg_convection_kernels.hpp"
+
+TEST_CASE("Convection Kernel Specializations", "[Specializations]")
+{
+   using namespace mfem;
+
+   ConvectionIntegrator::AddSpecialization<2, 2, 4>();
+}

tests/unit/fem/specializations/test_curlcurl_integ.cpp

@@ -0,0 +1,14 @@
+/// Tests which make sure adding user-defined kernel specializations work.
+/// These tests are compile/link-only tests
+
+#include "mfem.hpp"
+#include "unit_tests.hpp"
+
+#include "fem/integ/bilininteg_hcurl_kernels.hpp"
+
+TEST_CASE("CurlCurl Kernel Specializations", "[Specializations]")
+{
+   using namespace mfem;
+
+   CurlCurlIntegrator::AddSpecialization<3, 2, 4>();
+}